def get_bbox(model, images):
inp_dim = int(model.net_info["height"])
#If there's a GPU availible, put the model on GPU
if CUDA:
model.cuda()
#Set the model in evaluation mode
model.eval()
# print("creating py torch variables")
#PyTorch Variables for images
im_batches = list(
map(prep_image, images, [inp_dim for x in range(len(images))]))
#List containing dimensions of original images
im_dim_list = [(x.shape[1], x.shape[0]) for x in images]
im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2)
# print("creating im dim list")
if CUDA:
im_dim_list = im_dim_list.cuda()
leftover = 0
if (len(im_dim_list) % batch_size):
leftover = 1
if batch_size != 1:
num_batches = len(images) // batch_size + leftover
im_batches = [
torch.cat(
(im_batches[i * batch_size:min((i + 1) *
batch_size, len(im_batches))]))
for i in range(num_batches)
]
# print("creating prediction shit")
bboxes = []
classes_detected = []
class_names = []
lengths = []
for i, batch in enumerate(im_batches):
if CUDA:
batch = batch.cuda()
prediction = model(Variable(batch, requires_grad=False), CUDA)
prediction = write_results(prediction,
confidence,
num_classes,
nms_conf=nms_thresh)
# result has
# index in batch
# 4 box coordinates
# object scorness
# the score for the class
# maximum confidence
# and the index of the class
curr_image_bbox = torch.Tensor([[0, 0, 0, 0]])
curr_image_class = torch.Tensor([-1])
if type(prediction) != int:
curr_image_bbox = prediction[:, 1:
5] # [output[1:5] for output in prediction]
curr_image_class = prediction[:,
-1] # torch.FloatTensor([output[-1] for output in prediction])
elif CUDA:
curr_image_bbox = curr_image_bbox.cuda()
curr_image_class = curr_image_class.cuda()
class_names.append([
classes[int(id)] if id != -1 else "<unk>"
for id in curr_image_class
])
bboxes.append(curr_image_bbox)
classes_detected.append(curr_image_class)
lengths.append(len(curr_image_class))
return class_names, bboxes, lengths
def forward(self, im_data, im_info, gt_boxes, num_boxes, support_ims, all_cls_gt_boxes):
im_batch = im_data
im_info = im_info.data
gt_boxes = gt_boxes.data
num_boxes = num_boxes.data
support_ims = support_ims.data
batch_size = im_batch.size(0)
x = self.resnet_base(im_batch)
pos_sup_ims = support_ims[:, :self.num_shot, :, :, :].contiguous()
pos_sup_feats = self.resnet_base(pos_sup_ims.view(
batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80]
if self.training:
neg_sup_ims = support_ims[:, self.num_shot:, :, :, :].contiguous()
neg_sup_feats = self.resnet_base(neg_sup_ims.view(
batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80]
x1 = self.layer1(x)
x2 = self.layer2(x1)
x3 = self.layer3(x2)
x4 = self.layer4(x3)
pos_s1 = self.layer1(pos_sup_feats)
pos_s2 = self.layer2(pos_s1)
pos_s3 = self.layer3(pos_s2)
pos_s4 = self.layer4(pos_s3)
if self.training:
neg_s1 = self.layer1(neg_sup_feats)
neg_s2 = self.layer2(neg_s1)
neg_s3 = self.layer3(neg_s2)
neg_s4 = self.layer4(neg_s3)
features = self.fpn([x2, x3, x4])[1:]
pos_s_features = self.fpn([pos_s2, pos_s3, pos_s4])[1:]
if self.training:
neg_s_features = self.fpn([neg_s2, neg_s3, neg_s4])[1:]
correlation_features = self.attention_module(features, pos_s_features)
if self.training:
neg_correlation_features = self.attention_module(features, neg_s_features)
regression = torch.cat([self.regressionModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, 4]
classification = torch.cat([self.classificationModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, n_class]
if self.training:
neg_regression = torch.cat([self.regressionModel(feat) for feat in neg_correlation_features], dim=1)
neg_classification = torch.cat([self.classificationModel(feat) for feat in neg_correlation_features], dim=1)
anchors = self.anchors(im_batch)
rpn_loss_cls = torch.zeros(1).cuda()
rpn_loss_bbox = torch.zeros(1).cuda()
RCNN_loss_cls = torch.zeros(1).cuda()
RCNN_loss_bbox = torch.zeros(1).cuda()
rois_label = torch.zeros(10).cuda()
rois = None
cls_prob = None
bbox_pred = None
if self.training:
pos_RCNN_loss_cls, RCNN_loss_bbox = self.focalLoss(classification, regression, anchors, gt_boxes)
empty_gt_boxes = torch.zeros_like(gt_boxes) - 1.
neg_RCNN_loss_cls, _ = self.focalLoss(neg_classification, neg_regression, anchors, empty_gt_boxes)
RCNN_loss_cls = (pos_RCNN_loss_cls + neg_RCNN_loss_cls) / 2
return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label
else:
transformed_anchors = self.regressBoxes(anchors, regression)
transformed_anchors = self.clipBoxes(transformed_anchors, im_batch)
finalResult = [[], [], []]
finalScores = torch.Tensor([])
finalAnchorBoxesIndexes = torch.Tensor([]).long()
finalAnchorBoxesCoordinates = torch.Tensor([])
if torch.cuda.is_available():
finalScores = finalScores.cuda()
finalAnchorBoxesIndexes = finalAnchorBoxesIndexes.cuda()
finalAnchorBoxesCoordinates = finalAnchorBoxesCoordinates.cuda()
# scores = torch.squeeze(classification[:, :, 1])
scores = F.softmax(classification[0], 1)[:, 1]
scores_over_thresh = (scores > 0.5)
if scores_over_thresh.sum() == 0:
return None, None, np.array([0., 0., 0., 0.])
scores = scores[scores_over_thresh]
anchorBoxes = torch.squeeze(transformed_anchors)
anchorBoxes = anchorBoxes[scores_over_thresh]
anchors_nms_idx = nms(anchorBoxes, scores, 0.5)
finalScores = torch.cat((finalScores, scores[anchors_nms_idx])) # [n_predict]
finalAnchorBoxesIndexesValue = torch.tensor([1] * anchors_nms_idx.shape[0])
if torch.cuda.is_available():
finalAnchorBoxesIndexesValue = finalAnchorBoxesIndexesValue.cuda()
finalAnchorBoxesIndexes = torch.cat((finalAnchorBoxesIndexes, finalAnchorBoxesIndexesValue)) # [n_predict]
finalAnchorBoxesCoordinates = torch.cat((finalAnchorBoxesCoordinates, anchorBoxes[anchors_nms_idx])) # [n_predict, 4]
return [finalScores, finalAnchorBoxesIndexes, finalAnchorBoxesCoordinates]
torch.cuda.manual_seed(args.seed)
if torch.cuda.is_available() and not args.cuda:
print("Warning: You have Cuda but not use it. You are using CPU for training.")
# Set up the data for training
TEXT = data.Field(lower=True)
ED = data.Field()
train, dev, test = SQdataset.splits(TEXT, ED, args.data_dir)
TEXT.build_vocab(train, dev, test)
ED.build_vocab(train, dev, test)
match_embedding = 0
if os.path.isfile(args.vector_cache):
stoi, vectors, dim = torch.load(args.vector_cache)
TEXT.vocab.vectors = torch.Tensor(len(TEXT.vocab), dim)
for i, token in enumerate(TEXT.vocab.itos):
wv_index = stoi.get(token, None)
if wv_index is not None:
TEXT.vocab.vectors[i] = vectors[wv_index]
match_embedding += 1
else:
TEXT.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.25, 0.25)
else:
print("Error: Need word embedding pt file")
exit(1)
print("Embedding match number {} out of {}".format(match_embedding, len(TEXT.vocab)))
train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False,
sort=False, shuffle=True, sort_within_batch=False)
def attack(self, iterations, target_path, lr, bandwidth):
flag = 0
model = WaveToLetter.load_model(args.model_path)
model = model.to(device)
# model.eval()
signal = self.get_signal(self.audio_path)
orig = self.get_signal(self.orig_path)
# alpha, index_max, index_min= self.attention(signal)
start_attack_time = time.time()
for i in range(iterations):
print('Iteration:', str(i))
# print(signal[index_max:index_max+20])
# print(signal[index_max])
# if args.printSilence:
# print()
# print(sigindex_maxnal.shape)
# print("20：", signal[index_max:index_max+20])
mfcc = pytorch_mfcc.MFCC(samplerate=self.sample_rate,winlen=self.window_size,winstep=self.window_stride,numcep=13,nfilt=26,nfft=512,lowfreq=0,highfreq=None,preemph=0,ceplifter=22,appendEnergy=False).cuda()
mfccs = mfcc(signal)
mfccs = self.normalize(mfccs)
if args.printSilence:
print("mfccs", mfccs)
inputsMags = self.mfccs_to_inputs(mfccs)
out = model(inputsMags)
path = self.orig_path.split('wav')[0] + 'txt' + self.orig_path.split('wav')[1]
fp=open(path)
transcriptReal=fp.readlines()[0]
print("Ref:",transcriptReal.lower())
seq_length = out.size(1)
sizes = Variable(torch.Tensor([1.0]).mul_(int(seq_length)).int(), requires_grad=False)
decoded_output, _, = decoder.decode(out.data, sizes)
transcript = decoded_output[0][0]
print("Hyp:", transcript.lower())
out = out.transpose(0, 1)
if args.target:
transcriptTarget = args.target
else:
fp=open(target_path)
transcriptTarget=fp.readlines()[0]
print("Tar:", transcriptTarget.lower())
if transcript.lower()==transcriptTarget.lower() and i>0:
if args.target:
target_path = args.target
save_path = self.save(signal, target_path, self.orig_path, lr, iterations, i, bandwidth)
generate_time = time.time() - start_attack_time
print('Time taken (s): {generate_time:.4f}\t'.format(generate_time=generate_time))
self.save_figure(signal, save_path)
break
target = list(filter(None, [self.labels_map.get(x) for x in list(transcriptTarget.upper())]))
targets = torch.IntTensor(target)
target_sizes = torch.IntTensor([len(target)])
ctcloss = self.criterion(out, targets, sizes, target_sizes)
# print("ctcloss:", ctcloss)
# print("delta_2:", 100*torch.sum((signal - orig)**2))
# loss = ctcloss + 100*torch.sum((signal - orig)**2)
loss = ctcloss
print("loss:", loss)
loss.backward()
grad = np.array(signal.grad)
is_nan = np.isnan(grad)
is_nan_new = is_nan[is_nan == True]
for j in range(len(grad)):
if is_nan[j]:
grad[j] = 100
wer = decoder.wer(transcript.lower(), transcriptTarget.lower()) / float(len(transcriptTarget.lower().split()))
# the iterative proportional clipping method
# print('grad:{}'.format(grad[index_max:index_max+20]))
perturbation = lr*torch.from_numpy(grad)
# print('perturbation', perturbation[index_max])
# print((signal.data - perturbation)[index_max])
# print("signal_next_relative0", ((signal.data - perturbation)/orig)[index_max])
signal_next_relative = torch.clamp((signal.data - perturbation)/orig, min = 1 - bandwidth, max = 1 + bandwidth)
# print("signal_next_relative1:", signal_next_relative[index_max])
signal.data = signal_next_relative.mul(orig)
# print(signal_next_relative[index_max]*orig[index_max])
# print("signal.data:", signal.data[index_max])
# if (i + 1) % 10000 == 0:
# # anneal lr
# # lr *= 0.5
# lr = lr / args.learning_anneal
# print("wer", wer)
# print("lr", lr)
print("\n")
signal.grad.data.zero_()
print("Come to the end")
def position_matrix(J, d):
m = torch.Tensor(J, d)
for k in range(1, d + 1):
for j in range(1, J + 1):
m[j - 1, k - 1] = (1 - j / J) - (k / d) * (1 - 2 * j / J)
return m
def set_stddev(self, stddev):
self.stddev.data = torch.Tensor(stddev).to(self.stddev)
# save to file
np.save(os.path.join(dat_dir, 'Zp.npy'), np.array(Zp))
import corner as DFM
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
fig = DFM.corner(np.array(Zp)[:,:10])
fig.savefig(os.path.join(dat_dir, 'Zp.corner.png'), bbox_inches='tight')
'''
for i in range(100):
n_val = 10000
noise_y = torch.Tensor(n_val, np.sum(central_pixel)).normal_()
noise_x = torch.Tensor(n_val, np.sum(~central_pixel)).normal_()
p_Y_model.eval()
p_XgivenY_model.eval()
with torch.no_grad():
# sample Y' ~ p_NF(Y)
Yp = p_Y_model.sample(n_val, noise=noise_y).detach().cpu()
# sample X' ~ p_NF(X|Y')
Xp = p_XgivenY_model.sample(n_val, noise=noise_x,
cond_inputs=Yp).detach().cpu()
XpYp = torch.from_numpy(np.zeros((n_val, 28 * 28))).float()
XpYp[:, ~central_pixel] = Xp
XpYp[:, central_pixel] = Yp
def beam_search(self, data: dict):
encoder_feats = data.get('img').contiguous()
device = encoder_feats.device
beam_width = self.beam_width
# eos = '[s]'
eos = self.converter.dict['[s]']
def _inflate(tensor, times, dim):
##所有维度都重复一遍
repeat_dims = [1] * tensor.dim()
#在需要的维度重复times
repeat_dims[dim] = times
return tensor.repeat(*repeat_dims)
# https://github.com/IBM/pytorch-seq2seq/blob/fede87655ddce6c94b38886089e05321dc9802af/seq2seq/models/TopKDecoder.py
batch_size, seq_len, seq_dim = encoder_feats.size()
# inflated_encoder_feats = _inflate(encoder_feats, beam_width, 0) # ABC --> AABBCC -/-> ABCABC
inflated_encoder_feats = encoder_feats.unsqueeze(1)\
.permute((1, 0, 2, 3)).repeat((beam_width, 1, 1, 1)).permute(
(1, 0, 2, 3)).contiguous().view(-1, seq_len, seq_dim)
# Initialize the decoder
decoder_state = torch.zeros(1, batch_size * beam_width,
self.hidden_dim).to(device)
pos_index = (torch.Tensor(range(batch_size)) * beam_width).long().view(
-1, 1).to(device)
# Initialize the scores
sequence_scores = torch.Tensor(batch_size * beam_width, 1).to(device)
sequence_scores.fill_(-float('Inf'))
sequence_scores.index_fill_(
0,
torch.Tensor([i * beam_width
for i in range(0, batch_size)]).long().to(device),
0.0)
# sequence_scores.fill_(0.0)
# Initialize the input vector
y_prev = torch.zeros(
(batch_size * beam_width)).fill_(self.num_classes).to(device)
# Store decisions for backtracking
#(T,batch * beam_width,1)
stored_scores_tensor = torch.FloatTensor(self.max_len_labels + 1,
batch_size * beam_width,
1).fill_(0).to(device)
stored_predecessors_tensor = torch.LongTensor(self.max_len_labels + 1,
batch_size * beam_width,
1).fill_(0).to(device)
stored_emitted_symbols_tensor = torch.LongTensor(
self.max_len_labels + 1,
batch_size * beam_width).fill_(0).to(device)
# stored_scores = list()
# stored_predecessors = list()
# stored_emitted_symbols = list()
# +1 for [s] at end of sentence.
for i in range(self.max_len_labels + 1):
output, decoder_state = self.decoder(inflated_encoder_feats,
decoder_state, y_prev)
log_softmax_output = F.log_softmax(output, dim=1)
sequence_scores = _inflate(sequence_scores, self.num_classes, 1)
sequence_scores += log_softmax_output
#每一步选topk个
scores, candidates = sequence_scores.view(batch_size,
-1).topk(beam_width,
dim=1)
# Reshape input = (bk, 1) and sequence_scores = (bk, 1)
y_prev = (candidates % self.num_classes).view(batch_size *
beam_width)
sequence_scores = scores.view(batch_size * beam_width, 1)
# Update fields for next timestep
#torch 1.5.1 floor_divide
# predecessors = (candidates.floor_divide(self.num_classes) + pos_index.expand_as(candidates)).view(batch_size* beam_width,1)
# torch 1.3.1 /
predecessors = (candidates / self.num_classes +
pos_index.expand_as(candidates)).view(
batch_size * beam_width, 1)
decoder_state = decoder_state.index_select(1,
predecessors.squeeze())
# Update sequence socres and erase scores for <eos> symbol so that they aren't expanded
# stored_scores.append(sequence_scores.clone())
stored_scores_tensor[i] = sequence_scores.clone()
eos_indices = y_prev.view(-1, 1).eq(eos)
if eos_indices.nonzero().dim() > 0:
sequence_scores.masked_fill_(eos_indices, -float('inf'))
# Cache results for backtracking
# stored_predecessors.append(predecessors)
stored_predecessors_tensor[i] = predecessors
stored_emitted_symbols_tensor[i] = y_prev
# stored_emitted_symbols.append(y_prev)
# Do backtracking to return the optimal values
# ====== backtrak ======#
# Initialize return variables given different types
#反向搜索
p = []
l = [[self.max_len_labels] * beam_width for _ in range(batch_size)]
# the last step output of the beams are not sorted
# thus they are sorted here
# sorted_score,sorted_idx = stored_scores[-1].view(batch_size,beam_width).topk(beam_width)
# s = sorted_score.clone()
sorted_score, sorted_idx = stored_scores_tensor[-1].view(
batch_size, beam_width).topk(beam_width)
s = sorted_score.clone()
batch_eos_found = [0] * batch_size
# +1 for [s] at end of sentence.
#t = self.max_len_labels -1
t = self.max_len_labels
t_predecessors = (sorted_idx + pos_index.expand_as(sorted_idx)).view(
batch_size * beam_width)
while t >= 0:
# Re-order the variables with the back pointer
# current_symbol = stored_emitted_symbols[t].index_select(0,t_predecessors)
# t_predecessors = stored_predecessors[t].index_select(0,t_predecessors).squeeze()
# eos_indices = stored_emitted_symbols[t].eq(eos).nonzero()
current_symbol = stored_emitted_symbols_tensor[t].index_select(
0, t_predecessors)
t_predecessors = stored_predecessors_tensor[t].index_select(
0, t_predecessors).squeeze()
eos_indices = stored_emitted_symbols_tensor[t].eq(eos).nonzero()
if eos_indices.dim() > 0:
for i in range(eos_indices.size(0) - 1, -1, -1):
# Indices of the EOS symbol for both variables
# with b*k as the first dimension, and b, k for
# the first two dimensions
idx = eos_indices[i]
b_index = int(idx[0] / beam_width)
# The indices of the replacing position
# according to the replacement strategy noted above
res_k_idx = beam_width - (batch_eos_found[b_index] %
beam_width) - 1
batch_eos_found[b_index] += 1
res_idx = b_index * beam_width + res_k_idx
# Replace the old information in return variables
# with the new ended sequence information
t_predecessors[res_idx] = stored_predecessors_tensor[t][
idx[0]]
current_symbol[res_idx] = stored_emitted_symbols_tensor[t][
idx[0]]
s[b_index, res_k_idx] = stored_scores_tensor[t][idx[0],
[0]]
l[b_index][res_k_idx] = t + 1
# record the back tracked results
p.append(current_symbol)
t -= 1
# Sort and re-order again as the added ended sequences may change
# the order (very unlikely)
s, re_sorted_idx = s.topk(beam_width)
for b_index in range(batch_size):
l[b_index] = [
l[b_index][k_idx.item()] for k_idx in re_sorted_idx[b_index, :]
]
re_sorted_idx = (re_sorted_idx +
pos_index.expand_as(re_sorted_idx)).view(batch_size *
beam_width)
# Reverse the sequences and re-order at the same time
# It is reversed because the backtracking happens in reverse time order
p = [
step.index_select(0, re_sorted_idx).view(batch_size, beam_width,
-1)
for step in reversed(p)
]
rec_pred = torch.cat(p, -1)[:, 0, :]
return rec_pred
#print(inp['adj'].size())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = gcn_resnet101(num_classes=150, t=0.4, adj_file='food_adj.pkl')
model.load_state_dict(torch.load("model_best_87.7752.pth.tar")['state_dict'])
model = model.to(device)
model.eval()
normalize = transforms.Normalize(mean=model.image_normalization_mean,
std=model.image_normalization_std)
transform = transforms.Compose([
Warp('448'),
transforms.ToTensor(),
normalize,
])
fff = open('food_glove_word2vec.pkl', 'rb')
inp = pickle.load(fff)
inp = torch.Tensor(inp)
inp = inp.unsqueeze(0)
inp = inp.to(device)
'''
tempa = open("./train_anno.json","r")
tempb = open("./val_anno.json","r")
a = json.load(tempa)
b = json.load(tempb)
qqq=[]
www=[]
with tqdm(total=len(a)) as pbar:
for aaaa in a:
aaa = Image.open("./coco/data/train2014/"+aaaa['file_name']).convert('RGB')
def evaluate(experiment_directory, checkpoint_path):
chamfer_results = []
specs = ws.load_experiment_specifications(experiment_directory)
logging.info("Experiment description: \n" + specs["Description"])
data_source = specs["DataSource"]
test_split_file = specs["TestSplit"]
num_samp_per_scene = specs["SamplesPerScene"]
scene_per_batch = specs["ScenesPerBatch"]
clamp_dist = specs["ClampingDistance"]
minT = -clamp_dist
maxT = clamp_dist
enforce_minmax = False
num_data_loader_threads =1
# scene_per_subbatch =1
batch_split = 1
scene_per_subbatch = scene_per_batch // batch_split
checkpoints = list(
range(
specs["SnapshotFrequency"],
specs["NumEpochs"] + 1,
specs["SnapshotFrequency"],
)
)
def signal_handler(sig, frame):
logging.info("Stopping early...")
sys.exit(0)
with open(test_split_file,"r") as f:
test_split = json.load(f)
sdf_dataset = dt_vtk.SDFVTKSamples(
data_source, test_split, num_samp_per_scene
)
sdf_loader = data_utils.DataLoader(
sdf_dataset,
batch_size=scene_per_subbatch,
shuffle=True,
num_workers=num_data_loader_threads,
drop_last=True,
)
decoder_eval = decoder.Decoder(0, **specs["NetworkSpecs"]).cuda()
# for epoch in range(start_epoch, num_epochs + 1):
# start = time.time()
# logging.info("epoch {}...".format(epoch))
# pdb.set_trace()
checkpoint = torch.load(checkpoint_path)
decoder_eval.load_state_dict(checkpoint['model_state_dict'])
decoder_eval = decoder_eval.float()
# decoder_eval.eval()
for param in decoder_eval.parameters():
param.requires_grad = False
loss_l1 = torch.nn.L1Loss()
loss_l2 = torch.nn.MSELoss()
loss_log =[]
# theta_x = torch.randn(1, requires_grad=True, dtype=torch.float)*3.1415
# theta_y = torch.randn(1, requires_grad=True, dtype=torch.float)*3.1415
# theta_z = torch.randn(1, requires_grad=True, dtype=torch.float)*3.1415
# theta_x = theta_x.float()
# theta_y = theta_y.float()
# theta_z = theta_z.float()
# theta_x.retain_grad()
# theta_y.retain_grad()
# theta_z.retain_grad()
scale_one = torch.randn(1, requires_grad=True, dtype=torch.float)
scale_two = torch.randn(1, requires_grad=True, dtype=torch.float)
scale_three = torch.randn(1, requires_grad=True, dtype=torch.float)
scale_one.retain_grad()
scale_two.retain_grad()
scale_three.retain_grad()
# transform_matrix = torch.zeros(3,3).float().cuda()
# transform_matrix.requires_grad_(True)
# transform_matrix.retain_grad()
transform_inpt = torch.randn(3,3).float().cuda()
transform_inpt.requires_grad_(True)
transform_inpt.retain_grad()
bias = torch.zeros(3).float().cuda()
bias.requires_grad_(True)
bias.retain_grad()
# pdb.set_trace()
test_model = np.array(pd.read_csv("../chairs_segdata/points/1a8bbf2994788e2743e99e0cae970928.pts", header=None,sep=" ").values)
num_epochs = 500
learning_rate = 1e-3
test_pts = torch.from_numpy(test_model).float()
test_pts.requies_grad = False
bt_size = 32
num_batches = int(test_model.shape[0]//bt_size)
sub = torch.Tensor([1]).cuda()
reg = 1
# rot_x = torch.from_numpy(rotate_mat_x()).double().cuda()
# rot_x.requires_grad_(True)
# rot_y = torch.from_numpy(rotate_mat_y()).double().cuda()
# rot_y.requires_grad_(True)
# rot_z = torch.from_numpy(rotate_mat_z()).double().cuda()
# rot_z.requires_grad_(True)
with torch.enable_grad():
for j in range(num_epochs):
# pdb.set_trace()
# Process the input datag
# sdf_data.requires_grad = False
# sdf_data = (sdf_data.cuda()).reshape(
# num_samp_per_scene * scene_per_subbatch, 4
# )
# xyz = sdf_data[:, 0:3]
# transform_matrix_update = torch.add(transform_matrix,bias)
batch_loss=0
for i in range(num_batches):
test_torch = test_pts[i*bt_size:(i+1)*bt_size,:]
# pdb.set_trace()
# cosval_x = torch.cos(theta_x)
# sinval_x = torch.sin(theta_x)
# cosval_x.requires_grad_(True)
# sinval_x.requires_grad_(True)
# cosval_x.retain_grad()
# sinval_x.retain_grad()
# rot_x = torch.stack([torch.Tensor([1, 0, 0]),
# torch.cat([torch.Tensor([0]), cosval_x, -sinval_x]),
# torch.cat([torch.Tensor([0]), sinval_x, cosval_x])], dim=1).float().cuda()
# rot_x.requires_grad_(True)
# rot_x.retain_grad()
# cosval_y = torch.cos(theta_y)
# sinval_y = torch.sin(theta_y)
# cosval_y.requires_grad_(True)
# sinval_y.requires_grad_(True)
# cosval_y.retain_grad()
# sinval_y.retain_grad()
# rot_y = torch.stack([torch.cat([cosval_y, torch.Tensor([0]), sinval_y]),
# torch.Tensor([0, 1, 0]),
# torch.cat([-sinval_y, torch.Tensor([0]), cosval_y])],dim=1).float().cuda()
# rot_y.requires_grad_(True)
# rot_y.retain_grad()
# cosval_z = torch.cos(theta_z)
# sinval_z = torch.sin(theta_z)
# cosval_z.requires_grad_(True)
# sinval_z.requires_grad_(True)
# cosval_z.retain_grad()
# sinval_z.retain_grad()
# rot_z = torch.stack([torch.cat([cosval_z, -sinval_z, torch.Tensor([0])]),
# torch.cat([sinval_z, cosval_z, torch.Tensor([0])]),
# torch.Tensor([0, 0, 1])], dim=1).float().cuda()
# rot_z.requires_grad_(True)
# rot_z.retain_grad()
scale_matrix = torch.cat([torch.cat([scale_one,torch.Tensor([0]),torch.Tensor([0])]),
torch.cat([torch.Tensor([0]),scale_two,torch.Tensor([0])]),
torch.cat([torch.Tensor([0]),torch.Tensor([0]),scale_three])]).view(3,3).float().cuda()
# pdb.set_trace()
scale_matrix.retain_grad()
scale_matrix.requires_grad_(True)
# transform_matrix = torch.matmul(torch.matmul(torch.matmul(rot_z,rot_y),rot_x),scale_matrix)
transform_matrix = torch.matmul(transform_inpt, scale_matrix)
transform_matrix.requires_grad_(True)
transform_matrix.retain_grad()
xyz = test_torch.cuda()
xyz_transform = torch.matmul(xyz, transform_matrix)
xyz_transform.requires_grad_(True)
xyz_transform.retain_grad()
transform_bias = torch.add(xyz_transform, bias).float()
transform_bias.retain_grad()
# diag_sum = torch.abs(torch.sum(torch.diag(transform_matrix)))
# sdf_gt = sdf_data[:, 3].unsqueeze(1)
pred_sdf = decoder_eval(transform_bias)
# pred_sdf = decoder_eval(xyz_transform)
# loss = loss_l1(pred_sdf, sdf_gt)
target = torch.zeros(pred_sdf.shape[0],pred_sdf.shape[1]).float().cuda()
# batch_loss += loss.item()
# pdb.set_trace()
diag_sum = torch.norm(torch.sub(torch.diag(scale_matrix),sub),2)
diag_sum.retain_grad()
diag_sum.requires_grad_(True)
# diag_sum = torch.sum(torch.diag(transform_matrix)).cpu()
loss1 = loss_l1(pred_sdf,target)
loss2 = reg *diag_sum
# loss2 = torch.abs(torch.sub(diag_sum,1))
loss = torch.add(loss1,loss2)
loss.backward(retain_graph=True)
batch_loss+= loss.item()
print('Batch Loss {:6.4f}'.format(loss.item()))
with torch.no_grad():
# theta_z.data.sub_(theta_z.grad.data*learning_rate)
# theta_y.data.sub_(theta_y.data*learning_rate)
# theta_x.data.sub_(theta_x.grad.data*learning_rate)
bias.data.sub_(bias.grad.data*learning_rate)
scale_one.data.sub_(scale_one.grad.data*learning_rate)
scale_two.data.sub_(scale_two.grad.data*learning_rate)
scale_three.data.sub_(scale_three.grad.data*learning_rate)
transform_inpt.data.sub_(transform_inpt.grad.data*learning_rate)
# theta_z.grad.data.zero_()
# theta_y.grad.data.zero_()
# theta_x.grad.data.zero_()
bias.grad.data.zero_()
scale_one.grad.data.zero_()
scale_three.grad.data.zero_()
scale_two.grad.data.zero_()
scale_matrix.grad.data.zero_()
transform_bias.grad.data.zero_()
xyz_transform.grad.data.zero_()
transform_matrix.grad.data.zero_()
transform_inpt.grad.data.zero_()
diag_sum.grad.data.zero_()
# rot_z.grad.data.zero_()
# rot_x.grad.data.zero_()
# rot_y.grad.data.zero_()
# pdb.set_trace()
actual_loss = (batch_loss*bt_size)/(test_model.shape[0])
# print("Loss after {} epoch is {:6.4f}".format(j,batch_loss))
print("Loss after {} epoch is {:6.4f}".format(j,actual_loss))
loss_log.append(actual_loss)
pdb.set_trace()
fig,ax = plt.subplots()
ax.plot(np.arange(num_epochs),loss_log)
ax.set(xlabel='iterations',ylabel='transformationloss')
plt.savefig('Transformation_loss_new.png')
torch.save(transform_matrix,'transform_matrix_new.pt')
torch.save(bias,'bias_new.pt')
test_pts = torch.from_numpy(pd.read_csv('test_model.pts',header=None, sep=' ').values).cuda()
transform_pts = torch.matmul(test_pts, transform_matrix.double())
transform_pts = torch.add(transform_pts, bias.double()).cpu().detach().numpy()
np.savetxt('transform_points_new.pts',transform_pts)
plot_heatmap(experiment_directory, checkpoint_path)
def batchify_sentence_classification_with_label(input_batch_list,
gpu,
if_train=True):
"""
input: list of words, chars and labels, various length. [[words, features, chars, labels],[words, features, chars,labels],...]
words: word ids for one sentence. (batch_size, sent_len)
features: features ids for one sentence. (batch_size, feature_num), each sentence has one set of feature
chars: char ids for on sentences, various length. (batch_size, sent_len, each_word_length)
labels: label ids for one sentence. (batch_size,), each sentence has one set of feature
output:
zero padding for word and char, with their batch length
word_seq_tensor: (batch_size, max_sent_len) Variable
feature_seq_tensors: [(batch_size,), ... ] list of Variable
word_seq_lengths: (batch_size,1) Tensor
char_seq_tensor: (batch_size*max_sent_len, max_word_len) Variable
char_seq_lengths: (batch_size*max_sent_len,1) Tensor
char_seq_recover: (batch_size*max_sent_len,1) recover char sequence order
label_seq_tensor: (batch_size, )
mask: (batch_size, max_sent_len)
"""
batch_size = len(input_batch_list)
words = [sent[0] for sent in input_batch_list]
features = [np.asarray(sent[1]) for sent in input_batch_list]
feature_num = len(features[0])
chars = [sent[2] for sent in input_batch_list]
labels = [sent[3] for sent in input_batch_list]
word_seq_lengths = torch.LongTensor(list(map(len, words)))
max_seq_len = word_seq_lengths.max().item()
word_seq_tensor = torch.zeros((batch_size, max_seq_len),
requires_grad=if_train).long()
label_seq_tensor = torch.zeros((batch_size, ),
requires_grad=if_train).long()
feature_seq_tensors = []
for idx in range(feature_num):
feature_seq_tensors.append(
torch.zeros((batch_size, max_seq_len),
requires_grad=if_train).long())
mask = torch.zeros((batch_size, max_seq_len),
requires_grad=if_train).byte()
label_seq_tensor = torch.LongTensor(labels)
# exit(0)
for idx, (seq, seqlen) in enumerate(zip(words, word_seq_lengths)):
seqlen = seqlen.item()
word_seq_tensor[idx, :seqlen] = torch.LongTensor(seq)
mask[idx, :seqlen] = torch.Tensor([1] * seqlen)
for idy in range(feature_num):
feature_seq_tensors[idy][idx, :seqlen] = torch.LongTensor(
features[idx][:, idy])
word_seq_lengths, word_perm_idx = word_seq_lengths.sort(0, descending=True)
word_seq_tensor = word_seq_tensor[word_perm_idx]
for idx in range(feature_num):
feature_seq_tensors[idx] = feature_seq_tensors[idx][word_perm_idx]
label_seq_tensor = label_seq_tensor[word_perm_idx]
mask = mask[word_perm_idx]
### deal with char
# pad_chars (batch_size, max_seq_len)
pad_chars = [
chars[idx] + [[0]] * (max_seq_len - len(chars[idx]))
for idx in range(len(chars))
]
length_list = [list(map(len, pad_char)) for pad_char in pad_chars]
max_word_len = max(map(max, length_list))
char_seq_tensor = torch.zeros((batch_size, max_seq_len, max_word_len),
requires_grad=if_train).long()
char_seq_lengths = torch.LongTensor(length_list)
for idx, (seq, seqlen) in enumerate(zip(pad_chars, char_seq_lengths)):
for idy, (word, wordlen) in enumerate(zip(seq, seqlen)):
# print len(word), wordlen
char_seq_tensor[idx, idy, :wordlen] = torch.LongTensor(word)
char_seq_tensor = char_seq_tensor[word_perm_idx].view(
batch_size * max_seq_len, -1)
char_seq_lengths = char_seq_lengths[word_perm_idx].view(
batch_size * max_seq_len, )
char_seq_lengths, char_perm_idx = char_seq_lengths.sort(0, descending=True)
char_seq_tensor = char_seq_tensor[char_perm_idx]
_, char_seq_recover = char_perm_idx.sort(0, descending=False)
_, word_seq_recover = word_perm_idx.sort(0, descending=False)
if gpu:
word_seq_tensor = word_seq_tensor.cuda()
for idx in range(feature_num):
feature_seq_tensors[idx] = feature_seq_tensors[idx].cuda()
word_seq_lengths = word_seq_lengths.cuda()
word_seq_recover = word_seq_recover.cuda()
label_seq_tensor = label_seq_tensor.cuda()
char_seq_tensor = char_seq_tensor.cuda()
char_seq_recover = char_seq_recover.cuda()
mask = mask.cuda()
return word_seq_tensor, feature_seq_tensors, word_seq_lengths, word_seq_recover, char_seq_tensor, char_seq_lengths, char_seq_recover, label_seq_tensor, mask
def update(self):
img = Imgdata(
'dataset/json_data/posetrack_train.json',
'/media/jion/D/chenhaoming/DataSet/PoseTrack2017/posetrack_data/',
self.index)
images, bbox, key_gt, floder_path = img.get_Info()
im = images
gt_key = img.data_adjustment(key_gt)
path_current = '/media/jion/D/chenhaoming/DataSet/PoseTrack2017/posetrack_data/' + floder_path + '/'
trans_martix = utils.crop_img(
path_current=path_current, images=images,
bbox=bbox) #Generate new data in origin floder
#reshape data
gt_key_mid = []
trans = []
bboxes = []
# heatmaps_after = []
imgs_after = [[] for i in range(len(images))]
trans_img = []
for m in range(len(images)):
if len(bbox[m][0]) != 0:
for num_box in range(len(bbox[m])):
imgs_after[m].append(path_current + 'transform_%d_%d.png' %
(m, num_box))
# GET singe_people
for img_index in range(len(images)):
# data = cv2.imread()
if len(trans_martix[img_index]) > 0:
# heatmaps_after.append(data_heatmaps[img_index])
trans_img.append(imgs_after[img_index])
gt_key_mid.append(gt_key[img_index])
trans.append(trans_martix[img_index])
bboxes.append(bbox[img_index])
# todo The number og people need to adjust get minimal ok
numbers_of_people = []
for i in range(len(trans_img)):
numbers_of_people.append(len(trans_img[i]))
# print(len(trans_img))
num_people = min(numbers_of_people)
img_final = [[] for i in range(num_people)]
# heatmap_final = [[] for i in range(num_people)]
key_final = [[] for i in range(num_people)]
trans_final = [[] for i in range(num_people)]
bbox_final = [[] for i in range(num_people)]
for people_index in range(num_people):
for img_idx in range(len(trans_img)):
img_final[people_index].append(
trans_img[img_idx][people_index])
# heatmap_final[people_index].append(heatmaps_after[img_idx][people_index])
key_final[people_index].append(
gt_key_mid[img_idx][people_index])
trans_final[people_index].append(trans[img_idx][people_index])
bbox_final[people_index].append(bboxes[img_idx][people_index])
# todo delete destory image
img_des = []
for people_index in range(num_people):
for img_idx in range(len(img_final[0])):
data = cv2.imread(img_final[people_index][img_idx])
if data is None:
img_des.append(img_idx)
img_des_final = list(set(img_des))
img_des_final.sort()
count = 0
if len(img_des) > 0:
for i in range(len(img_des_final)):
for people_index in range(num_people):
idx = img_des_final[i] - count
del img_final[people_index][idx]
del key_final[people_index][idx]
del bbox_final[people_index][idx]
# del trans_final[people_index][idx]
count = count + 1
# for people_index in range(num_people):
# for img_idx in range(len(img_final[0])):
# del img_final[people_index][img_idx]
# del key_final[people_index][img_idx]
#joints
Get_joints = utils.Joints_data(images, img_final)
#coordinates transform
key_new = [[[] for i in range(len(img_final[0]))]
for i in range(num_people)]
for people_index in range(num_people):
for img_idx in range(len(bbox_final[0])):
for joint in range(15):
coordinates = np.array(
key_final[people_index][img_idx][joint])
correct = coordinates[-1]
new_coordinates = np.dot(
trans_final[people_index][img_idx], coordinates)
new_coordinates = new_coordinates.tolist()
new_coordinates.append(correct)
key_new[people_index][img_idx].append(
np.array(new_coordinates))
# key_new[people_index][img_idx][joint].append(correct)
heatmaps, heatmaps_weight = Get_joints.get_heatmaps(joints=key_new)
center_map = self.genCenterMap(x=184,
y=184,
sigma=21,
size_w=368,
size_h=368)
center_map = torch.Tensor(center_map)
center_map = center_map.unsqueeze(0)
center_map = center_map.unsqueeze(0)
return img_final, heatmaps, heatmaps_weight, center_map, bbox_final, im, key_new
def train(self,
batch_size=64,
learning_rate=1e-3,
num_epochs=5,
max_num=-1,
best_path='keyboard_model_best.tar',
current_path='keyboard_model_latest.tar',
decay_every=10,
save_model=True,
dirs=[0]):
model = self.model
criterion = nn.MSELoss()
optimizer = optim.Adam(self.model.parameters(), lr=learning_rate)
scheduler = optim.lr_scheduler.StepLR(optimizer,
step_size=decay_every,
gamma=0.05)
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_loss = None
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch + 1, num_epochs))
# Each epoch has a training and validation phase
for phase in ['train', 'val']:
if phase == 'train':
scheduler.step()
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
# Iterate over data.
max_num_for_this_epoch = max_num if phase == 'train' else -1
total = dataset.get_num_of_data(
phase) if max_num == -1 else max_num
bar = IntProgress(max=total)
display(bar)
for inputs, labels in dataset.data_batch(
type=phase,
batch_size=batch_size,
max_num=max_num_for_this_epoch,
dirs=dirs):
inputs = torch.Tensor(inputs)
labels = torch.Tensor(labels)
inputs = inputs.to(device)
labels = labels.to(device)
# zero the parameter gradients
optimizer.zero_grad()
# forward
outputs = model(inputs)
labels = torch.reshape(labels, [-1, 8])
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * batch_size
# free unoccupied memory
if torch.cuda.is_available():
torch.cuda.empty_cache()
else:
torch.cpu.empty_cache()
bar.value += batch_size
bar.description = f'{bar.value} / {total}'
bar.close()
epoch_loss = running_loss / dataset.get_num_of_data(phase)
print('{} Loss: {:.4f}'.format(phase, epoch_loss))
# deep copy the model
if phase == 'val' and (best_loss == None
or epoch_loss < best_loss):
best_loss = epoch_loss
best_model_wts = copy.deepcopy(model.state_dict())
torch.save(model.state_dict(), best_path)
print(f'The best model has been saved to {best_path} ...')
torch.save(model.state_dict(), current_path)
print(f'Current mode has been saved to {current_path} ...')
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val loss: {:4f}'.format(best_loss))
# load best model weights
model.load_state_dict(best_model_wts)
self.model = model
def set_up_next_model(self):
'''this sets up the next unconstrained variable to be run,
going in the predetermined variable order.'''
# check to see whether we are just done
if self.complete:
return
if len(self.remaining_uas)==0:
self.complete = True
return
self.this_ua = self.remaining_uas.pop()
# print 'about to generate_disallowed_symbols_list'
self.generate_disallowed_symbols_list()
self.model_complete = False
#self.vclass = self.prop.f[this_ua].vclass # the vclass of the corresponding variable in prop
#self.current_models = [Model(v, self.config, self.tree, self.context, self.fit, self.this_ua, self.prop) for v in self.vs]
#self.h = [model.initial_h for model in self.current_models]
# prepare inputs
#in_tokens = torch.LongTensor([self.gen_model.config.encode[self.next_symbol] ]).to(self.args.device)
#in_trees = torch.Tensor([-1,-1,-1,-1]).to(self.args.device)
in_tokens, structured_data = self.prepare_inputs(self.tree, self.context, self.fit, self.this_ua, self.prop)
in_tokens = torch.LongTensor(in_tokens).to(self.args.device)
in_trees = torch.Tensor(structured_data).to(self.args.device)
if self.args.train_rl:
#self.gen_logs.append([])
#self.gen_rewards.append([])
in_packed_feats, in_perm_idx = self.gen_model.embed([in_tokens], [in_trees])
encoder_out, encoder_h = self.gen_model.encode(in_packed_feats, in_perm_idx)
h = self.gen_model.decoder.to_middle(encoder_h)
else:
with torch.no_grad():
in_packed_feats, in_perm_idx = self.gen_model.embed([in_tokens], [in_trees])
encoder_out, encoder_h = self.gen_model.encode(in_packed_feats, in_perm_idx)
h = self.gen_model.decoder.to_middle(encoder_h)
self.h = h
#print ('input_h', h.shape)
# feed in the start symbol
left = [None]#*len(self.vs)
parent = [None]#*len(self.vs)
self.next_symbol = 'START_OUTPUT'
self.get_logits_and_next_state(left, parent, [-1,-1,-1,-1])
if self.gt_step is not None:
gt_tree = self.gt_step.unconstrained[self.prop.unconstrained_variables.index(self.this_ua)]
self.gt_string = list(self.this_ua)
#self.gt_string.insert(0, 'START_OUTPUT')
self.gt_string.append('END_OF_SECTION')
print ('set up a new model', self.gt_step.tree, self.prop.label, self.this_ua)
print ('correct string', self.gt_string)
# and assemble the stacks
self.parent_symbol_stack = [None]
self.parent_stack = [None]
self.left_sibling_stack = [None]
self.string = []
self.position_into_arity_stack = [0]
self.parent_arity_stack = [-1]
self.complete = False
self.next_symbol = None
def _run_one_epoch(self, epoch):
start = time.time()
total_loss = 0
data_loader = self.data_loader
# visualizing loss using visdom
if self.visdom_epoch:
vis_opts_epoch = dict(title=self.visdom_id + " epoch " +
str(epoch),
ylabel='Loss',
xlabel='Epoch')
vis_window_epoch = None
vis_iters = torch.arange(1, len(data_loader) + 1)
vis_iters_loss = torch.Tensor(len(data_loader))
for i, (data) in enumerate(data_loader):
self.step += 1
text_padded, input_lengths, feat_padded, stop_token_padded, encoder_mask, decoder_mask = data
if self.use_cuda:
text_padded = text_padded.cuda()
input_lengths = input_lengths.cuda()
feat_padded = feat_padded.cuda()
stop_token_padded = stop_token_padded.cuda()
encoder_mask = encoder_mask.cuda()
decoder_mask = decoder_mask.cuda()
y_pred = self.model(text_padded, input_lengths, feat_padded,
encoder_mask, decoder_mask)
y_target = (feat_padded, stop_token_padded)
loss = self.criterion(y_pred, y_target)
self.optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(self.model.parameters(),
self.max_norm)
self.optimizer.step()
total_loss += loss.item()
if i % self.print_freq == 0:
print('Epoch {0} | Iter {1} | Average Loss {2:.3f} | '
'Current Loss {3:.3f} | {4:.1f} ms/batch | {5} step'.
format(epoch + 1, i + 1, total_loss / (i + 1),
loss.item(),
1000 * (time.time() - start) / (i + 1),
self.step),
flush=True)
# visualizing loss using visdom
if self.visdom_epoch:
vis_iters_loss[i] = loss.item()
if i % self.print_freq == 0:
x_axis = vis_iters[:i + 1]
y_axis = vis_iters_loss[:i + 1]
if vis_window_epoch is None:
vis_window_epoch = self.vis.line(X=x_axis,
Y=y_axis,
opts=vis_opts_epoch)
else:
self.vis.line(X=x_axis,
Y=y_axis,
win=vis_window_epoch,
update='replace')
return total_loss / (i + 1)
net = Net().to(device)
if REBUILD_DATA:
# Если True нормализуем и перемешиваем данные
mandf = MandF()
mandf.make_training_data()
training_data = np.load("training_data.npy", allow_pickle=True)
print(len(training_data))
optimizer = optim.Adam(net.parameters(), lr=0.001)
loss_function = nn.MSELoss()
# преобразовываем данные в тензоры и разбиваем на тренировочные и тестрируемые
X = torch.Tensor([i[0] for i in training_data]).view(-1, 50, 50)
X = X / 255.0
y = torch.Tensor([i[1] for i in training_data])
VAL_PCT = 0.1
val_size = int(len(X) * VAL_PCT)
print(val_size)
train_X = X[:-val_size]
train_y = y[:-val_size]
test_X = X[-val_size:]
test_y = y[-val_size:]
print(len(train_X))
print(len(test_X))
def __init__(
self,
env_spec,
policy,
qf1,
qf2,
replay_buffer,
*, # Everything after this is numbers.
max_episode_length,
max_eval_path_length=None,
gradient_steps_per_itr,
fixed_alpha=None,
target_entropy=None,
initial_log_entropy=0.,
discount=0.99,
buffer_batch_size=64,
min_buffer_size=int(1e4),
target_update_tau=5e-3,
policy_lr=3e-4,
qf_lr=3e-4,
reward_scale=1.0,
optimizer=torch.optim.Adam,
steps_per_epoch=1,
num_evaluation_trajectories=10,
eval_env=None,
):
self._qf1 = qf1
self._qf2 = qf2
self.replay_buffer = replay_buffer
self._tau = target_update_tau
self._policy_lr = policy_lr
self._qf_lr = qf_lr
self._initial_log_entropy = initial_log_entropy
self._gradient_steps = gradient_steps_per_itr
self._optimizer = optimizer
self._num_evaluation_trajectories = num_evaluation_trajectories
self._eval_env = eval_env
self._min_buffer_size = min_buffer_size
self._steps_per_epoch = steps_per_epoch
self._buffer_batch_size = buffer_batch_size
self._discount = discount
self._reward_scale = reward_scale
self.max_episode_length = max_episode_length
self._max_eval_path_length = max_eval_path_length
self.policy = policy
self.env_spec = env_spec
self.replay_buffer = replay_buffer
self.sampler_cls = RaySampler
self.worker_cls = FragmentWorker
self._reward_scale = reward_scale
# use 2 target q networks
self._target_qf1 = copy.deepcopy(self._qf1)
self._target_qf2 = copy.deepcopy(self._qf2)
self._policy_optimizer = self._optimizer(self.policy.parameters(),
lr=self._policy_lr)
self._qf1_optimizer = self._optimizer(self._qf1.parameters(),
lr=self._qf_lr)
self._qf2_optimizer = self._optimizer(self._qf2.parameters(),
lr=self._qf_lr)
# automatic entropy coefficient tuning
self._use_automatic_entropy_tuning = fixed_alpha is None
self._fixed_alpha = fixed_alpha
if self._use_automatic_entropy_tuning:
if target_entropy:
self._target_entropy = target_entropy
else:
self._target_entropy = -np.prod(
self.env_spec.action_space.shape).item()
self._log_alpha = torch.Tensor([self._initial_log_entropy
]).requires_grad_()
self._alpha_optimizer = optimizer([self._log_alpha],
lr=self._policy_lr)
else:
self._log_alpha = torch.Tensor([self._fixed_alpha]).log()
self.episode_rewards = deque(maxlen=30)
def __init__(self, mode, input_size, hidden_size,
num_layers=1, bias=True, batch_first=False,
dropout=0., bidirectional=False):
super(RNNBase, self).__init__()
self.mode = mode
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.bias = bias
self.batch_first = batch_first
self.dropout = float(dropout)
self.bidirectional = bidirectional
num_directions = 2 if bidirectional else 1
if not isinstance(dropout, numbers.Number) or not 0 <= dropout <= 1 or \
isinstance(dropout, bool):
raise ValueError("dropout should be a number in range [0, 1] "
"representing the probability of an element being "
"zeroed")
if dropout > 0 and num_layers == 1:
warnings.warn("dropout option adds dropout after all but last "
"recurrent layer, so non-zero dropout expects "
"num_layers greater than 1, but got dropout={} and "
"num_layers={}".format(dropout, num_layers))
if mode == 'LSTM':
gate_size = 4 * hidden_size
elif mode == 'GRU':
gate_size = 3 * hidden_size
elif mode == 'RNN_TANH':
gate_size = hidden_size
elif mode == 'RNN_RELU':
gate_size = hidden_size
else:
raise ValueError("Unrecognized RNN mode: " + mode)
self._flat_weights_names = []
self._all_weights = []
for layer in range(num_layers):
for direction in range(num_directions):
layer_input_size = input_size if layer == 0 else hidden_size * num_directions
w_ih = Parameter(torch.Tensor(gate_size, layer_input_size))
w_hh = Parameter(torch.Tensor(gate_size, hidden_size))
b_ih = Parameter(torch.Tensor(gate_size))
# Second bias vector included for CuDNN compatibility. Only one
# bias vector is needed in standard definition.
b_hh = Parameter(torch.Tensor(gate_size))
layer_params = (w_ih, w_hh, b_ih, b_hh)
suffix = '_reverse' if direction == 1 else ''
param_names = ['weight_ih_l{}{}', 'weight_hh_l{}{}']
if bias:
param_names += ['bias_ih_l{}{}', 'bias_hh_l{}{}']
param_names = [x.format(layer, suffix) for x in param_names]
for name, param in zip(param_names, layer_params):
setattr(self, name, param)
self._flat_weights_names.extend(param_names)
self._all_weights.append(param_names)
self._flat_weights = [(lambda wn: getattr(self, wn) if hasattr(self, wn) else None)(wn) for wn in self._flat_weights_names]
self.flatten_parameters()
self.reset_parameters()
def __init__(self,
*,
backbone,
encoder,
decoder,
anchor_generator,
box2box_transform,
anchor_matcher,
num_classes,
backbone_level="res5",
pos_ignore_thresh=0.15,
neg_ignore_thresh=0.7,
focal_loss_alpha=0.25,
focal_loss_gamma=2.0,
box_reg_loss_type="giou",
test_score_thresh=0.05,
test_topk_candidates=1000,
test_nms_thresh=0.6,
max_detections_per_image=100,
pixel_mean,
pixel_std,
vis_period=0,
input_format="BGR"):
"""
NOTE: this interface is experimental.
Args:
backbone: a backbone module, must follow detectron2's backbone
interface
encoder (nn.Module): a module that encodes feature from backbone
decoder (nn.Module): a module to generate cls_score and box_reg
anchor_generator (nn.Module): a module that creates anchors from a
list of features. Usually an instance of
:class:`AnchorGenerator`
box2box_transform (Box2BoxTransform): defines the transform from
anchors boxes to instance boxes
anchor_matcher (Matcher): label the anchors by matching them
with ground truth.
num_classes (int): number of classes. Used to label background
proposals.
# ignore thresholds
pos_ignore_thresh (float): the threshold to ignore positive anchors
neg_ignore_thresh (float): the threshold to ignore negative anchors
# Loss parameters:
focal_loss_alpha (float): focal_loss_alpha
focal_loss_gamma (float): focal_loss_gamma
box_reg_loss_type (str): Options are "smooth_l1", "giou"
# Inference parameters:
test_score_thresh (float): Inference cls score threshold, only
anchors with score > INFERENCE_TH are considered for
inference (to improve speed)
test_topk_candidates (int): Select topk candidates before NMS
test_nms_thresh (float): Overlap threshold used for non-maximum
suppression (suppress boxes with IoU >= this threshold)
max_detections_per_image (int):
Maximum number of detections to return per image during
inference (100 is based on the limit established for the
COCO dataset).
# Input parameters
pixel_mean (Tuple[float]):
Values to be used for image normalization (BGR order).
To train on images of different number of channels, set
different mean & std.
Default values are the mean pixel value from ImageNet:
[103.53, 116.28, 123.675]
pixel_std (Tuple[float]):
When using pre-trained models in Detectron1 or any MSRA models,
std has been absorbed into its conv1 weights, so the std needs
to be set 1. Otherwise, you can use [57.375, 57.120, 58.395]
(ImageNet std)
vis_period (int):
The period (in terms of steps) for minibatch visualization at
train time. Set to 0 to disable.
input_format (str): Whether the model needs RGB, YUV, HSV etc.
"""
super().__init__()
self.backbone = backbone
self.encoder = encoder
self.decoder = decoder
# Anchors
self.anchor_generator = anchor_generator
self.box2box_transform = box2box_transform
self.anchor_matcher = anchor_matcher
self.num_classes = num_classes
self.backbone_level = backbone_level
# Ignore thresholds:
self.pos_ignore_thresh = pos_ignore_thresh
self.neg_ignore_thresh = neg_ignore_thresh
# Loss parameters:
self.focal_loss_alpha = focal_loss_alpha
self.focal_loss_gamma = focal_loss_gamma
self.box_reg_loss_type = box_reg_loss_type
assert self.box_reg_loss_type == 'giou', "Only support GIoU Loss."
# Inference parameters:
self.test_score_thresh = test_score_thresh
self.test_topk_candidates = test_topk_candidates
self.test_nms_thresh = test_nms_thresh
self.max_detections_per_image = max_detections_per_image
# Vis parameters
self.vis_period = vis_period
self.input_format = input_format
self.register_buffer("pixel_mean",
torch.Tensor(pixel_mean).view(-1, 1, 1))
self.register_buffer("pixel_std",
torch.Tensor(pixel_std).view(-1, 1, 1))
def test_span_metrics_are_computed_correctly(self):
gold_labels = [
"O", "B-ARG1", "I-ARG1", "O", "B-ARG2", "I-ARG2", "O", "O", "O"
]
gold_indices = [
self.vocab.get_token_index(x, "tags") for x in gold_labels
]
gold_tensor = torch.Tensor([gold_indices])
prediction_tensor = torch.rand(
[2, 9, self.vocab.get_vocab_size("tags")])
# Test that the span measure ignores completely masked sequences by
# passing a mask with a fully masked row.
mask = torch.LongTensor([[1, 1, 1, 1, 1, 1, 1, 1, 1],
[0, 0, 0, 0, 0, 0, 0, 0, 0]])
prediction_tensor[:, 0, 0] = 1
prediction_tensor[:, 1, 1] = 1 # (True positive - ARG1
prediction_tensor[:, 2, 2] = 1 # *)
prediction_tensor[:, 3, 0] = 1
prediction_tensor[:, 4, 0] = 1 # (False Negative - ARG2
prediction_tensor[:, 5, 0] = 1 # *)
prediction_tensor[:, 6, 0] = 1
prediction_tensor[:, 7, 1] = 1 # (False Positive - ARG1
prediction_tensor[:, 8, 2] = 1 # *)
metric = SpanBasedF1Measure(self.vocab, "tags")
metric(prediction_tensor, gold_tensor, mask)
assert metric._true_positives["ARG1"] == 1
assert metric._true_positives["ARG2"] == 0
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 1
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 1
assert metric._false_positives["ARG2"] == 0
assert "O" not in metric._false_positives.keys()
# Check things are accumulating correctly.
metric(prediction_tensor, gold_tensor, mask)
assert metric._true_positives["ARG1"] == 2
assert metric._true_positives["ARG2"] == 0
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 2
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 2
assert metric._false_positives["ARG2"] == 0
assert "O" not in metric._false_positives.keys()
metric_dict = metric.get_metric()
numpy.testing.assert_almost_equal(metric_dict["recall-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["recall-ARG1"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG1"], 0.5)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG1"],
0.666666666)
numpy.testing.assert_almost_equal(metric_dict["recall-overall"], 0.5)
numpy.testing.assert_almost_equal(metric_dict["precision-overall"],
0.5)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-overall"],
0.5)
# convert the roi to grayscale and blur it
gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (7, 7), 0)
# cv2.drawContours(clone, [segmented + (right, top)], -1, (0, 0, 255))
img = gray
# img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
# img=cv2.imread("240fn.jpg",cv2.IMREAD_GRAYSCALE)
# img=cv2.cvtColor(bw_image,cv2.COLOR_BGR2GRAY)
img = cv2.resize(img, (IMG_SIZE, IMG_SIZE))
test_data = img
orig = img
data = torch.Tensor(img).view(-1, 1, 96, 96)
net_out = net(data) # returns a list,
# print("net_out",net_out)
predicted_class = torch.argmax(net_out)
# print("predicted_class",predicted_class)
pnb = predicted_class.item()
# print("pnb",pnb)
# print(str(np.argmax(predicted_class))+" "+str(out_label[pnb]))
pre.append(out_label[pnb])
if out_label[pnb] == 'blnk':
pass
else:
def test_span_metrics_are_computed_correcly_with_prediction_map(self):
# In this example, datapoint1 only has access to ARG1 and V labels,
# whereas datapoint2 only has access to ARG2 and V labels.
# gold_labels = [["O", "B-ARG1", "I-ARG1", "O", "B-V", "O"],
# ["B-ARG2", "I-ARG2", "O", "B-V", "I-V", "O"]]
gold_indices = [[0, 1, 2, 0, 3, 0], [1, 2, 0, 3, 4, 0]]
prediction_map_indices = [[0, 1, 2, 5, 6], [0, 3, 4, 5, 6]]
gold_tensor = torch.Tensor(gold_indices)
prediction_map_tensor = torch.Tensor(prediction_map_indices)
prediction_tensor = torch.rand([2, 6, 5])
prediction_tensor[0, 0, 0] = 1
prediction_tensor[0, 1, 1] = 1 # (True Positive - ARG1
prediction_tensor[0, 2, 2] = 1 # *)
prediction_tensor[0, 3, 0] = 1
prediction_tensor[0, 4, 3] = 1 # (True Positive - V)
prediction_tensor[0, 5, 1] = 1 # (False Positive - ARG1)
prediction_tensor[1, 0, 0] = 1 # (False Negative - ARG2
prediction_tensor[1, 1, 0] = 1 # *)
prediction_tensor[1, 2, 0] = 1
prediction_tensor[1, 3, 3] = 1 # (True Positive - V
prediction_tensor[1, 4, 4] = 1 # *)
prediction_tensor[1, 5, 1] = 1 # (False Positive - ARG2)
metric = SpanBasedF1Measure(self.vocab, "tags")
metric(prediction_tensor,
gold_tensor,
prediction_map=prediction_map_tensor)
assert metric._true_positives["ARG1"] == 1
assert metric._true_positives["ARG2"] == 0
assert metric._true_positives["V"] == 2
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 1
assert metric._false_negatives["V"] == 0
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 1
assert metric._false_positives["ARG2"] == 1
assert metric._false_positives["V"] == 0
assert "O" not in metric._false_positives.keys()
# Check things are accumulating correctly.
metric(prediction_tensor,
gold_tensor,
prediction_map=prediction_map_tensor)
assert metric._true_positives["ARG1"] == 2
assert metric._true_positives["ARG2"] == 0
assert metric._true_positives["V"] == 4
assert "O" not in metric._true_positives.keys()
assert metric._false_negatives["ARG1"] == 0
assert metric._false_negatives["ARG2"] == 2
assert metric._false_negatives["V"] == 0
assert "O" not in metric._false_negatives.keys()
assert metric._false_positives["ARG1"] == 2
assert metric._false_positives["ARG2"] == 2
assert metric._false_positives["V"] == 0
assert "O" not in metric._false_positives.keys()
metric_dict = metric.get_metric()
numpy.testing.assert_almost_equal(metric_dict["recall-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG2"], 0.0)
numpy.testing.assert_almost_equal(metric_dict["recall-ARG1"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-ARG1"], 0.5)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG1"],
0.666666666)
numpy.testing.assert_almost_equal(metric_dict["recall-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["precision-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-V"], 1.0)
numpy.testing.assert_almost_equal(metric_dict["recall-overall"], 0.75)
numpy.testing.assert_almost_equal(metric_dict["precision-overall"],
0.6)
numpy.testing.assert_almost_equal(metric_dict["f1-measure-overall"],
0.666666666)
global_step += 1 * args.num_envs
obs[step] = next_obs
dones[step] = next_done
# ALGO LOGIC: put action logic here
with torch.no_grad():
values[step] = agent.get_value(obs[step]).flatten()
action, logproba, _, invalid_action_masks[step] = agent.get_action(
obs[step], envs=envs)
actions[step] = action.T
logprobs[step] = logproba
# TRY NOT TO MODIFY: execute the game and log data.
next_obs, rs, ds, infos = envs.step(action.T)
rewards[step], next_done = rs.view(-1), torch.Tensor(ds).to(device)
for info in infos:
if 'episode' in info.keys():
print(
f"global_step={global_step}, episode_reward={info['episode']['r']}"
)
writer.add_scalar("charts/episode_reward",
info['episode']['r'], global_step)
for key in info['microrts_stats']:
writer.add_scalar(f"charts/episode_reward/{key}",
info['microrts_stats'][key], global_step)
break
# bootstrap reward if not done. reached the batch limit
with torch.no_grad():
def forward(self, predictions, priors, targets):
"""Multibox Loss
Args:
predictions (tuple): A tuple containing loc preds, conf preds,
and prior boxes from SSD net.
conf shape: torch.size(batch_size,num_priors,num_classes)
loc shape: torch.size(batch_size,num_priors,4)
priors shape: torch.size(num_priors,4)
ground_truth (tensor): Ground truth boxes and labels for a batch,
shape: [batch_size,num_objs,5] (last idx is the label).
"""
loc_data, conf_data, landm_data = predictions
priors = priors
num = loc_data.size(0)
num_priors = (priors.size(0))
# match priors (default boxes) and ground truth boxes
loc_t = torch.Tensor(num, num_priors, 4)
landm_t = torch.Tensor(num, num_priors, 10)
conf_t = torch.LongTensor(num, num_priors)
for idx in range(num):
truths = targets[idx][:, :4].data
labels = targets[idx][:, -1].data
landms = targets[idx][:, 4:14].data
defaults = priors.data
match(self.threshold, truths, defaults, self.variance, labels,
landms, loc_t, conf_t, landm_t, idx)
if GPU:
loc_t = loc_t.cuda()
conf_t = conf_t.cuda()
landm_t = landm_t.cuda()
zeros = torch.tensor(0).cuda()
# landm Loss (Smooth L1)
# Shape: [batch,num_priors,10]
pos1 = conf_t > zeros
num_pos_landm = pos1.long().sum(1, keepdim=True)
N1 = max(num_pos_landm.data.sum().float(), 1)
pos_idx1 = pos1.unsqueeze(pos1.dim()).expand_as(landm_data)
landm_p = landm_data[pos_idx1].view(-1, 10)
landm_t = landm_t[pos_idx1].view(-1, 10)
loss_landm = F.smooth_l1_loss(landm_p, landm_t, reduction='sum')
pos = conf_t != zeros
conf_t[pos] = 1
# Localization Loss (Smooth L1)
# Shape: [batch,num_priors,4]
pos_idx = pos.unsqueeze(pos.dim()).expand_as(loc_data)
loc_p = loc_data[pos_idx].view(-1, 4)
loc_t = loc_t[pos_idx].view(-1, 4)
loss_l = F.smooth_l1_loss(loc_p, loc_t, reduction='sum')
# Compute max conf across batch for hard negative mining
batch_conf = conf_data.view(-1, self.num_classes)
loss_c = log_sum_exp(batch_conf) - batch_conf.gather(
1, conf_t.view(-1, 1))
# Hard Negative Mining
loss_c[pos.view(-1, 1)] = 0 # filter out pos boxes for now
loss_c = loss_c.view(num, -1)
_, loss_idx = loss_c.sort(1, descending=True)
_, idx_rank = loss_idx.sort(1)
num_pos = pos.long().sum(1, keepdim=True)
num_neg = torch.clamp(self.negpos_ratio * num_pos, max=pos.size(1) - 1)
neg = idx_rank < num_neg.expand_as(idx_rank)
# Confidence Loss Including Positive and Negative Examples
pos_idx = pos.unsqueeze(2).expand_as(conf_data)
neg_idx = neg.unsqueeze(2).expand_as(conf_data)
conf_p = conf_data[(pos_idx + neg_idx).gt(0)].view(
-1, self.num_classes)
targets_weighted = conf_t[(pos + neg).gt(0)]
loss_c = F.cross_entropy(conf_p, targets_weighted, reduction='sum')
# Sum of losses: L(x,c,l,g) = (Lconf(x, c) + αLloc(x,l,g)) / N
N = max(num_pos.data.sum().float(), 1)
loss_l /= N
loss_c /= N
loss_landm /= N1
return loss_l, loss_c, loss_landm
def _add_gaussian_noise(self, single_view):
return single_view + torch.Tensor(*single_view.shape).normal_(std=self.gaussian_noise_std).to(self.device)
def generate_batch(nb, height, width):
batch_images = torch.Tensor(nb, height, width, 3)
for i in tqdm(range(nb)):
batch_images[i] = torch.from_numpy(random_shapes((height, width), 6, 3, min_size=30, num_trials=25)[0])
return batch_images.permute(0,3,1,2)
def test(self):
all_psnr = [] # All signal to noise ratios over all the batches
all_percentages = [] # All percentage accuracies
dummy_metrics = [] # Metrics for the averaging value
# Load the score network
states = torch.load(os.path.join(self.args.log, 'checkpoint.pth'), map_location=self.config.device)
scorenet = CondRefineNetDilated(self.config).to(self.config.device)
scorenet = torch.nn.DataParallel(scorenet)
scorenet.load_state_dict(states[0])
scorenet.eval()
# Grab the first two samples from MNIST
trans = transforms.Compose([transforms.ToTensor()])
dataset = MNIST(os.path.join(self.args.run, 'datasets', 'mnist'), train=False, download=True)
first_digits_idx = dataset.train_labels <= 4
second_digits_idx = dataset.train_labels >=5
first_digits = dataset.train_data[first_digits_idx]
second_digits = dataset.train_data[second_digits_idx]
for iteration in range(100):
print("Iteration {}".format(iteration))
curr_dir = os.path.join(SAVE_DIR, "{:07d}".format(iteration))
if not os.path.exists(curr_dir):
os.makedirs(curr_dir)
# image1, image2 = get_images_split(first_digits, second_digits)
gt_images = []
for _ in range(N):
gt_images.append(get_single_image(dataset))
mixed = sum(gt_images).float()
mixed_grid = make_grid(mixed.detach() / float(N), nrow=GRID_SIZE, pad_value=1., padding=1)
save_image(mixed_grid, os.path.join(curr_dir, "mixed.png"))
for i in range(N):
gt_grid = make_grid(gt_images[i], nrow=GRID_SIZE, pad_value=1., padding=1)
save_image(gt_grid, os.path.join(curr_dir, "gt{}.png".format(i)))
mixed = torch.Tensor(mixed).cuda().view(BATCH_SIZE, 1, 28, 28)
xs = []
for _ in range(N):
xs.append(nn.Parameter(torch.Tensor(BATCH_SIZE, 1, 28, 28).uniform_()).cuda())
step_lr=0.00002
# Noise amounts
sigmas = np.array([1., 0.59948425, 0.35938137, 0.21544347, 0.12915497,
0.07742637, 0.04641589, 0.02782559, 0.01668101, 0.01])
n_steps_each = 100
for idx, sigma in enumerate(sigmas):
# if idx <= 5:
# lambda_recon = 1./(sigma**2)
# else:
# lambda_recon = 2./(sigma**2)
lambda_recon = 0.5/(sigma**2)
# Not completely sure what this part is for
labels = torch.ones(1, device=xs[0].device) * idx
labels = labels.long()
step_size = step_lr * (sigma / sigmas[-1]) ** 2
for step in range(n_steps_each):
noises = []
for _ in range(N):
noises.append(torch.randn_like(xs[0]) * np.sqrt(step_size * 2))
grads = []
for i in range(N):
grads.append(scorenet(xs[i].view(BATCH_SIZE, 1, 28, 28), labels).detach())
recon_loss = (torch.norm(torch.flatten(sum(xs) - mixed)) ** 2)
recon_grads = torch.autograd.grad(recon_loss, xs)
for i in range(N):
xs[i] = xs[i] + (step_size * grads[i]) + (-step_size * lambda_recon * recon_grads[i].detach()) + noises[i]
# x = x + (-step_size * lambda_recon * recon_grads[0].detach()) + noise_x
# y = y + (-step_size * lambda_recon * recon_grads[1].detach()) + noise_y
for i in range(N):
xs[i] = torch.clamp(xs[i], 0, 1)
x_to_write = []
for i in range(N):
x_to_write.append(torch.Tensor(xs[i].detach().cpu()))
# PSNR Measure
for idx in range(BATCH_SIZE):
best_psnr = -10000
best_permutation = None
for permutation in permutations(range(N)):
curr_psnr = sum([psnr(xs[permutation[i]][idx], gt_images[i][idx].cuda()) for i in range(N)])
if curr_psnr > best_psnr:
best_psnr = curr_psnr
best_permutation = permutation
all_psnr.append(best_psnr / float(N))
for i in range(N):
x_to_write[i][idx] = xs[best_permutation[i]][idx]
mixed_psnr = psnr(mixed.detach().cpu()[idx] / float(N), gt_images[i][idx])
dummy_metrics.append(mixed_psnr)
for i in range(N):
x_grid = make_grid(x_to_write[i], nrow=GRID_SIZE, pad_value=1., padding=1)
save_image(x_grid, os.path.join(curr_dir, "x{}.png".format(i)))
mixed_grid = make_grid(sum(xs)/float(N), nrow=GRID_SIZE, pad_value=1., padding=1)
save_image(mixed_grid, os.path.join(curr_dir, "recon.png".format(i)))
# average_grid = make_grid(mixed.detach()/2., nrow=GRID_SIZE)
# save_image(average_grid, "results/average_cifar.png")
print("Curr mean {}".format(np.array(all_psnr).mean()))
print("Const mean {}".format(np.array(dummy_metrics).mean()))
img[:,out_color.shape[1] : out_color.shape[1]*2, :] = out
print(img.shape)
img = cv2.resize(img, (2*target_res[1], target_res[0]))
cv2.imwrite("%d_depth.jpg" % i, img)
from torch.autograd import Variable
#inputs=torch.rand((1,3,240,320),dtype=torch.float32)
#input_var = Variable(inputs.cpu())
im = cv2.imread("girl.png")
im = cv2.resize(im,(320,240))
im = np.asarray(im,dtype=np.float32) / 255
im = im - 0.5
imt = im.transpose(2, 0, 1)
inputs = torch.Tensor([imt])
print(inputs.shape)
input_var = Variable(inputs)
print(input_var.size(),input_var.type())
output_var = model(input_var.cpu())
print(output_var.size(),output_var.type())
target_res=[240,320]
vis_depth(output_var.data.cpu().numpy(), inputs.numpy(), 0, target_res)
# Commented out IPython magic to ensure Python compatibility.
# %cd /content/YouTube3D
def convert_to_onnx(net, output_name):
input = torch.randn(1,3,240,320)
input_names = ['data']
model.load_state_dict(torch.load("model/chkpoint_000.pt"))
model.cuda()
# read the adjustment network checkpoint
adjustmentnet.load_state_dict(torch.load("model/chkpoint_adj_000.pt"))
adjustmentnet.cuda()
# read the img file from system argument
filenamein = sys.argv[1]
original = io.imread(filenamein)
new_h,new_w,d = original.shape
# reshape image to 224x224
batchsize=1
resized = cv2.resize(original,(224,224),interpolation=cv2.INTER_AREA)
img = torch.Tensor(resized).float().cuda()
img = img.permute(2,0,1).unsqueeze(0)
# apply model on input
y_pred = model(img)
alphas = y_pred[:,:199]
betas = y_pred[:,199:228]
s = y_pred[:,228]
t = torch.tanh(y_pred[:,229:231])
r = torch.tanh(y_pred[:,231:235]) * (3.14/4)
# create 3dmm
alpha_matrix = alphas.unsqueeze(2).expand(*alphas.size(),alphas.size(1)) * torch.eye(alphas.size(1)).cuda()
beta_matrix = betas.unsqueeze(2).expand(*betas.size(),betas.size(1))*torch.eye(betas.size(1)).cuda()
shape_cov = torch.bmm(torch.stack(batchsize*[face3dmm.shape_eigenvec]),alpha_matrix)
exp_cov = torch.bmm(torch.stack(batchsize*[face3dmm.exp_eigenvec]),beta_matrix)
def process_file(input_file, output_file, use_gpu):
write_out("Processing raw data file", input_file)
# create output file
file = h5py.File(output_file, 'w')
current_buffer_size = 1
current_buffer_allocation = 0
dset1 = file.create_dataset('primary', (current_buffer_size, MAX_SEQUENCE_LENGTH),
maxshape=(None, MAX_SEQUENCE_LENGTH), dtype='int32')
dset2 = file.create_dataset('tertiary', (current_buffer_size, MAX_SEQUENCE_LENGTH, 9),
maxshape=(None, MAX_SEQUENCE_LENGTH, 9), dtype='float')
dset3 = file.create_dataset('mask', (current_buffer_size, MAX_SEQUENCE_LENGTH),
maxshape=(None, MAX_SEQUENCE_LENGTH),
dtype='uint8')
input_file_pointer = open(input_file, "r")
while True:
# while there's more proteins to process
next_protein, missing_aa = read_protein_from_file(input_file_pointer)
if next_protein is None: # no more proteins to process
break
sequence_length = len(next_protein['primary'])
if sequence_length > MAX_SEQUENCE_LENGTH:
write_out("Dropping protein as length too long:", sequence_length)
continue
if missing_aa is True:
continue
if current_buffer_allocation >= current_buffer_size:
current_buffer_size = current_buffer_size + 1
dset1.resize((current_buffer_size, MAX_SEQUENCE_LENGTH))
dset2.resize((current_buffer_size, MAX_SEQUENCE_LENGTH, 9))
dset3.resize((current_buffer_size, MAX_SEQUENCE_LENGTH))
primary_padded = np.zeros(MAX_SEQUENCE_LENGTH)
tertiary_padded = np.zeros((9, MAX_SEQUENCE_LENGTH))
mask_padded = np.zeros(MAX_SEQUENCE_LENGTH)
# masking and padding here happens so that the stored dataset is of the same size.
# when the data is loaded in this padding is removed again.
primary_padded[:sequence_length] = next_protein['primary']
t_transposed = np.ravel(np.array(next_protein['tertiary']).T)
t_reshaped = np.reshape(t_transposed, (sequence_length, 9)).T
tertiary_padded[:, :sequence_length] = t_reshaped
mask_padded[:sequence_length] = next_protein['mask']
mask = torch.Tensor(mask_padded).type(dtype=torch.bool)
prim = torch.masked_select(torch.Tensor(primary_padded)\
.type(dtype=torch.long), mask)
pos = torch.masked_select(torch.Tensor(tertiary_padded), mask)\
.view(9, -1).transpose(0, 1).unsqueeze(1)
pos_angstrom = pos / 100
if use_gpu:
pos_angstrom = pos_angstrom.cuda()
# map to angles and back to tertiary
angles, batch_sizes = calculate_dihedral_angles_over_minibatch(pos_angstrom,
torch.tensor([len(prim)]),
use_gpu=use_gpu)
tertiary, _ = get_backbone_positions_from_angles(angles,
batch_sizes,
use_gpu=use_gpu)
tertiary = tertiary.squeeze(1)
# create variables to store padded sequences in
primary_padded = np.zeros(MAX_SEQUENCE_LENGTH)
tertiary_padded = np.zeros((MAX_SEQUENCE_LENGTH, 9))
mask_padded = np.zeros(MAX_SEQUENCE_LENGTH)
# store padded sequences
length_after_mask_removed = len(prim)
primary_padded[:length_after_mask_removed] = prim.data.cpu().numpy()
tertiary_padded[:length_after_mask_removed, :] = tertiary.data.cpu().numpy()
mask_padded[:length_after_mask_removed] = np.ones(length_after_mask_removed)
# save padded sequences on disk
dset1[current_buffer_allocation] = primary_padded
dset2[current_buffer_allocation] = tertiary_padded
dset3[current_buffer_allocation] = mask_padded
current_buffer_allocation += 1
if current_buffer_allocation == 0:
write_out("Preprocessing was selected but no proteins in the input file "
"were accepted. Please check your input.")
os._exit(1)
write_out("Wrote output to", current_buffer_allocation, "proteins to", output_file)
