def get_bbox(model, images): inp_dim = int(model.net_info["height"]) #If there's a GPU availible, put the model on GPU if CUDA: model.cuda() #Set the model in evaluation mode model.eval() # print("creating py torch variables") #PyTorch Variables for images im_batches = list( map(prep_image, images, [inp_dim for x in range(len(images))])) #List containing dimensions of original images im_dim_list = [(x.shape[1], x.shape[0]) for x in images] im_dim_list = torch.FloatTensor(im_dim_list).repeat(1, 2) # print("creating im dim list") if CUDA: im_dim_list = im_dim_list.cuda() leftover = 0 if (len(im_dim_list) % batch_size): leftover = 1 if batch_size != 1: num_batches = len(images) // batch_size + leftover im_batches = [ torch.cat( (im_batches[i * batch_size:min((i + 1) * batch_size, len(im_batches))])) for i in range(num_batches) ] # print("creating prediction shit") bboxes = [] classes_detected = [] class_names = [] lengths = [] for i, batch in enumerate(im_batches): if CUDA: batch = batch.cuda() prediction = model(Variable(batch, requires_grad=False), CUDA) prediction = write_results(prediction, confidence, num_classes, nms_conf=nms_thresh) # result has # index in batch # 4 box coordinates # object scorness # the score for the class # maximum confidence # and the index of the class curr_image_bbox = torch.Tensor([[0, 0, 0, 0]]) curr_image_class = torch.Tensor([-1]) if type(prediction) != int: curr_image_bbox = prediction[:, 1: 5] # [output[1:5] for output in prediction] curr_image_class = prediction[:, -1] # torch.FloatTensor([output[-1] for output in prediction]) elif CUDA: curr_image_bbox = curr_image_bbox.cuda() curr_image_class = curr_image_class.cuda() class_names.append([ classes[int(id)] if id != -1 else "<unk>" for id in curr_image_class ]) bboxes.append(curr_image_bbox) classes_detected.append(curr_image_class) lengths.append(len(curr_image_class)) return class_names, bboxes, lengths
def forward(self, im_data, im_info, gt_boxes, num_boxes, support_ims, all_cls_gt_boxes): im_batch = im_data im_info = im_info.data gt_boxes = gt_boxes.data num_boxes = num_boxes.data support_ims = support_ims.data batch_size = im_batch.size(0) x = self.resnet_base(im_batch) pos_sup_ims = support_ims[:, :self.num_shot, :, :, :].contiguous() pos_sup_feats = self.resnet_base(pos_sup_ims.view( batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80] if self.training: neg_sup_ims = support_ims[:, self.num_shot:, :, :, :].contiguous() neg_sup_feats = self.resnet_base(neg_sup_ims.view( batch_size * self.num_shot, 3, self.support_im_size, self.support_im_size)) # [B*S, 64, 80, 80] x1 = self.layer1(x) x2 = self.layer2(x1) x3 = self.layer3(x2) x4 = self.layer4(x3) pos_s1 = self.layer1(pos_sup_feats) pos_s2 = self.layer2(pos_s1) pos_s3 = self.layer3(pos_s2) pos_s4 = self.layer4(pos_s3) if self.training: neg_s1 = self.layer1(neg_sup_feats) neg_s2 = self.layer2(neg_s1) neg_s3 = self.layer3(neg_s2) neg_s4 = self.layer4(neg_s3) features = self.fpn([x2, x3, x4])[1:] pos_s_features = self.fpn([pos_s2, pos_s3, pos_s4])[1:] if self.training: neg_s_features = self.fpn([neg_s2, neg_s3, neg_s4])[1:] correlation_features = self.attention_module(features, pos_s_features) if self.training: neg_correlation_features = self.attention_module(features, neg_s_features) regression = torch.cat([self.regressionModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, 4] classification = torch.cat([self.classificationModel(feat) for feat in correlation_features], dim=1) # [1, n_proposal, n_class] if self.training: neg_regression = torch.cat([self.regressionModel(feat) for feat in neg_correlation_features], dim=1) neg_classification = torch.cat([self.classificationModel(feat) for feat in neg_correlation_features], dim=1) anchors = self.anchors(im_batch) rpn_loss_cls = torch.zeros(1).cuda() rpn_loss_bbox = torch.zeros(1).cuda() RCNN_loss_cls = torch.zeros(1).cuda() RCNN_loss_bbox = torch.zeros(1).cuda() rois_label = torch.zeros(10).cuda() rois = None cls_prob = None bbox_pred = None if self.training: pos_RCNN_loss_cls, RCNN_loss_bbox = self.focalLoss(classification, regression, anchors, gt_boxes) empty_gt_boxes = torch.zeros_like(gt_boxes) - 1. neg_RCNN_loss_cls, _ = self.focalLoss(neg_classification, neg_regression, anchors, empty_gt_boxes) RCNN_loss_cls = (pos_RCNN_loss_cls + neg_RCNN_loss_cls) / 2 return rois, cls_prob, bbox_pred, rpn_loss_cls, rpn_loss_bbox, RCNN_loss_cls, RCNN_loss_bbox, rois_label else: transformed_anchors = self.regressBoxes(anchors, regression) transformed_anchors = self.clipBoxes(transformed_anchors, im_batch) finalResult = [[], [], []] finalScores = torch.Tensor([]) finalAnchorBoxesIndexes = torch.Tensor([]).long() finalAnchorBoxesCoordinates = torch.Tensor([]) if torch.cuda.is_available(): finalScores = finalScores.cuda() finalAnchorBoxesIndexes = finalAnchorBoxesIndexes.cuda() finalAnchorBoxesCoordinates = finalAnchorBoxesCoordinates.cuda() # scores = torch.squeeze(classification[:, :, 1]) scores = F.softmax(classification[0], 1)[:, 1] scores_over_thresh = (scores > 0.5) if scores_over_thresh.sum() == 0: return None, None, np.array([0., 0., 0., 0.]) scores = scores[scores_over_thresh] anchorBoxes = torch.squeeze(transformed_anchors) anchorBoxes = anchorBoxes[scores_over_thresh] anchors_nms_idx = nms(anchorBoxes, scores, 0.5) finalScores = torch.cat((finalScores, scores[anchors_nms_idx])) # [n_predict] finalAnchorBoxesIndexesValue = torch.tensor([1] * anchors_nms_idx.shape[0]) if torch.cuda.is_available(): finalAnchorBoxesIndexesValue = finalAnchorBoxesIndexesValue.cuda() finalAnchorBoxesIndexes = torch.cat((finalAnchorBoxesIndexes, finalAnchorBoxesIndexesValue)) # [n_predict] finalAnchorBoxesCoordinates = torch.cat((finalAnchorBoxesCoordinates, anchorBoxes[anchors_nms_idx])) # [n_predict, 4] return [finalScores, finalAnchorBoxesIndexes, finalAnchorBoxesCoordinates]
torch.cuda.manual_seed(args.seed) if torch.cuda.is_available() and not args.cuda: print("Warning: You have Cuda but not use it. You are using CPU for training.") # Set up the data for training TEXT = data.Field(lower=True) ED = data.Field() train, dev, test = SQdataset.splits(TEXT, ED, args.data_dir) TEXT.build_vocab(train, dev, test) ED.build_vocab(train, dev, test) match_embedding = 0 if os.path.isfile(args.vector_cache): stoi, vectors, dim = torch.load(args.vector_cache) TEXT.vocab.vectors = torch.Tensor(len(TEXT.vocab), dim) for i, token in enumerate(TEXT.vocab.itos): wv_index = stoi.get(token, None) if wv_index is not None: TEXT.vocab.vectors[i] = vectors[wv_index] match_embedding += 1 else: TEXT.vocab.vectors[i] = torch.FloatTensor(dim).uniform_(-0.25, 0.25) else: print("Error: Need word embedding pt file") exit(1) print("Embedding match number {} out of {}".format(match_embedding, len(TEXT.vocab))) train_iter = data.Iterator(train, batch_size=args.batch_size, device=args.gpu, train=True, repeat=False, sort=False, shuffle=True, sort_within_batch=False)
def attack(self, iterations, target_path, lr, bandwidth): flag = 0 model = WaveToLetter.load_model(args.model_path) model = model.to(device) # model.eval() signal = self.get_signal(self.audio_path) orig = self.get_signal(self.orig_path) # alpha, index_max, index_min= self.attention(signal) start_attack_time = time.time() for i in range(iterations): print('Iteration:', str(i)) # print(signal[index_max:index_max+20]) # print(signal[index_max]) # if args.printSilence: # print() # print(sigindex_maxnal.shape) # print("20:", signal[index_max:index_max+20]) mfcc = pytorch_mfcc.MFCC(samplerate=self.sample_rate,winlen=self.window_size,winstep=self.window_stride,numcep=13,nfilt=26,nfft=512,lowfreq=0,highfreq=None,preemph=0,ceplifter=22,appendEnergy=False).cuda() mfccs = mfcc(signal) mfccs = self.normalize(mfccs) if args.printSilence: print("mfccs", mfccs) inputsMags = self.mfccs_to_inputs(mfccs) out = model(inputsMags) path = self.orig_path.split('wav')[0] + 'txt' + self.orig_path.split('wav')[1] fp=open(path) transcriptReal=fp.readlines()[0] print("Ref:",transcriptReal.lower()) seq_length = out.size(1) sizes = Variable(torch.Tensor([1.0]).mul_(int(seq_length)).int(), requires_grad=False) decoded_output, _, = decoder.decode(out.data, sizes) transcript = decoded_output[0][0] print("Hyp:", transcript.lower()) out = out.transpose(0, 1) if args.target: transcriptTarget = args.target else: fp=open(target_path) transcriptTarget=fp.readlines()[0] print("Tar:", transcriptTarget.lower()) if transcript.lower()==transcriptTarget.lower() and i>0: if args.target: target_path = args.target save_path = self.save(signal, target_path, self.orig_path, lr, iterations, i, bandwidth) generate_time = time.time() - start_attack_time print('Time taken (s): {generate_time:.4f}\t'.format(generate_time=generate_time)) self.save_figure(signal, save_path) break target = list(filter(None, [self.labels_map.get(x) for x in list(transcriptTarget.upper())])) targets = torch.IntTensor(target) target_sizes = torch.IntTensor([len(target)]) ctcloss = self.criterion(out, targets, sizes, target_sizes) # print("ctcloss:", ctcloss) # print("delta_2:", 100*torch.sum((signal - orig)**2)) # loss = ctcloss + 100*torch.sum((signal - orig)**2) loss = ctcloss print("loss:", loss) loss.backward() grad = np.array(signal.grad) is_nan = np.isnan(grad) is_nan_new = is_nan[is_nan == True] for j in range(len(grad)): if is_nan[j]: grad[j] = 100 wer = decoder.wer(transcript.lower(), transcriptTarget.lower()) / float(len(transcriptTarget.lower().split())) # the iterative proportional clipping method # print('grad:{}'.format(grad[index_max:index_max+20])) perturbation = lr*torch.from_numpy(grad) # print('perturbation', perturbation[index_max]) # print((signal.data - perturbation)[index_max]) # print("signal_next_relative0", ((signal.data - perturbation)/orig)[index_max]) signal_next_relative = torch.clamp((signal.data - perturbation)/orig, min = 1 - bandwidth, max = 1 + bandwidth) # print("signal_next_relative1:", signal_next_relative[index_max]) signal.data = signal_next_relative.mul(orig) # print(signal_next_relative[index_max]*orig[index_max]) # print("signal.data:", signal.data[index_max]) # if (i + 1) % 10000 == 0: # # anneal lr # # lr *= 0.5 # lr = lr / args.learning_anneal # print("wer", wer) # print("lr", lr) print("\n") signal.grad.data.zero_() print("Come to the end")
def position_matrix(J, d): m = torch.Tensor(J, d) for k in range(1, d + 1): for j in range(1, J + 1): m[j - 1, k - 1] = (1 - j / J) - (k / d) * (1 - 2 * j / J) return m
def set_stddev(self, stddev): self.stddev.data = torch.Tensor(stddev).to(self.stddev)
# save to file np.save(os.path.join(dat_dir, 'Zp.npy'), np.array(Zp)) import corner as DFM import matplotlib as mpl mpl.use('Agg') import matplotlib.pyplot as plt fig = DFM.corner(np.array(Zp)[:,:10]) fig.savefig(os.path.join(dat_dir, 'Zp.corner.png'), bbox_inches='tight') ''' for i in range(100): n_val = 10000 noise_y = torch.Tensor(n_val, np.sum(central_pixel)).normal_() noise_x = torch.Tensor(n_val, np.sum(~central_pixel)).normal_() p_Y_model.eval() p_XgivenY_model.eval() with torch.no_grad(): # sample Y' ~ p_NF(Y) Yp = p_Y_model.sample(n_val, noise=noise_y).detach().cpu() # sample X' ~ p_NF(X|Y') Xp = p_XgivenY_model.sample(n_val, noise=noise_x, cond_inputs=Yp).detach().cpu() XpYp = torch.from_numpy(np.zeros((n_val, 28 * 28))).float() XpYp[:, ~central_pixel] = Xp XpYp[:, central_pixel] = Yp
def beam_search(self, data: dict): encoder_feats = data.get('img').contiguous() device = encoder_feats.device beam_width = self.beam_width # eos = '[s]' eos = self.converter.dict['[s]'] def _inflate(tensor, times, dim): ##所有维度都重复一遍 repeat_dims = [1] * tensor.dim() #在需要的维度重复times repeat_dims[dim] = times return tensor.repeat(*repeat_dims) # https://github.com/IBM/pytorch-seq2seq/blob/fede87655ddce6c94b38886089e05321dc9802af/seq2seq/models/TopKDecoder.py batch_size, seq_len, seq_dim = encoder_feats.size() # inflated_encoder_feats = _inflate(encoder_feats, beam_width, 0) # ABC --> AABBCC -/-> ABCABC inflated_encoder_feats = encoder_feats.unsqueeze(1)\ .permute((1, 0, 2, 3)).repeat((beam_width, 1, 1, 1)).permute( (1, 0, 2, 3)).contiguous().view(-1, seq_len, seq_dim) # Initialize the decoder decoder_state = torch.zeros(1, batch_size * beam_width, self.hidden_dim).to(device) pos_index = (torch.Tensor(range(batch_size)) * beam_width).long().view( -1, 1).to(device) # Initialize the scores sequence_scores = torch.Tensor(batch_size * beam_width, 1).to(device) sequence_scores.fill_(-float('Inf')) sequence_scores.index_fill_( 0, torch.Tensor([i * beam_width for i in range(0, batch_size)]).long().to(device), 0.0) # sequence_scores.fill_(0.0) # Initialize the input vector y_prev = torch.zeros( (batch_size * beam_width)).fill_(self.num_classes).to(device) # Store decisions for backtracking #(T,batch * beam_width,1) stored_scores_tensor = torch.FloatTensor(self.max_len_labels + 1, batch_size * beam_width, 1).fill_(0).to(device) stored_predecessors_tensor = torch.LongTensor(self.max_len_labels + 1, batch_size * beam_width, 1).fill_(0).to(device) stored_emitted_symbols_tensor = torch.LongTensor( self.max_len_labels + 1, batch_size * beam_width).fill_(0).to(device) # stored_scores = list() # stored_predecessors = list() # stored_emitted_symbols = list() # +1 for [s] at end of sentence. for i in range(self.max_len_labels + 1): output, decoder_state = self.decoder(inflated_encoder_feats, decoder_state, y_prev) log_softmax_output = F.log_softmax(output, dim=1) sequence_scores = _inflate(sequence_scores, self.num_classes, 1) sequence_scores += log_softmax_output #每一步选topk个 scores, candidates = sequence_scores.view(batch_size, -1).topk(beam_width, dim=1) # Reshape input = (bk, 1) and sequence_scores = (bk, 1) y_prev = (candidates % self.num_classes).view(batch_size * beam_width) sequence_scores = scores.view(batch_size * beam_width, 1) # Update fields for next timestep #torch 1.5.1 floor_divide # predecessors = (candidates.floor_divide(self.num_classes) + pos_index.expand_as(candidates)).view(batch_size* beam_width,1) # torch 1.3.1 / predecessors = (candidates / self.num_classes + pos_index.expand_as(candidates)).view( batch_size * beam_width, 1) decoder_state = decoder_state.index_select(1, predecessors.squeeze()) # Update sequence socres and erase scores for <eos> symbol so that they aren't expanded # stored_scores.append(sequence_scores.clone()) stored_scores_tensor[i] = sequence_scores.clone() eos_indices = y_prev.view(-1, 1).eq(eos) if eos_indices.nonzero().dim() > 0: sequence_scores.masked_fill_(eos_indices, -float('inf')) # Cache results for backtracking # stored_predecessors.append(predecessors) stored_predecessors_tensor[i] = predecessors stored_emitted_symbols_tensor[i] = y_prev # stored_emitted_symbols.append(y_prev) # Do backtracking to return the optimal values # ====== backtrak ======# # Initialize return variables given different types #反向搜索 p = [] l = [[self.max_len_labels] * beam_width for _ in range(batch_size)] # the last step output of the beams are not sorted # thus they are sorted here # sorted_score,sorted_idx = stored_scores[-1].view(batch_size,beam_width).topk(beam_width) # s = sorted_score.clone() sorted_score, sorted_idx = stored_scores_tensor[-1].view( batch_size, beam_width).topk(beam_width) s = sorted_score.clone() batch_eos_found = [0] * batch_size # +1 for [s] at end of sentence. #t = self.max_len_labels -1 t = self.max_len_labels t_predecessors = (sorted_idx + pos_index.expand_as(sorted_idx)).view( batch_size * beam_width) while t >= 0: # Re-order the variables with the back pointer # current_symbol = stored_emitted_symbols[t].index_select(0,t_predecessors) # t_predecessors = stored_predecessors[t].index_select(0,t_predecessors).squeeze() # eos_indices = stored_emitted_symbols[t].eq(eos).nonzero() current_symbol = stored_emitted_symbols_tensor[t].index_select( 0, t_predecessors) t_predecessors = stored_predecessors_tensor[t].index_select( 0, t_predecessors).squeeze() eos_indices = stored_emitted_symbols_tensor[t].eq(eos).nonzero() if eos_indices.dim() > 0: for i in range(eos_indices.size(0) - 1, -1, -1): # Indices of the EOS symbol for both variables # with b*k as the first dimension, and b, k for # the first two dimensions idx = eos_indices[i] b_index = int(idx[0] / beam_width) # The indices of the replacing position # according to the replacement strategy noted above res_k_idx = beam_width - (batch_eos_found[b_index] % beam_width) - 1 batch_eos_found[b_index] += 1 res_idx = b_index * beam_width + res_k_idx # Replace the old information in return variables # with the new ended sequence information t_predecessors[res_idx] = stored_predecessors_tensor[t][ idx[0]] current_symbol[res_idx] = stored_emitted_symbols_tensor[t][ idx[0]] s[b_index, res_k_idx] = stored_scores_tensor[t][idx[0], [0]] l[b_index][res_k_idx] = t + 1 # record the back tracked results p.append(current_symbol) t -= 1 # Sort and re-order again as the added ended sequences may change # the order (very unlikely) s, re_sorted_idx = s.topk(beam_width) for b_index in range(batch_size): l[b_index] = [ l[b_index][k_idx.item()] for k_idx in re_sorted_idx[b_index, :] ] re_sorted_idx = (re_sorted_idx + pos_index.expand_as(re_sorted_idx)).view(batch_size * beam_width) # Reverse the sequences and re-order at the same time # It is reversed because the backtracking happens in reverse time order p = [ step.index_select(0, re_sorted_idx).view(batch_size, beam_width, -1) for step in reversed(p) ] rec_pred = torch.cat(p, -1)[:, 0, :] return rec_pred
#print(inp['adj'].size()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = gcn_resnet101(num_classes=150, t=0.4, adj_file='food_adj.pkl') model.load_state_dict(torch.load("model_best_87.7752.pth.tar")['state_dict']) model = model.to(device) model.eval() normalize = transforms.Normalize(mean=model.image_normalization_mean, std=model.image_normalization_std) transform = transforms.Compose([ Warp('448'), transforms.ToTensor(), normalize, ]) fff = open('food_glove_word2vec.pkl', 'rb') inp = pickle.load(fff) inp = torch.Tensor(inp) inp = inp.unsqueeze(0) inp = inp.to(device) ''' tempa = open("./train_anno.json","r") tempb = open("./val_anno.json","r") a = json.load(tempa) b = json.load(tempb) qqq=[] www=[] with tqdm(total=len(a)) as pbar: for aaaa in a: aaa = Image.open("./coco/data/train2014/"+aaaa['file_name']).convert('RGB')
def evaluate(experiment_directory, checkpoint_path): chamfer_results = [] specs = ws.load_experiment_specifications(experiment_directory) logging.info("Experiment description: \n" + specs["Description"]) data_source = specs["DataSource"] test_split_file = specs["TestSplit"] num_samp_per_scene = specs["SamplesPerScene"] scene_per_batch = specs["ScenesPerBatch"] clamp_dist = specs["ClampingDistance"] minT = -clamp_dist maxT = clamp_dist enforce_minmax = False num_data_loader_threads =1 # scene_per_subbatch =1 batch_split = 1 scene_per_subbatch = scene_per_batch // batch_split checkpoints = list( range( specs["SnapshotFrequency"], specs["NumEpochs"] + 1, specs["SnapshotFrequency"], ) ) def signal_handler(sig, frame): logging.info("Stopping early...") sys.exit(0) with open(test_split_file,"r") as f: test_split = json.load(f) sdf_dataset = dt_vtk.SDFVTKSamples( data_source, test_split, num_samp_per_scene ) sdf_loader = data_utils.DataLoader( sdf_dataset, batch_size=scene_per_subbatch, shuffle=True, num_workers=num_data_loader_threads, drop_last=True, ) decoder_eval = decoder.Decoder(0, **specs["NetworkSpecs"]).cuda() # for epoch in range(start_epoch, num_epochs + 1): # start = time.time() # logging.info("epoch {}...".format(epoch)) # pdb.set_trace() checkpoint = torch.load(checkpoint_path) decoder_eval.load_state_dict(checkpoint['model_state_dict']) decoder_eval = decoder_eval.float() # decoder_eval.eval() for param in decoder_eval.parameters(): param.requires_grad = False loss_l1 = torch.nn.L1Loss() loss_l2 = torch.nn.MSELoss() loss_log =[] # theta_x = torch.randn(1, requires_grad=True, dtype=torch.float)*3.1415 # theta_y = torch.randn(1, requires_grad=True, dtype=torch.float)*3.1415 # theta_z = torch.randn(1, requires_grad=True, dtype=torch.float)*3.1415 # theta_x = theta_x.float() # theta_y = theta_y.float() # theta_z = theta_z.float() # theta_x.retain_grad() # theta_y.retain_grad() # theta_z.retain_grad() scale_one = torch.randn(1, requires_grad=True, dtype=torch.float) scale_two = torch.randn(1, requires_grad=True, dtype=torch.float) scale_three = torch.randn(1, requires_grad=True, dtype=torch.float) scale_one.retain_grad() scale_two.retain_grad() scale_three.retain_grad() # transform_matrix = torch.zeros(3,3).float().cuda() # transform_matrix.requires_grad_(True) # transform_matrix.retain_grad() transform_inpt = torch.randn(3,3).float().cuda() transform_inpt.requires_grad_(True) transform_inpt.retain_grad() bias = torch.zeros(3).float().cuda() bias.requires_grad_(True) bias.retain_grad() # pdb.set_trace() test_model = np.array(pd.read_csv("../chairs_segdata/points/1a8bbf2994788e2743e99e0cae970928.pts", header=None,sep=" ").values) num_epochs = 500 learning_rate = 1e-3 test_pts = torch.from_numpy(test_model).float() test_pts.requies_grad = False bt_size = 32 num_batches = int(test_model.shape[0]//bt_size) sub = torch.Tensor([1]).cuda() reg = 1 # rot_x = torch.from_numpy(rotate_mat_x()).double().cuda() # rot_x.requires_grad_(True) # rot_y = torch.from_numpy(rotate_mat_y()).double().cuda() # rot_y.requires_grad_(True) # rot_z = torch.from_numpy(rotate_mat_z()).double().cuda() # rot_z.requires_grad_(True) with torch.enable_grad(): for j in range(num_epochs): # pdb.set_trace() # Process the input datag # sdf_data.requires_grad = False # sdf_data = (sdf_data.cuda()).reshape( # num_samp_per_scene * scene_per_subbatch, 4 # ) # xyz = sdf_data[:, 0:3] # transform_matrix_update = torch.add(transform_matrix,bias) batch_loss=0 for i in range(num_batches): test_torch = test_pts[i*bt_size:(i+1)*bt_size,:] # pdb.set_trace() # cosval_x = torch.cos(theta_x) # sinval_x = torch.sin(theta_x) # cosval_x.requires_grad_(True) # sinval_x.requires_grad_(True) # cosval_x.retain_grad() # sinval_x.retain_grad() # rot_x = torch.stack([torch.Tensor([1, 0, 0]), # torch.cat([torch.Tensor([0]), cosval_x, -sinval_x]), # torch.cat([torch.Tensor([0]), sinval_x, cosval_x])], dim=1).float().cuda() # rot_x.requires_grad_(True) # rot_x.retain_grad() # cosval_y = torch.cos(theta_y) # sinval_y = torch.sin(theta_y) # cosval_y.requires_grad_(True) # sinval_y.requires_grad_(True) # cosval_y.retain_grad() # sinval_y.retain_grad() # rot_y = torch.stack([torch.cat([cosval_y, torch.Tensor([0]), sinval_y]), # torch.Tensor([0, 1, 0]), # torch.cat([-sinval_y, torch.Tensor([0]), cosval_y])],dim=1).float().cuda() # rot_y.requires_grad_(True) # rot_y.retain_grad() # cosval_z = torch.cos(theta_z) # sinval_z = torch.sin(theta_z) # cosval_z.requires_grad_(True) # sinval_z.requires_grad_(True) # cosval_z.retain_grad() # sinval_z.retain_grad() # rot_z = torch.stack([torch.cat([cosval_z, -sinval_z, torch.Tensor([0])]), # torch.cat([sinval_z, cosval_z, torch.Tensor([0])]), # torch.Tensor([0, 0, 1])], dim=1).float().cuda() # rot_z.requires_grad_(True) # rot_z.retain_grad() scale_matrix = torch.cat([torch.cat([scale_one,torch.Tensor([0]),torch.Tensor([0])]), torch.cat([torch.Tensor([0]),scale_two,torch.Tensor([0])]), torch.cat([torch.Tensor([0]),torch.Tensor([0]),scale_three])]).view(3,3).float().cuda() # pdb.set_trace() scale_matrix.retain_grad() scale_matrix.requires_grad_(True) # transform_matrix = torch.matmul(torch.matmul(torch.matmul(rot_z,rot_y),rot_x),scale_matrix) transform_matrix = torch.matmul(transform_inpt, scale_matrix) transform_matrix.requires_grad_(True) transform_matrix.retain_grad() xyz = test_torch.cuda() xyz_transform = torch.matmul(xyz, transform_matrix) xyz_transform.requires_grad_(True) xyz_transform.retain_grad() transform_bias = torch.add(xyz_transform, bias).float() transform_bias.retain_grad() # diag_sum = torch.abs(torch.sum(torch.diag(transform_matrix))) # sdf_gt = sdf_data[:, 3].unsqueeze(1) pred_sdf = decoder_eval(transform_bias) # pred_sdf = decoder_eval(xyz_transform) # loss = loss_l1(pred_sdf, sdf_gt) target = torch.zeros(pred_sdf.shape[0],pred_sdf.shape[1]).float().cuda() # batch_loss += loss.item() # pdb.set_trace() diag_sum = torch.norm(torch.sub(torch.diag(scale_matrix),sub),2) diag_sum.retain_grad() diag_sum.requires_grad_(True) # diag_sum = torch.sum(torch.diag(transform_matrix)).cpu() loss1 = loss_l1(pred_sdf,target) loss2 = reg *diag_sum # loss2 = torch.abs(torch.sub(diag_sum,1)) loss = torch.add(loss1,loss2) loss.backward(retain_graph=True) batch_loss+= loss.item() print('Batch Loss {:6.4f}'.format(loss.item())) with torch.no_grad(): # theta_z.data.sub_(theta_z.grad.data*learning_rate) # theta_y.data.sub_(theta_y.data*learning_rate) # theta_x.data.sub_(theta_x.grad.data*learning_rate) bias.data.sub_(bias.grad.data*learning_rate) scale_one.data.sub_(scale_one.grad.data*learning_rate) scale_two.data.sub_(scale_two.grad.data*learning_rate) scale_three.data.sub_(scale_three.grad.data*learning_rate) transform_inpt.data.sub_(transform_inpt.grad.data*learning_rate) # theta_z.grad.data.zero_() # theta_y.grad.data.zero_() # theta_x.grad.data.zero_() bias.grad.data.zero_() scale_one.grad.data.zero_() scale_three.grad.data.zero_() scale_two.grad.data.zero_() scale_matrix.grad.data.zero_() transform_bias.grad.data.zero_() xyz_transform.grad.data.zero_() transform_matrix.grad.data.zero_() transform_inpt.grad.data.zero_() diag_sum.grad.data.zero_() # rot_z.grad.data.zero_() # rot_x.grad.data.zero_() # rot_y.grad.data.zero_() # pdb.set_trace() actual_loss = (batch_loss*bt_size)/(test_model.shape[0]) # print("Loss after {} epoch is {:6.4f}".format(j,batch_loss)) print("Loss after {} epoch is {:6.4f}".format(j,actual_loss)) loss_log.append(actual_loss) pdb.set_trace() fig,ax = plt.subplots() ax.plot(np.arange(num_epochs),loss_log) ax.set(xlabel='iterations',ylabel='transformationloss') plt.savefig('Transformation_loss_new.png') torch.save(transform_matrix,'transform_matrix_new.pt') torch.save(bias,'bias_new.pt') test_pts = torch.from_numpy(pd.read_csv('test_model.pts',header=None, sep=' ').values).cuda() transform_pts = torch.matmul(test_pts, transform_matrix.double()) transform_pts = torch.add(transform_pts, bias.double()).cpu().detach().numpy() np.savetxt('transform_points_new.pts',transform_pts) plot_heatmap(experiment_directory, checkpoint_path)
def batchify_sentence_classification_with_label(input_batch_list, gpu, if_train=True): """ input: list of words, chars and labels, various length. [[words, features, chars, labels],[words, features, chars,labels],...] words: word ids for one sentence. (batch_size, sent_len) features: features ids for one sentence. (batch_size, feature_num), each sentence has one set of feature chars: char ids for on sentences, various length. (batch_size, sent_len, each_word_length) labels: label ids for one sentence. (batch_size,), each sentence has one set of feature output: zero padding for word and char, with their batch length word_seq_tensor: (batch_size, max_sent_len) Variable feature_seq_tensors: [(batch_size,), ... ] list of Variable word_seq_lengths: (batch_size,1) Tensor char_seq_tensor: (batch_size*max_sent_len, max_word_len) Variable char_seq_lengths: (batch_size*max_sent_len,1) Tensor char_seq_recover: (batch_size*max_sent_len,1) recover char sequence order label_seq_tensor: (batch_size, ) mask: (batch_size, max_sent_len) """ batch_size = len(input_batch_list) words = [sent[0] for sent in input_batch_list] features = [np.asarray(sent[1]) for sent in input_batch_list] feature_num = len(features[0]) chars = [sent[2] for sent in input_batch_list] labels = [sent[3] for sent in input_batch_list] word_seq_lengths = torch.LongTensor(list(map(len, words))) max_seq_len = word_seq_lengths.max().item() word_seq_tensor = torch.zeros((batch_size, max_seq_len), requires_grad=if_train).long() label_seq_tensor = torch.zeros((batch_size, ), requires_grad=if_train).long() feature_seq_tensors = [] for idx in range(feature_num): feature_seq_tensors.append( torch.zeros((batch_size, max_seq_len), requires_grad=if_train).long()) mask = torch.zeros((batch_size, max_seq_len), requires_grad=if_train).byte() label_seq_tensor = torch.LongTensor(labels) # exit(0) for idx, (seq, seqlen) in enumerate(zip(words, word_seq_lengths)): seqlen = seqlen.item() word_seq_tensor[idx, :seqlen] = torch.LongTensor(seq) mask[idx, :seqlen] = torch.Tensor([1] * seqlen) for idy in range(feature_num): feature_seq_tensors[idy][idx, :seqlen] = torch.LongTensor( features[idx][:, idy]) word_seq_lengths, word_perm_idx = word_seq_lengths.sort(0, descending=True) word_seq_tensor = word_seq_tensor[word_perm_idx] for idx in range(feature_num): feature_seq_tensors[idx] = feature_seq_tensors[idx][word_perm_idx] label_seq_tensor = label_seq_tensor[word_perm_idx] mask = mask[word_perm_idx] ### deal with char # pad_chars (batch_size, max_seq_len) pad_chars = [ chars[idx] + [[0]] * (max_seq_len - len(chars[idx])) for idx in range(len(chars)) ] length_list = [list(map(len, pad_char)) for pad_char in pad_chars] max_word_len = max(map(max, length_list)) char_seq_tensor = torch.zeros((batch_size, max_seq_len, max_word_len), requires_grad=if_train).long() char_seq_lengths = torch.LongTensor(length_list) for idx, (seq, seqlen) in enumerate(zip(pad_chars, char_seq_lengths)): for idy, (word, wordlen) in enumerate(zip(seq, seqlen)): # print len(word), wordlen char_seq_tensor[idx, idy, :wordlen] = torch.LongTensor(word) char_seq_tensor = char_seq_tensor[word_perm_idx].view( batch_size * max_seq_len, -1) char_seq_lengths = char_seq_lengths[word_perm_idx].view( batch_size * max_seq_len, ) char_seq_lengths, char_perm_idx = char_seq_lengths.sort(0, descending=True) char_seq_tensor = char_seq_tensor[char_perm_idx] _, char_seq_recover = char_perm_idx.sort(0, descending=False) _, word_seq_recover = word_perm_idx.sort(0, descending=False) if gpu: word_seq_tensor = word_seq_tensor.cuda() for idx in range(feature_num): feature_seq_tensors[idx] = feature_seq_tensors[idx].cuda() word_seq_lengths = word_seq_lengths.cuda() word_seq_recover = word_seq_recover.cuda() label_seq_tensor = label_seq_tensor.cuda() char_seq_tensor = char_seq_tensor.cuda() char_seq_recover = char_seq_recover.cuda() mask = mask.cuda() return word_seq_tensor, feature_seq_tensors, word_seq_lengths, word_seq_recover, char_seq_tensor, char_seq_lengths, char_seq_recover, label_seq_tensor, mask
def update(self): img = Imgdata( 'dataset/json_data/posetrack_train.json', '/media/jion/D/chenhaoming/DataSet/PoseTrack2017/posetrack_data/', self.index) images, bbox, key_gt, floder_path = img.get_Info() im = images gt_key = img.data_adjustment(key_gt) path_current = '/media/jion/D/chenhaoming/DataSet/PoseTrack2017/posetrack_data/' + floder_path + '/' trans_martix = utils.crop_img( path_current=path_current, images=images, bbox=bbox) #Generate new data in origin floder #reshape data gt_key_mid = [] trans = [] bboxes = [] # heatmaps_after = [] imgs_after = [[] for i in range(len(images))] trans_img = [] for m in range(len(images)): if len(bbox[m][0]) != 0: for num_box in range(len(bbox[m])): imgs_after[m].append(path_current + 'transform_%d_%d.png' % (m, num_box)) # GET singe_people for img_index in range(len(images)): # data = cv2.imread() if len(trans_martix[img_index]) > 0: # heatmaps_after.append(data_heatmaps[img_index]) trans_img.append(imgs_after[img_index]) gt_key_mid.append(gt_key[img_index]) trans.append(trans_martix[img_index]) bboxes.append(bbox[img_index]) # todo The number og people need to adjust get minimal ok numbers_of_people = [] for i in range(len(trans_img)): numbers_of_people.append(len(trans_img[i])) # print(len(trans_img)) num_people = min(numbers_of_people) img_final = [[] for i in range(num_people)] # heatmap_final = [[] for i in range(num_people)] key_final = [[] for i in range(num_people)] trans_final = [[] for i in range(num_people)] bbox_final = [[] for i in range(num_people)] for people_index in range(num_people): for img_idx in range(len(trans_img)): img_final[people_index].append( trans_img[img_idx][people_index]) # heatmap_final[people_index].append(heatmaps_after[img_idx][people_index]) key_final[people_index].append( gt_key_mid[img_idx][people_index]) trans_final[people_index].append(trans[img_idx][people_index]) bbox_final[people_index].append(bboxes[img_idx][people_index]) # todo delete destory image img_des = [] for people_index in range(num_people): for img_idx in range(len(img_final[0])): data = cv2.imread(img_final[people_index][img_idx]) if data is None: img_des.append(img_idx) img_des_final = list(set(img_des)) img_des_final.sort() count = 0 if len(img_des) > 0: for i in range(len(img_des_final)): for people_index in range(num_people): idx = img_des_final[i] - count del img_final[people_index][idx] del key_final[people_index][idx] del bbox_final[people_index][idx] # del trans_final[people_index][idx] count = count + 1 # for people_index in range(num_people): # for img_idx in range(len(img_final[0])): # del img_final[people_index][img_idx] # del key_final[people_index][img_idx] #joints Get_joints = utils.Joints_data(images, img_final) #coordinates transform key_new = [[[] for i in range(len(img_final[0]))] for i in range(num_people)] for people_index in range(num_people): for img_idx in range(len(bbox_final[0])): for joint in range(15): coordinates = np.array( key_final[people_index][img_idx][joint]) correct = coordinates[-1] new_coordinates = np.dot( trans_final[people_index][img_idx], coordinates) new_coordinates = new_coordinates.tolist() new_coordinates.append(correct) key_new[people_index][img_idx].append( np.array(new_coordinates)) # key_new[people_index][img_idx][joint].append(correct) heatmaps, heatmaps_weight = Get_joints.get_heatmaps(joints=key_new) center_map = self.genCenterMap(x=184, y=184, sigma=21, size_w=368, size_h=368) center_map = torch.Tensor(center_map) center_map = center_map.unsqueeze(0) center_map = center_map.unsqueeze(0) return img_final, heatmaps, heatmaps_weight, center_map, bbox_final, im, key_new
def train(self, batch_size=64, learning_rate=1e-3, num_epochs=5, max_num=-1, best_path='keyboard_model_best.tar', current_path='keyboard_model_latest.tar', decay_every=10, save_model=True, dirs=[0]): model = self.model criterion = nn.MSELoss() optimizer = optim.Adam(self.model.parameters(), lr=learning_rate) scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=decay_every, gamma=0.05) since = time.time() best_model_wts = copy.deepcopy(model.state_dict()) best_loss = None for epoch in range(num_epochs): print('Epoch {}/{}'.format(epoch + 1, num_epochs)) # Each epoch has a training and validation phase for phase in ['train', 'val']: if phase == 'train': scheduler.step() model.train() # Set model to training mode else: model.eval() # Set model to evaluate mode running_loss = 0.0 # Iterate over data. max_num_for_this_epoch = max_num if phase == 'train' else -1 total = dataset.get_num_of_data( phase) if max_num == -1 else max_num bar = IntProgress(max=total) display(bar) for inputs, labels in dataset.data_batch( type=phase, batch_size=batch_size, max_num=max_num_for_this_epoch, dirs=dirs): inputs = torch.Tensor(inputs) labels = torch.Tensor(labels) inputs = inputs.to(device) labels = labels.to(device) # zero the parameter gradients optimizer.zero_grad() # forward outputs = model(inputs) labels = torch.reshape(labels, [-1, 8]) loss = criterion(outputs, labels) # backward + optimize only if in training phase if phase == 'train': loss.backward() optimizer.step() # statistics running_loss += loss.item() * batch_size # free unoccupied memory if torch.cuda.is_available(): torch.cuda.empty_cache() else: torch.cpu.empty_cache() bar.value += batch_size bar.description = f'{bar.value} / {total}' bar.close() epoch_loss = running_loss / dataset.get_num_of_data(phase) print('{} Loss: {:.4f}'.format(phase, epoch_loss)) # deep copy the model if phase == 'val' and (best_loss == None or epoch_loss < best_loss): best_loss = epoch_loss best_model_wts = copy.deepcopy(model.state_dict()) torch.save(model.state_dict(), best_path) print(f'The best model has been saved to {best_path} ...') torch.save(model.state_dict(), current_path) print(f'Current mode has been saved to {current_path} ...') print() time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val loss: {:4f}'.format(best_loss)) # load best model weights model.load_state_dict(best_model_wts) self.model = model
def set_up_next_model(self): '''this sets up the next unconstrained variable to be run, going in the predetermined variable order.''' # check to see whether we are just done if self.complete: return if len(self.remaining_uas)==0: self.complete = True return self.this_ua = self.remaining_uas.pop() # print 'about to generate_disallowed_symbols_list' self.generate_disallowed_symbols_list() self.model_complete = False #self.vclass = self.prop.f[this_ua].vclass # the vclass of the corresponding variable in prop #self.current_models = [Model(v, self.config, self.tree, self.context, self.fit, self.this_ua, self.prop) for v in self.vs] #self.h = [model.initial_h for model in self.current_models] # prepare inputs #in_tokens = torch.LongTensor([self.gen_model.config.encode[self.next_symbol] ]).to(self.args.device) #in_trees = torch.Tensor([-1,-1,-1,-1]).to(self.args.device) in_tokens, structured_data = self.prepare_inputs(self.tree, self.context, self.fit, self.this_ua, self.prop) in_tokens = torch.LongTensor(in_tokens).to(self.args.device) in_trees = torch.Tensor(structured_data).to(self.args.device) if self.args.train_rl: #self.gen_logs.append([]) #self.gen_rewards.append([]) in_packed_feats, in_perm_idx = self.gen_model.embed([in_tokens], [in_trees]) encoder_out, encoder_h = self.gen_model.encode(in_packed_feats, in_perm_idx) h = self.gen_model.decoder.to_middle(encoder_h) else: with torch.no_grad(): in_packed_feats, in_perm_idx = self.gen_model.embed([in_tokens], [in_trees]) encoder_out, encoder_h = self.gen_model.encode(in_packed_feats, in_perm_idx) h = self.gen_model.decoder.to_middle(encoder_h) self.h = h #print ('input_h', h.shape) # feed in the start symbol left = [None]#*len(self.vs) parent = [None]#*len(self.vs) self.next_symbol = 'START_OUTPUT' self.get_logits_and_next_state(left, parent, [-1,-1,-1,-1]) if self.gt_step is not None: gt_tree = self.gt_step.unconstrained[self.prop.unconstrained_variables.index(self.this_ua)] self.gt_string = list(self.this_ua) #self.gt_string.insert(0, 'START_OUTPUT') self.gt_string.append('END_OF_SECTION') print ('set up a new model', self.gt_step.tree, self.prop.label, self.this_ua) print ('correct string', self.gt_string) # and assemble the stacks self.parent_symbol_stack = [None] self.parent_stack = [None] self.left_sibling_stack = [None] self.string = [] self.position_into_arity_stack = [0] self.parent_arity_stack = [-1] self.complete = False self.next_symbol = None
def _run_one_epoch(self, epoch): start = time.time() total_loss = 0 data_loader = self.data_loader # visualizing loss using visdom if self.visdom_epoch: vis_opts_epoch = dict(title=self.visdom_id + " epoch " + str(epoch), ylabel='Loss', xlabel='Epoch') vis_window_epoch = None vis_iters = torch.arange(1, len(data_loader) + 1) vis_iters_loss = torch.Tensor(len(data_loader)) for i, (data) in enumerate(data_loader): self.step += 1 text_padded, input_lengths, feat_padded, stop_token_padded, encoder_mask, decoder_mask = data if self.use_cuda: text_padded = text_padded.cuda() input_lengths = input_lengths.cuda() feat_padded = feat_padded.cuda() stop_token_padded = stop_token_padded.cuda() encoder_mask = encoder_mask.cuda() decoder_mask = decoder_mask.cuda() y_pred = self.model(text_padded, input_lengths, feat_padded, encoder_mask, decoder_mask) y_target = (feat_padded, stop_token_padded) loss = self.criterion(y_pred, y_target) self.optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm_(self.model.parameters(), self.max_norm) self.optimizer.step() total_loss += loss.item() if i % self.print_freq == 0: print('Epoch {0} | Iter {1} | Average Loss {2:.3f} | ' 'Current Loss {3:.3f} | {4:.1f} ms/batch | {5} step'. format(epoch + 1, i + 1, total_loss / (i + 1), loss.item(), 1000 * (time.time() - start) / (i + 1), self.step), flush=True) # visualizing loss using visdom if self.visdom_epoch: vis_iters_loss[i] = loss.item() if i % self.print_freq == 0: x_axis = vis_iters[:i + 1] y_axis = vis_iters_loss[:i + 1] if vis_window_epoch is None: vis_window_epoch = self.vis.line(X=x_axis, Y=y_axis, opts=vis_opts_epoch) else: self.vis.line(X=x_axis, Y=y_axis, win=vis_window_epoch, update='replace') return total_loss / (i + 1)
net = Net().to(device) if REBUILD_DATA: # Если True нормализуем и перемешиваем данные mandf = MandF() mandf.make_training_data() training_data = np.load("training_data.npy", allow_pickle=True) print(len(training_data)) optimizer = optim.Adam(net.parameters(), lr=0.001) loss_function = nn.MSELoss() # преобразовываем данные в тензоры и разбиваем на тренировочные и тестрируемые X = torch.Tensor([i[0] for i in training_data]).view(-1, 50, 50) X = X / 255.0 y = torch.Tensor([i[1] for i in training_data]) VAL_PCT = 0.1 val_size = int(len(X) * VAL_PCT) print(val_size) train_X = X[:-val_size] train_y = y[:-val_size] test_X = X[-val_size:] test_y = y[-val_size:] print(len(train_X)) print(len(test_X))
def __init__( self, env_spec, policy, qf1, qf2, replay_buffer, *, # Everything after this is numbers. max_episode_length, max_eval_path_length=None, gradient_steps_per_itr, fixed_alpha=None, target_entropy=None, initial_log_entropy=0., discount=0.99, buffer_batch_size=64, min_buffer_size=int(1e4), target_update_tau=5e-3, policy_lr=3e-4, qf_lr=3e-4, reward_scale=1.0, optimizer=torch.optim.Adam, steps_per_epoch=1, num_evaluation_trajectories=10, eval_env=None, ): self._qf1 = qf1 self._qf2 = qf2 self.replay_buffer = replay_buffer self._tau = target_update_tau self._policy_lr = policy_lr self._qf_lr = qf_lr self._initial_log_entropy = initial_log_entropy self._gradient_steps = gradient_steps_per_itr self._optimizer = optimizer self._num_evaluation_trajectories = num_evaluation_trajectories self._eval_env = eval_env self._min_buffer_size = min_buffer_size self._steps_per_epoch = steps_per_epoch self._buffer_batch_size = buffer_batch_size self._discount = discount self._reward_scale = reward_scale self.max_episode_length = max_episode_length self._max_eval_path_length = max_eval_path_length self.policy = policy self.env_spec = env_spec self.replay_buffer = replay_buffer self.sampler_cls = RaySampler self.worker_cls = FragmentWorker self._reward_scale = reward_scale # use 2 target q networks self._target_qf1 = copy.deepcopy(self._qf1) self._target_qf2 = copy.deepcopy(self._qf2) self._policy_optimizer = self._optimizer(self.policy.parameters(), lr=self._policy_lr) self._qf1_optimizer = self._optimizer(self._qf1.parameters(), lr=self._qf_lr) self._qf2_optimizer = self._optimizer(self._qf2.parameters(), lr=self._qf_lr) # automatic entropy coefficient tuning self._use_automatic_entropy_tuning = fixed_alpha is None self._fixed_alpha = fixed_alpha if self._use_automatic_entropy_tuning: if target_entropy: self._target_entropy = target_entropy else: self._target_entropy = -np.prod( self.env_spec.action_space.shape).item() self._log_alpha = torch.Tensor([self._initial_log_entropy ]).requires_grad_() self._alpha_optimizer = optimizer([self._log_alpha], lr=self._policy_lr) else: self._log_alpha = torch.Tensor([self._fixed_alpha]).log() self.episode_rewards = deque(maxlen=30)
def __init__(self, mode, input_size, hidden_size, num_layers=1, bias=True, batch_first=False, dropout=0., bidirectional=False): super(RNNBase, self).__init__() self.mode = mode self.input_size = input_size self.hidden_size = hidden_size self.num_layers = num_layers self.bias = bias self.batch_first = batch_first self.dropout = float(dropout) self.bidirectional = bidirectional num_directions = 2 if bidirectional else 1 if not isinstance(dropout, numbers.Number) or not 0 <= dropout <= 1 or \ isinstance(dropout, bool): raise ValueError("dropout should be a number in range [0, 1] " "representing the probability of an element being " "zeroed") if dropout > 0 and num_layers == 1: warnings.warn("dropout option adds dropout after all but last " "recurrent layer, so non-zero dropout expects " "num_layers greater than 1, but got dropout={} and " "num_layers={}".format(dropout, num_layers)) if mode == 'LSTM': gate_size = 4 * hidden_size elif mode == 'GRU': gate_size = 3 * hidden_size elif mode == 'RNN_TANH': gate_size = hidden_size elif mode == 'RNN_RELU': gate_size = hidden_size else: raise ValueError("Unrecognized RNN mode: " + mode) self._flat_weights_names = [] self._all_weights = [] for layer in range(num_layers): for direction in range(num_directions): layer_input_size = input_size if layer == 0 else hidden_size * num_directions w_ih = Parameter(torch.Tensor(gate_size, layer_input_size)) w_hh = Parameter(torch.Tensor(gate_size, hidden_size)) b_ih = Parameter(torch.Tensor(gate_size)) # Second bias vector included for CuDNN compatibility. Only one # bias vector is needed in standard definition. b_hh = Parameter(torch.Tensor(gate_size)) layer_params = (w_ih, w_hh, b_ih, b_hh) suffix = '_reverse' if direction == 1 else '' param_names = ['weight_ih_l{}{}', 'weight_hh_l{}{}'] if bias: param_names += ['bias_ih_l{}{}', 'bias_hh_l{}{}'] param_names = [x.format(layer, suffix) for x in param_names] for name, param in zip(param_names, layer_params): setattr(self, name, param) self._flat_weights_names.extend(param_names) self._all_weights.append(param_names) self._flat_weights = [(lambda wn: getattr(self, wn) if hasattr(self, wn) else None)(wn) for wn in self._flat_weights_names] self.flatten_parameters() self.reset_parameters()
def __init__(self, *, backbone, encoder, decoder, anchor_generator, box2box_transform, anchor_matcher, num_classes, backbone_level="res5", pos_ignore_thresh=0.15, neg_ignore_thresh=0.7, focal_loss_alpha=0.25, focal_loss_gamma=2.0, box_reg_loss_type="giou", test_score_thresh=0.05, test_topk_candidates=1000, test_nms_thresh=0.6, max_detections_per_image=100, pixel_mean, pixel_std, vis_period=0, input_format="BGR"): """ NOTE: this interface is experimental. Args: backbone: a backbone module, must follow detectron2's backbone interface encoder (nn.Module): a module that encodes feature from backbone decoder (nn.Module): a module to generate cls_score and box_reg anchor_generator (nn.Module): a module that creates anchors from a list of features. Usually an instance of :class:`AnchorGenerator` box2box_transform (Box2BoxTransform): defines the transform from anchors boxes to instance boxes anchor_matcher (Matcher): label the anchors by matching them with ground truth. num_classes (int): number of classes. Used to label background proposals. # ignore thresholds pos_ignore_thresh (float): the threshold to ignore positive anchors neg_ignore_thresh (float): the threshold to ignore negative anchors # Loss parameters: focal_loss_alpha (float): focal_loss_alpha focal_loss_gamma (float): focal_loss_gamma box_reg_loss_type (str): Options are "smooth_l1", "giou" # Inference parameters: test_score_thresh (float): Inference cls score threshold, only anchors with score > INFERENCE_TH are considered for inference (to improve speed) test_topk_candidates (int): Select topk candidates before NMS test_nms_thresh (float): Overlap threshold used for non-maximum suppression (suppress boxes with IoU >= this threshold) max_detections_per_image (int): Maximum number of detections to return per image during inference (100 is based on the limit established for the COCO dataset). # Input parameters pixel_mean (Tuple[float]): Values to be used for image normalization (BGR order). To train on images of different number of channels, set different mean & std. Default values are the mean pixel value from ImageNet: [103.53, 116.28, 123.675] pixel_std (Tuple[float]): When using pre-trained models in Detectron1 or any MSRA models, std has been absorbed into its conv1 weights, so the std needs to be set 1. Otherwise, you can use [57.375, 57.120, 58.395] (ImageNet std) vis_period (int): The period (in terms of steps) for minibatch visualization at train time. Set to 0 to disable. input_format (str): Whether the model needs RGB, YUV, HSV etc. """ super().__init__() self.backbone = backbone self.encoder = encoder self.decoder = decoder # Anchors self.anchor_generator = anchor_generator self.box2box_transform = box2box_transform self.anchor_matcher = anchor_matcher self.num_classes = num_classes self.backbone_level = backbone_level # Ignore thresholds: self.pos_ignore_thresh = pos_ignore_thresh self.neg_ignore_thresh = neg_ignore_thresh # Loss parameters: self.focal_loss_alpha = focal_loss_alpha self.focal_loss_gamma = focal_loss_gamma self.box_reg_loss_type = box_reg_loss_type assert self.box_reg_loss_type == 'giou', "Only support GIoU Loss." # Inference parameters: self.test_score_thresh = test_score_thresh self.test_topk_candidates = test_topk_candidates self.test_nms_thresh = test_nms_thresh self.max_detections_per_image = max_detections_per_image # Vis parameters self.vis_period = vis_period self.input_format = input_format self.register_buffer("pixel_mean", torch.Tensor(pixel_mean).view(-1, 1, 1)) self.register_buffer("pixel_std", torch.Tensor(pixel_std).view(-1, 1, 1))
def test_span_metrics_are_computed_correctly(self): gold_labels = [ "O", "B-ARG1", "I-ARG1", "O", "B-ARG2", "I-ARG2", "O", "O", "O" ] gold_indices = [ self.vocab.get_token_index(x, "tags") for x in gold_labels ] gold_tensor = torch.Tensor([gold_indices]) prediction_tensor = torch.rand( [2, 9, self.vocab.get_vocab_size("tags")]) # Test that the span measure ignores completely masked sequences by # passing a mask with a fully masked row. mask = torch.LongTensor([[1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]]) prediction_tensor[:, 0, 0] = 1 prediction_tensor[:, 1, 1] = 1 # (True positive - ARG1 prediction_tensor[:, 2, 2] = 1 # *) prediction_tensor[:, 3, 0] = 1 prediction_tensor[:, 4, 0] = 1 # (False Negative - ARG2 prediction_tensor[:, 5, 0] = 1 # *) prediction_tensor[:, 6, 0] = 1 prediction_tensor[:, 7, 1] = 1 # (False Positive - ARG1 prediction_tensor[:, 8, 2] = 1 # *) metric = SpanBasedF1Measure(self.vocab, "tags") metric(prediction_tensor, gold_tensor, mask) assert metric._true_positives["ARG1"] == 1 assert metric._true_positives["ARG2"] == 0 assert "O" not in metric._true_positives.keys() assert metric._false_negatives["ARG1"] == 0 assert metric._false_negatives["ARG2"] == 1 assert "O" not in metric._false_negatives.keys() assert metric._false_positives["ARG1"] == 1 assert metric._false_positives["ARG2"] == 0 assert "O" not in metric._false_positives.keys() # Check things are accumulating correctly. metric(prediction_tensor, gold_tensor, mask) assert metric._true_positives["ARG1"] == 2 assert metric._true_positives["ARG2"] == 0 assert "O" not in metric._true_positives.keys() assert metric._false_negatives["ARG1"] == 0 assert metric._false_negatives["ARG2"] == 2 assert "O" not in metric._false_negatives.keys() assert metric._false_positives["ARG1"] == 2 assert metric._false_positives["ARG2"] == 0 assert "O" not in metric._false_positives.keys() metric_dict = metric.get_metric() numpy.testing.assert_almost_equal(metric_dict["recall-ARG2"], 0.0) numpy.testing.assert_almost_equal(metric_dict["precision-ARG2"], 0.0) numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG2"], 0.0) numpy.testing.assert_almost_equal(metric_dict["recall-ARG1"], 1.0) numpy.testing.assert_almost_equal(metric_dict["precision-ARG1"], 0.5) numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG1"], 0.666666666) numpy.testing.assert_almost_equal(metric_dict["recall-overall"], 0.5) numpy.testing.assert_almost_equal(metric_dict["precision-overall"], 0.5) numpy.testing.assert_almost_equal(metric_dict["f1-measure-overall"], 0.5)
# convert the roi to grayscale and blur it gray = cv2.cvtColor(roi, cv2.COLOR_BGR2GRAY) gray = cv2.GaussianBlur(gray, (7, 7), 0) # cv2.drawContours(clone, [segmented + (right, top)], -1, (0, 0, 255)) img = gray # img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # img=cv2.imread("240fn.jpg",cv2.IMREAD_GRAYSCALE) # img=cv2.cvtColor(bw_image,cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (IMG_SIZE, IMG_SIZE)) test_data = img orig = img data = torch.Tensor(img).view(-1, 1, 96, 96) net_out = net(data) # returns a list, # print("net_out",net_out) predicted_class = torch.argmax(net_out) # print("predicted_class",predicted_class) pnb = predicted_class.item() # print("pnb",pnb) # print(str(np.argmax(predicted_class))+" "+str(out_label[pnb])) pre.append(out_label[pnb]) if out_label[pnb] == 'blnk': pass else:
def test_span_metrics_are_computed_correcly_with_prediction_map(self): # In this example, datapoint1 only has access to ARG1 and V labels, # whereas datapoint2 only has access to ARG2 and V labels. # gold_labels = [["O", "B-ARG1", "I-ARG1", "O", "B-V", "O"], # ["B-ARG2", "I-ARG2", "O", "B-V", "I-V", "O"]] gold_indices = [[0, 1, 2, 0, 3, 0], [1, 2, 0, 3, 4, 0]] prediction_map_indices = [[0, 1, 2, 5, 6], [0, 3, 4, 5, 6]] gold_tensor = torch.Tensor(gold_indices) prediction_map_tensor = torch.Tensor(prediction_map_indices) prediction_tensor = torch.rand([2, 6, 5]) prediction_tensor[0, 0, 0] = 1 prediction_tensor[0, 1, 1] = 1 # (True Positive - ARG1 prediction_tensor[0, 2, 2] = 1 # *) prediction_tensor[0, 3, 0] = 1 prediction_tensor[0, 4, 3] = 1 # (True Positive - V) prediction_tensor[0, 5, 1] = 1 # (False Positive - ARG1) prediction_tensor[1, 0, 0] = 1 # (False Negative - ARG2 prediction_tensor[1, 1, 0] = 1 # *) prediction_tensor[1, 2, 0] = 1 prediction_tensor[1, 3, 3] = 1 # (True Positive - V prediction_tensor[1, 4, 4] = 1 # *) prediction_tensor[1, 5, 1] = 1 # (False Positive - ARG2) metric = SpanBasedF1Measure(self.vocab, "tags") metric(prediction_tensor, gold_tensor, prediction_map=prediction_map_tensor) assert metric._true_positives["ARG1"] == 1 assert metric._true_positives["ARG2"] == 0 assert metric._true_positives["V"] == 2 assert "O" not in metric._true_positives.keys() assert metric._false_negatives["ARG1"] == 0 assert metric._false_negatives["ARG2"] == 1 assert metric._false_negatives["V"] == 0 assert "O" not in metric._false_negatives.keys() assert metric._false_positives["ARG1"] == 1 assert metric._false_positives["ARG2"] == 1 assert metric._false_positives["V"] == 0 assert "O" not in metric._false_positives.keys() # Check things are accumulating correctly. metric(prediction_tensor, gold_tensor, prediction_map=prediction_map_tensor) assert metric._true_positives["ARG1"] == 2 assert metric._true_positives["ARG2"] == 0 assert metric._true_positives["V"] == 4 assert "O" not in metric._true_positives.keys() assert metric._false_negatives["ARG1"] == 0 assert metric._false_negatives["ARG2"] == 2 assert metric._false_negatives["V"] == 0 assert "O" not in metric._false_negatives.keys() assert metric._false_positives["ARG1"] == 2 assert metric._false_positives["ARG2"] == 2 assert metric._false_positives["V"] == 0 assert "O" not in metric._false_positives.keys() metric_dict = metric.get_metric() numpy.testing.assert_almost_equal(metric_dict["recall-ARG2"], 0.0) numpy.testing.assert_almost_equal(metric_dict["precision-ARG2"], 0.0) numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG2"], 0.0) numpy.testing.assert_almost_equal(metric_dict["recall-ARG1"], 1.0) numpy.testing.assert_almost_equal(metric_dict["precision-ARG1"], 0.5) numpy.testing.assert_almost_equal(metric_dict["f1-measure-ARG1"], 0.666666666) numpy.testing.assert_almost_equal(metric_dict["recall-V"], 1.0) numpy.testing.assert_almost_equal(metric_dict["precision-V"], 1.0) numpy.testing.assert_almost_equal(metric_dict["f1-measure-V"], 1.0) numpy.testing.assert_almost_equal(metric_dict["recall-overall"], 0.75) numpy.testing.assert_almost_equal(metric_dict["precision-overall"], 0.6) numpy.testing.assert_almost_equal(metric_dict["f1-measure-overall"], 0.666666666)
global_step += 1 * args.num_envs obs[step] = next_obs dones[step] = next_done # ALGO LOGIC: put action logic here with torch.no_grad(): values[step] = agent.get_value(obs[step]).flatten() action, logproba, _, invalid_action_masks[step] = agent.get_action( obs[step], envs=envs) actions[step] = action.T logprobs[step] = logproba # TRY NOT TO MODIFY: execute the game and log data. next_obs, rs, ds, infos = envs.step(action.T) rewards[step], next_done = rs.view(-1), torch.Tensor(ds).to(device) for info in infos: if 'episode' in info.keys(): print( f"global_step={global_step}, episode_reward={info['episode']['r']}" ) writer.add_scalar("charts/episode_reward", info['episode']['r'], global_step) for key in info['microrts_stats']: writer.add_scalar(f"charts/episode_reward/{key}", info['microrts_stats'][key], global_step) break # bootstrap reward if not done. reached the batch limit with torch.no_grad():
def forward(self, predictions, priors, targets): """Multibox Loss Args: predictions (tuple): A tuple containing loc preds, conf preds, and prior boxes from SSD net. conf shape: torch.size(batch_size,num_priors,num_classes) loc shape: torch.size(batch_size,num_priors,4) priors shape: torch.size(num_priors,4) ground_truth (tensor): Ground truth boxes and labels for a batch, shape: [batch_size,num_objs,5] (last idx is the label). """ loc_data, conf_data, landm_data = predictions priors = priors num = loc_data.size(0) num_priors = (priors.size(0)) # match priors (default boxes) and ground truth boxes loc_t = torch.Tensor(num, num_priors, 4) landm_t = torch.Tensor(num, num_priors, 10) conf_t = torch.LongTensor(num, num_priors) for idx in range(num): truths = targets[idx][:, :4].data labels = targets[idx][:, -1].data landms = targets[idx][:, 4:14].data defaults = priors.data match(self.threshold, truths, defaults, self.variance, labels, landms, loc_t, conf_t, landm_t, idx) if GPU: loc_t = loc_t.cuda() conf_t = conf_t.cuda() landm_t = landm_t.cuda() zeros = torch.tensor(0).cuda() # landm Loss (Smooth L1) # Shape: [batch,num_priors,10] pos1 = conf_t > zeros num_pos_landm = pos1.long().sum(1, keepdim=True) N1 = max(num_pos_landm.data.sum().float(), 1) pos_idx1 = pos1.unsqueeze(pos1.dim()).expand_as(landm_data) landm_p = landm_data[pos_idx1].view(-1, 10) landm_t = landm_t[pos_idx1].view(-1, 10) loss_landm = F.smooth_l1_loss(landm_p, landm_t, reduction='sum') pos = conf_t != zeros conf_t[pos] = 1 # Localization Loss (Smooth L1) # Shape: [batch,num_priors,4] pos_idx = pos.unsqueeze(pos.dim()).expand_as(loc_data) loc_p = loc_data[pos_idx].view(-1, 4) loc_t = loc_t[pos_idx].view(-1, 4) loss_l = F.smooth_l1_loss(loc_p, loc_t, reduction='sum') # Compute max conf across batch for hard negative mining batch_conf = conf_data.view(-1, self.num_classes) loss_c = log_sum_exp(batch_conf) - batch_conf.gather( 1, conf_t.view(-1, 1)) # Hard Negative Mining loss_c[pos.view(-1, 1)] = 0 # filter out pos boxes for now loss_c = loss_c.view(num, -1) _, loss_idx = loss_c.sort(1, descending=True) _, idx_rank = loss_idx.sort(1) num_pos = pos.long().sum(1, keepdim=True) num_neg = torch.clamp(self.negpos_ratio * num_pos, max=pos.size(1) - 1) neg = idx_rank < num_neg.expand_as(idx_rank) # Confidence Loss Including Positive and Negative Examples pos_idx = pos.unsqueeze(2).expand_as(conf_data) neg_idx = neg.unsqueeze(2).expand_as(conf_data) conf_p = conf_data[(pos_idx + neg_idx).gt(0)].view( -1, self.num_classes) targets_weighted = conf_t[(pos + neg).gt(0)] loss_c = F.cross_entropy(conf_p, targets_weighted, reduction='sum') # Sum of losses: L(x,c,l,g) = (Lconf(x, c) + αLloc(x,l,g)) / N N = max(num_pos.data.sum().float(), 1) loss_l /= N loss_c /= N loss_landm /= N1 return loss_l, loss_c, loss_landm
def _add_gaussian_noise(self, single_view): return single_view + torch.Tensor(*single_view.shape).normal_(std=self.gaussian_noise_std).to(self.device)
def generate_batch(nb, height, width): batch_images = torch.Tensor(nb, height, width, 3) for i in tqdm(range(nb)): batch_images[i] = torch.from_numpy(random_shapes((height, width), 6, 3, min_size=30, num_trials=25)[0]) return batch_images.permute(0,3,1,2)
def test(self): all_psnr = [] # All signal to noise ratios over all the batches all_percentages = [] # All percentage accuracies dummy_metrics = [] # Metrics for the averaging value # Load the score network states = torch.load(os.path.join(self.args.log, 'checkpoint.pth'), map_location=self.config.device) scorenet = CondRefineNetDilated(self.config).to(self.config.device) scorenet = torch.nn.DataParallel(scorenet) scorenet.load_state_dict(states[0]) scorenet.eval() # Grab the first two samples from MNIST trans = transforms.Compose([transforms.ToTensor()]) dataset = MNIST(os.path.join(self.args.run, 'datasets', 'mnist'), train=False, download=True) first_digits_idx = dataset.train_labels <= 4 second_digits_idx = dataset.train_labels >=5 first_digits = dataset.train_data[first_digits_idx] second_digits = dataset.train_data[second_digits_idx] for iteration in range(100): print("Iteration {}".format(iteration)) curr_dir = os.path.join(SAVE_DIR, "{:07d}".format(iteration)) if not os.path.exists(curr_dir): os.makedirs(curr_dir) # image1, image2 = get_images_split(first_digits, second_digits) gt_images = [] for _ in range(N): gt_images.append(get_single_image(dataset)) mixed = sum(gt_images).float() mixed_grid = make_grid(mixed.detach() / float(N), nrow=GRID_SIZE, pad_value=1., padding=1) save_image(mixed_grid, os.path.join(curr_dir, "mixed.png")) for i in range(N): gt_grid = make_grid(gt_images[i], nrow=GRID_SIZE, pad_value=1., padding=1) save_image(gt_grid, os.path.join(curr_dir, "gt{}.png".format(i))) mixed = torch.Tensor(mixed).cuda().view(BATCH_SIZE, 1, 28, 28) xs = [] for _ in range(N): xs.append(nn.Parameter(torch.Tensor(BATCH_SIZE, 1, 28, 28).uniform_()).cuda()) step_lr=0.00002 # Noise amounts sigmas = np.array([1., 0.59948425, 0.35938137, 0.21544347, 0.12915497, 0.07742637, 0.04641589, 0.02782559, 0.01668101, 0.01]) n_steps_each = 100 for idx, sigma in enumerate(sigmas): # if idx <= 5: # lambda_recon = 1./(sigma**2) # else: # lambda_recon = 2./(sigma**2) lambda_recon = 0.5/(sigma**2) # Not completely sure what this part is for labels = torch.ones(1, device=xs[0].device) * idx labels = labels.long() step_size = step_lr * (sigma / sigmas[-1]) ** 2 for step in range(n_steps_each): noises = [] for _ in range(N): noises.append(torch.randn_like(xs[0]) * np.sqrt(step_size * 2)) grads = [] for i in range(N): grads.append(scorenet(xs[i].view(BATCH_SIZE, 1, 28, 28), labels).detach()) recon_loss = (torch.norm(torch.flatten(sum(xs) - mixed)) ** 2) recon_grads = torch.autograd.grad(recon_loss, xs) for i in range(N): xs[i] = xs[i] + (step_size * grads[i]) + (-step_size * lambda_recon * recon_grads[i].detach()) + noises[i] # x = x + (-step_size * lambda_recon * recon_grads[0].detach()) + noise_x # y = y + (-step_size * lambda_recon * recon_grads[1].detach()) + noise_y for i in range(N): xs[i] = torch.clamp(xs[i], 0, 1) x_to_write = [] for i in range(N): x_to_write.append(torch.Tensor(xs[i].detach().cpu())) # PSNR Measure for idx in range(BATCH_SIZE): best_psnr = -10000 best_permutation = None for permutation in permutations(range(N)): curr_psnr = sum([psnr(xs[permutation[i]][idx], gt_images[i][idx].cuda()) for i in range(N)]) if curr_psnr > best_psnr: best_psnr = curr_psnr best_permutation = permutation all_psnr.append(best_psnr / float(N)) for i in range(N): x_to_write[i][idx] = xs[best_permutation[i]][idx] mixed_psnr = psnr(mixed.detach().cpu()[idx] / float(N), gt_images[i][idx]) dummy_metrics.append(mixed_psnr) for i in range(N): x_grid = make_grid(x_to_write[i], nrow=GRID_SIZE, pad_value=1., padding=1) save_image(x_grid, os.path.join(curr_dir, "x{}.png".format(i))) mixed_grid = make_grid(sum(xs)/float(N), nrow=GRID_SIZE, pad_value=1., padding=1) save_image(mixed_grid, os.path.join(curr_dir, "recon.png".format(i))) # average_grid = make_grid(mixed.detach()/2., nrow=GRID_SIZE) # save_image(average_grid, "results/average_cifar.png") print("Curr mean {}".format(np.array(all_psnr).mean())) print("Const mean {}".format(np.array(dummy_metrics).mean()))
img[:,out_color.shape[1] : out_color.shape[1]*2, :] = out print(img.shape) img = cv2.resize(img, (2*target_res[1], target_res[0])) cv2.imwrite("%d_depth.jpg" % i, img) from torch.autograd import Variable #inputs=torch.rand((1,3,240,320),dtype=torch.float32) #input_var = Variable(inputs.cpu()) im = cv2.imread("girl.png") im = cv2.resize(im,(320,240)) im = np.asarray(im,dtype=np.float32) / 255 im = im - 0.5 imt = im.transpose(2, 0, 1) inputs = torch.Tensor([imt]) print(inputs.shape) input_var = Variable(inputs) print(input_var.size(),input_var.type()) output_var = model(input_var.cpu()) print(output_var.size(),output_var.type()) target_res=[240,320] vis_depth(output_var.data.cpu().numpy(), inputs.numpy(), 0, target_res) # Commented out IPython magic to ensure Python compatibility. # %cd /content/YouTube3D def convert_to_onnx(net, output_name): input = torch.randn(1,3,240,320) input_names = ['data']
model.load_state_dict(torch.load("model/chkpoint_000.pt")) model.cuda() # read the adjustment network checkpoint adjustmentnet.load_state_dict(torch.load("model/chkpoint_adj_000.pt")) adjustmentnet.cuda() # read the img file from system argument filenamein = sys.argv[1] original = io.imread(filenamein) new_h,new_w,d = original.shape # reshape image to 224x224 batchsize=1 resized = cv2.resize(original,(224,224),interpolation=cv2.INTER_AREA) img = torch.Tensor(resized).float().cuda() img = img.permute(2,0,1).unsqueeze(0) # apply model on input y_pred = model(img) alphas = y_pred[:,:199] betas = y_pred[:,199:228] s = y_pred[:,228] t = torch.tanh(y_pred[:,229:231]) r = torch.tanh(y_pred[:,231:235]) * (3.14/4) # create 3dmm alpha_matrix = alphas.unsqueeze(2).expand(*alphas.size(),alphas.size(1)) * torch.eye(alphas.size(1)).cuda() beta_matrix = betas.unsqueeze(2).expand(*betas.size(),betas.size(1))*torch.eye(betas.size(1)).cuda() shape_cov = torch.bmm(torch.stack(batchsize*[face3dmm.shape_eigenvec]),alpha_matrix) exp_cov = torch.bmm(torch.stack(batchsize*[face3dmm.exp_eigenvec]),beta_matrix)
def process_file(input_file, output_file, use_gpu): write_out("Processing raw data file", input_file) # create output file file = h5py.File(output_file, 'w') current_buffer_size = 1 current_buffer_allocation = 0 dset1 = file.create_dataset('primary', (current_buffer_size, MAX_SEQUENCE_LENGTH), maxshape=(None, MAX_SEQUENCE_LENGTH), dtype='int32') dset2 = file.create_dataset('tertiary', (current_buffer_size, MAX_SEQUENCE_LENGTH, 9), maxshape=(None, MAX_SEQUENCE_LENGTH, 9), dtype='float') dset3 = file.create_dataset('mask', (current_buffer_size, MAX_SEQUENCE_LENGTH), maxshape=(None, MAX_SEQUENCE_LENGTH), dtype='uint8') input_file_pointer = open(input_file, "r") while True: # while there's more proteins to process next_protein, missing_aa = read_protein_from_file(input_file_pointer) if next_protein is None: # no more proteins to process break sequence_length = len(next_protein['primary']) if sequence_length > MAX_SEQUENCE_LENGTH: write_out("Dropping protein as length too long:", sequence_length) continue if missing_aa is True: continue if current_buffer_allocation >= current_buffer_size: current_buffer_size = current_buffer_size + 1 dset1.resize((current_buffer_size, MAX_SEQUENCE_LENGTH)) dset2.resize((current_buffer_size, MAX_SEQUENCE_LENGTH, 9)) dset3.resize((current_buffer_size, MAX_SEQUENCE_LENGTH)) primary_padded = np.zeros(MAX_SEQUENCE_LENGTH) tertiary_padded = np.zeros((9, MAX_SEQUENCE_LENGTH)) mask_padded = np.zeros(MAX_SEQUENCE_LENGTH) # masking and padding here happens so that the stored dataset is of the same size. # when the data is loaded in this padding is removed again. primary_padded[:sequence_length] = next_protein['primary'] t_transposed = np.ravel(np.array(next_protein['tertiary']).T) t_reshaped = np.reshape(t_transposed, (sequence_length, 9)).T tertiary_padded[:, :sequence_length] = t_reshaped mask_padded[:sequence_length] = next_protein['mask'] mask = torch.Tensor(mask_padded).type(dtype=torch.bool) prim = torch.masked_select(torch.Tensor(primary_padded)\ .type(dtype=torch.long), mask) pos = torch.masked_select(torch.Tensor(tertiary_padded), mask)\ .view(9, -1).transpose(0, 1).unsqueeze(1) pos_angstrom = pos / 100 if use_gpu: pos_angstrom = pos_angstrom.cuda() # map to angles and back to tertiary angles, batch_sizes = calculate_dihedral_angles_over_minibatch(pos_angstrom, torch.tensor([len(prim)]), use_gpu=use_gpu) tertiary, _ = get_backbone_positions_from_angles(angles, batch_sizes, use_gpu=use_gpu) tertiary = tertiary.squeeze(1) # create variables to store padded sequences in primary_padded = np.zeros(MAX_SEQUENCE_LENGTH) tertiary_padded = np.zeros((MAX_SEQUENCE_LENGTH, 9)) mask_padded = np.zeros(MAX_SEQUENCE_LENGTH) # store padded sequences length_after_mask_removed = len(prim) primary_padded[:length_after_mask_removed] = prim.data.cpu().numpy() tertiary_padded[:length_after_mask_removed, :] = tertiary.data.cpu().numpy() mask_padded[:length_after_mask_removed] = np.ones(length_after_mask_removed) # save padded sequences on disk dset1[current_buffer_allocation] = primary_padded dset2[current_buffer_allocation] = tertiary_padded dset3[current_buffer_allocation] = mask_padded current_buffer_allocation += 1 if current_buffer_allocation == 0: write_out("Preprocessing was selected but no proteins in the input file " "were accepted. Please check your input.") os._exit(1) write_out("Wrote output to", current_buffer_allocation, "proteins to", output_file)