def forward(self, next_input: torch.Tensor) -> torch.Tensor: # move the input between devices next_input = next_input.to(self.device) if self.cur_mode is ForwardMode.backward: return self.backward_mode(next_input) # check if the input that waits for the submodule is relevant (garbage # will be propagated before and after data passes through submodule). if self.counter.is_last_input_valid(self.gpu_num): # the input is relevant. cur_input = self.last_input with autograd.no_grad(): output = self.module(cur_input) else: # the input is garbage. output = torch.zeros(*self.output_shape).to(self.device) # check if the input to be replaced and scheduled to run on the next if self.counter.is_input_valid(self.gpu_num): if self.cur_mode is ForwardMode.train: self.save_activation(next_input) self.last_input = next_input else: self.last_input = None return output
def test(epoch, length, dataloader, model, writer=None, batch_size=1): print('Testing start...') model.eval() sum_pre = 0.0 sum_recall = 0.0 sum_ndcg_score = 0.0 num_val = 0 with no_grad(): val_pbar = tqdm(total=length) for user_tensor, item_tensor in dataloader: val_pbar.update(batch_size) precision, recall, ndcg_score = model.accuracy( user_tensor, item_tensor) if recall < 0: continue sum_pre += precision sum_recall += recall sum_ndcg_score += ndcg_score num_val += batch_size val_pbar.close() print( '---------------------------------{0}-th Precition:{1:.4f} Recall:{2:.4f} NDCG:{3:.4f}---------------------------------' .format(epoch, sum_pre / num_val, sum_recall / num_val, sum_ndcg_score / num_val))
def full_vt(epoch, model, data, prefix, writer=None): print(prefix+' start...') model.eval() with no_grad(): precision, recall, ndcg_score = model.full_accuracy(data) print('---------------------------------{0}-th Precition:{1:.4f} Recall:{2:.4f} NDCG:{3:.4f}---------------------------------'.format( epoch, precision, recall, ndcg_score)) if writer is not None: writer.add_scalar(prefix+'_Precition', precision, epoch) writer.add_scalar(prefix+'_Recall', recall, epoch) writer.add_scalar(prefix+'_NDCG', ndcg_score, epoch) writer.add_histogram(prefix+'_visual_distribution', model.v_rep, epoch) writer.add_histogram(prefix+'_acoustic_distribution', model.a_rep, epoch) writer.add_histogram(prefix+'_textual_distribution', model.t_rep, epoch) writer.add_histogram(prefix+'_user_visual_distribution', model.user_preferences[:,:44], epoch) writer.add_histogram(prefix+'_user_acoustic_distribution', model.user_preferences[:, 44:-44], epoch) writer.add_histogram(prefix+'_user_textual_distribution', model.user_preferences[:, -44:], epoch) writer.add_embedding(model.v_rep) #writer.add_embedding(model.a_rep) #writer.add_embedding(model.t_rep) #writer.add_embedding(model.user_preferences[:,:44]) #writer.add_embedding(model.user_preferences[:, 44:-44]) #writer.add_embedding(model.user_preferences[:, -44:]) return precision, recall, ndcg_score
def get_laplacian_weights(points, normals, iso_points, iso_normals, neighborhood_size=8): """ compute distance based on iso local neighborhood """ with autograd.no_grad(): P, _ = points.view(-1, 3).shape search_radius = 0.15 dim = iso_points.view(-1, 3).norm(dim=-1).max() * 2 avg_spacing = iso_points.shape[1] / dim / 16 dists, idxs, nn, _ = frnn.frnn_grid_points(points, iso_points, K=1, return_nn=True, grid=None, r=search_radius) nn_normals = frnn.frnn_gather(iso_normals, idxs) dists = torch.sum((points - nn.view_as(points)) * (normals + nn_normals.view_as(normals)), dim=-1) dists = dists * dists spatial_w = torch.exp(-dists * avg_spacing) spatial_w[idxs.view_as(spatial_w) < 0] = 0 return spatial_w.view(points.shape[:-1])
def extract_feature(raw_texts): # if os.path.exists(os.path.join(PROOT, 'chinese_L-12_H-768_A-12', 'pytorch_model.bin')): # bert_model = BertModel.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12')).cuda() # else: # bert_model = BertModel.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12'), from_tf=True) # torch.save(bert_model.state_dict(), os.path.join(PROOT, 'chinese_L-12_H-768_A-12', 'pytorch_model.bin')) # bert_model = BertModel.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12')).cuda() bert_model = BertModel.from_pretrained('bert-base-chinese').cuda() tokenizer = BertTokenizer.from_pretrained('bert-base-chinese') tokenizer.max_len = 20000 # tokenizer = BertTokenizer.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12/vocab.txt')) bert_model.eval() if type(raw_texts) is not list: raw_texts = [raw_texts] ids = [] for text in raw_texts: tokens = tokenizer.tokenize(text) ids.append( tokenizer.convert_tokens_to_ids(['[CLS]'] + tokens + ['[SEP]'])) x, mask = pad_sequences(ids, MAX_LEN) x = torch.LongTensor(x) mask = torch.LongTensor(mask) with autograd.no_grad(): _, pooled = bert_model(x.cuda(), attention_mask=mask.cuda(), output_all_encoded_layers=False) return pooled
def shape_inference(model, tensor, dtype=None): if isinstance(tensor, (tuple, list)): tensor = torch.zeros(tensor, dtype=dtype) model.eval() with autograd.no_grad(): model(tensor) replace_modules(model, flex_replacer)
def train_eval(epoch, model, prefix, ranklist, args, result_log_path, writer=None): print(prefix + ' start...') model.eval() with no_grad(): precision_10, recall_10, ndcg_score_10, precision_5, recall_5, ndcg_score_5, precision_1, recall_1, ndcg_score_1 = model.accuracy( ranklist) print( '---------------------------------{0}-th Precition:{1:.4f} Recall:{2:.4f} NDCG:{3:.4f}---------------------------------' .format(epoch, precision_10, recall_10, ndcg_score_10)) with open(result_log_path, "a") as f: f.write( '---------------------------------Tra: {0}-th epoch {1}-th top Precition:{2:.4f} Recall:{3:.4f} NDCG:{4:.4f}---------------------------------' .format(epoch, 10, precision_10, recall_10, ndcg_score_10)) # 将字符串写入文件中 f.write("\n") with open(result_log_path, "a") as f: f.write( '---------------------------------Tra: {0}-th epoch {1}-th top Precition:{2:.4f} Recall:{3:.4f} NDCG:{4:.4f}---------------------------------' .format(epoch, 5, precision_5, recall_5, ndcg_score_5)) # 将字符串写入文件中 f.write("\n") with open(result_log_path, "a") as f: f.write( '---------------------------------Tra: {0}-th epoch {1}-th top Precition:{2:.4f} Recall:{3:.4f} NDCG:{4:.4f}---------------------------------' .format(epoch, 1, precision_1, recall_1, ndcg_score_1)) # 将字符串写入文件中 f.write("\n")
def val(self, test_control=True): print('Running Testing') if self.args.test_prediction: start = time.time() self.model_val.load_state_dict(self.model.state_dict()) if self._hp.model_test is not None: self.model_test.load_state_dict(self.model.state_dict()) losses_meter = RecursiveAverageMeter() with autograd.no_grad(): for batch_idx, sample_batched in enumerate(self.val_loader): inputs = AttrDict(map_dict(lambda x: x.to(self.device), sample_batched)) output = self.model_val(inputs) losses = self.model_val.loss(output) if self._hp.model_test is not None: run_through_traj(self.model_test, inputs) losses_meter.update(losses) del losses if self.run_testmetrics: print("Finished Evaluation! Exiting...") exit(0) self.model_val.log_outputs( output, inputs, losses_meter.avg, self.global_step, log_images=True, phase='val') print(('\nTest set: Average loss: {:.4f} in {:.2f}s\n' .format(losses_meter.avg.total_loss.item(), time.time() - start))) del output
def get_heat_kernel_weights(points, normals, iso_points, iso_normals, neighborhood_size=8, sigma_p=0.4, sigma_n=0.7): """ find closest k points, compute point2face distance, and normal distance """ P, _ = points.view(-1, 3).shape search_radius = 0.15 dim = iso_points.view(-1, 3).norm(dim=-1).max() avg_spacing = iso_points.shape[1] / (dim * 2**2) / 16 dists, idxs, nn, _ = frnn.frnn_grid_points(points, iso_points, K=neighborhood_size, return_nn=True, grid=None, r=search_radius) # features with autograd.no_grad(): # normalize just to be sure iso_normals = F.normalize(iso_normals, dim=-1, eps=1e-15) normals = F.normalize(normals, dim=-1, eps=1e-15) # features are composite of points and normals features = torch.cat([points / sigma_p, normals / sigma_n], dim=-1) features_iso = torch.cat([iso_points / sigma_p, iso_normals / sigma_n], dim=-1) # compute kernels (N,P,K) k(x,xi), xi \in Neighbor(x) knn_idx = idxs # features_nb = knn_gather(features_iso, knn_idx) features_nb = frnn.frnn_gather(features_iso, knn_idx) # (N,P,K,D) features_diff = features.unsqueeze(2) - features_nb features_dist = torch.sum(features_diff**2, dim=-1) kernels = torch.exp(-features_dist) kernels[knn_idx < 0] = 0 # N,P,K,K,D features_diff_ij = features_nb[:, :, :, None, :] - features_nb[:, :, None, :, :] features_dist_ij = torch.sum(features_diff_ij**2, dim=-1) kernel_matrices = torch.exp(-features_dist_ij) kernel_matrices[knn_idx < 0] = 0 kernel_matrices[knn_idx.unsqueeze(-2).expand_as(kernel_matrices) < 0] kernel_matrices_inv = pinverse(kernel_matrices) weight = kernels.unsqueeze( -2) @ kernel_matrices_inv @ kernels.unsqueeze(-1) weight.clamp_max_(1.0) return weight.view(points.shape[:-1])
def transform(data, model): model.eval() latent_zs = [] n_batch = (len(data) + BATCH_SIZE - 1) // BATCH_SIZE for i in range(n_batch): data_batch = data[i * BATCH_SIZE:(i + 1) * BATCH_SIZE] with autograd.no_grad(): latent_z = model(data_batch, encoder='v1') latent_zs.append(latent_z.cpu().data.numpy()) latent_zs = np.concatenate(latent_zs) return latent_zs
def _process_func(engine, batch): model.eval() data, labels, cam_ids, img_paths = batch[:4] data = data.to(device, non_blocking=non_blocking) with no_grad(): feat = model(data, cam_ids=cam_ids.to(device, non_blocking=non_blocking)) return feat.data.float().cpu(), labels, cam_ids, np.array(img_paths)
def forward(self, input: torch.Tensor) -> torch.Tensor: """ forward propagation of the submodule will save the first activation and make sure no autograd activation savings will happen :param input: the input of the submodule :return: the output as it should be calculated normally """ self.first_activations.append(input.clone().detach_()) with autograd.no_grad(): return self.module(input.to(self.device))
def _process_func(engine, batch): model.eval() data, label, cam = batch[:3] data = data.to(device, non_blocking=non_blocking) with no_grad(): feat = model(data, cam=cam.clone().to(device, non_blocking=non_blocking)) return feat.data.float().cpu(), label, cam
def _process_func(engine, batch): model.eval() inputs, label, cam_id = batch if device is not None: inputs = inputs.to(device, non_blocking=non_blocking) with no_grad(): feat = model(inputs) return feat.data.cpu(), label.cpu(), cam_id.cpu()
def get_iso_points(self, points, sdf_net, ear=False, outlier_tolerance=0.01): if not ear: projection = self.projection else: # first resample uniformly projection = self.ear_projection with autograd.no_grad(): proj_results = projection.project_points(points.view(1, -1, 3), sdf_net) mask_iso = proj_results['mask'].view(1, -1) iso_points = proj_results['levelset_points'].view(1, -1, 3) iso_points = iso_points[mask_iso].view(1, -1, 3) # iso_points = remove_outliers(iso_points, tolerance=outlier_tolerance, neighborhood_size=31).points_padded() return iso_points
def generate_mesh(self, data, **kwargs): ''' Generates the output mesh. Args: data (tensor): data tensor ''' with autograd.no_grad(): self.model.eval() mesh = get_surface_high_res_mesh( lambda x: self.model.decode(x).sdf.squeeze(), resolution=self.resolution, box_side_length=self.object_bounding_sphere * 2) return mesh
def eval_step(self, data, **kwargs): """ evaluate with image mask iou or image rgb psnr """ lights_model = kwargs.get('lights', self.val_loader.dataset.get_lights()) cameras_model = kwargs.get('cameras', self.val_loader.dataset.get_cameras()) img_size = self.generator.img_size eval_dict = {'iou': 0.0, 'psnr': 0.0} with autograd.no_grad(): self.model.eval() data = self.process_data_dict(data, cameras_model, lights=lights_model) img_mask = data['mask_img'] img = data['img'] # render image rgbas = self.generator.raytrace_images(img_size, img_mask, cameras=data['camera'], lights=data['light']) assert (len(rgbas) == 1) rgba = rgbas[0] rgba = torch.tensor(rgba[None, ...], dtype=torch.float, device=img_mask.device).permute(0, 3, 1, 2) # compare iou mask_gt = F.interpolate(img_mask.float(), img_size, mode='bilinear', align_corners=False).squeeze(1) mask_pred = rgba[:, 3, :, :] eval_dict['iou'] += self.iou_loss(mask_gt.float(), mask_pred.float(), reduction='mean') # compare psnr rgb_gt = F.interpolate(img, img_size, mode='bilinear', align_corners=False) rgb_pred = rgba[:, :3, :, :] eval_dict['psnr'] += self.l2_loss(rgb_gt, rgb_pred, channel_dim=1, reduction='mean', align_corners=False).detach() return eval_dict
def __extract_feature(train=True, force_new=False): path_x = os.path.join(ROOT, 'tmp', 'fea_x.npy') path_y = os.path.join(ROOT, 'tmp', 'fea_y.npy') path_type = os.path.join(ROOT, 'tmp', 'trainval_split_.npy') if not force_new and os.path.exists(path_x) and os.path.exists( path_y) and os.path.exists(path_type): x = np.load(path_x) y = np.load(path_y) trainval_split = np.load(path_type) flag = trainval_split if train else ~trainval_split x = torch.FloatTensor(x[flag]) y = torch.LongTensor(y[flag]) else: x, y, mask, trainval_split = __read() trainval = torch.LongTensor(trainval_split.astype(np.int32)) mask = torch.LongTensor(mask) x = torch.LongTensor(x) y = torch.LongTensor(y) temp = dataset.TensorDataset(x, y, mask, trainval) # if os.path.exists(os.path.join(PROOT, 'chinese_L-12_H-768_A-12', 'pytorch_model.bin')): # bert_model = BertModel.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12')).cuda() # else: # bert_model = BertModel.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12'), from_tf=True) # torch.save(bert_model.state_dict(), os.path.join(PROOT, 'chinese_L-12_H-768_A-12', 'pytorch_model.bin')) # bert_model = BertModel.from_pretrained(os.path.join(PROOT, 'chinese_L-12_H-768_A-12')).cuda() bert_model = BertModel.from_pretrained('bert-base-chinese').cuda() bert_model = torch.nn.DataParallel(bert_model) bert_model.eval() Xs = [] ys = [] types = [] for x, y, m, t in tqdm( DataLoader(temp, batch_size=200, shuffle=False, num_workers=2)): with autograd.no_grad(): _, pooled = bert_model(x.cuda(), attention_mask=m.cuda(), output_all_encoded_layers=False) Xs.append(pooled.cpu()) ys.append(y) types.append(t) x = torch.cat(Xs, dim=0).numpy() y = torch.cat(ys, dim=0).numpy() types = torch.cat(types, dim=0).numpy().astype(np.bool) np.save(path_x, x) np.save(path_y, y) np.save(path_type, types) flag = types if train else ~types x = torch.FloatTensor(x[flag]) y = torch.LongTensor(y[flag]) return x, y
def generate_iso_contour(self, **kwargs): self.model.eval() with autograd.no_grad(): def decode_pts_func(p): return self.eval_points(p, **kwargs) box_size = (self.object_bounding_sphere * 2 + self.padding, ) * 3 imgs = plot_cuts(decode_pts_func, box_size=box_size, max_n_eval_pts=self.points_batch_size, thres=0.0, imgs_per_cut=kwargs.get('imgs_per_cut', 1)) return imgs
def val(self, test_control=True): print('Running Testing') if self.cmd_args.test_prediction: start = time.time() losses_meter = RecursiveAverageMeter() infer_time = AverageMeter() # self.model.eval() with autograd.no_grad(): for batch_idx, sample_batched in enumerate(self.val_loader): inputs = AttrDict( map_dict(self.try_move_to_dev, sample_batched)) with self.model.val_mode(pred_length=False): infer_start = time.time() output = self.model(inputs, 'test') infer_time.update(time.time() - infer_start) if self.evaluator is not None: # force eval on all batches for reduced noise self.evaluator.eval(inputs, output, self.model) # run train model to get NLL on validation data output_train_mdl = self.model(inputs) losses = self.model.loss(inputs, output_train_mdl) losses.total = self.model.get_total_loss(inputs, losses) losses_meter.update(losses) del losses del output_train_mdl # if batch_idx == 0: # break if not self.cmd_args.dont_save: if self.evaluator is not None: self.evaluator.dump_results(self.global_step) if self.cmd_args.metric: print("Finished Evaluation! Exiting...") exit(0) self.model.log_outputs(output, inputs, losses_meter.avg, self.global_step, log_images=self.cmd_args.log_images, phase='val') print(( '\nTest set: Average loss: {:.4f} in {:.2f}s\n'.format( losses_meter.avg.total.value.item(), time.time() - start))) if self.cmd_args.verbose_timing: print("avg Inference time: {:.3f}s/batch".format( infer_time.avg)) del output
def test_eval(epoch, model, data, prefix, ranklist, args, result_log_path, cold_start, writer=None): print(prefix + ' start...') model.eval() with no_grad(): precision_10, recall_10, ndcg_score_10, precision_5, recall_5, ndcg_score_5, precision_1, recall_1, ndcg_score_1 = model.full_accuracy( data, ranklist, cold_start) print( '---------------------------------{0}-th Precition:{1:.4f} Recall:{2:.4f} NDCG:{3:.4f}---------------------------------' .format(epoch, precision_10, recall_10, ndcg_score_10)) with open(result_log_path, "a") as f: f.write( '---------------------------------' + prefix + ': {0}-th epoch {1}-th top Precition:{2:.4f} Recall:{3:.4f} NDCG:{4:.4f}---------------------------------' .format(epoch, 10, precision_10, recall_10, ndcg_score_10)) # 将字符串写入文件中 f.write("\n") with open(result_log_path, "a") as f: f.write( '---------------------------------' + prefix + ': {0}-th epoch {1}-th top Precition:{2:.4f} Recall:{3:.4f} NDCG:{4:.4f}---------------------------------' .format(epoch, 5, precision_5, recall_5, ndcg_score_5)) # 将字符串写入文件中 f.write("\n") with open(result_log_path, "a") as f: f.write( '---------------------------------' + prefix + ': {0}-th epoch {1}-th top Precition:{2:.4f} Recall:{3:.4f} NDCG:{4:.4f}---------------------------------' .format(epoch, 1, precision_1, recall_1, ndcg_score_1)) # 将字符串写入文件中 f.write("\n") # writer.add_scalar(prefix+'_Precition', precision, epoch) # writer.add_scalar(prefix+'_Recall', recall, epoch) # writer.add_scalar(prefix+'_NDCG', ndcg_score, epoch) # writer.add_histogram(prefix+'_visual_distribution', model.v_rep, epoch) # writer.add_histogram(prefix+'_acoustic_distribution', model.a_rep, epoch) # writer.add_histogram(prefix+'_textual_distribution', model.t_rep, epoch) return precision_10, recall_10, ndcg_score_10
def val(self, verbose=False): with autograd.no_grad(): losses = [] x_l, y_l, z_l = [], [], [] totals = [] total_real = [] for batch_idx, batch in enumerate(self.val_dataloader): inputs = deep_map(lambda x: x.to(self.device), batch) output = self.model(inputs) loss = self.model.loss(output, inputs['label']) if verbose: true_state_batch = denormalize(batch['state'].cpu().numpy(), self.conf.dataset.norms.state_norm.mean, self.conf.dataset.norms.state_norm.scale) true_action_batch = denormalize_action(batch['label'].cpu().numpy(), self.conf.dataset.norms.action_norm) policy_action_batch = denormalize_action(output.cpu().numpy(), self.conf.dataset.norms.action_norm) for true_action, policy_action, true_state in zip(true_action_batch, policy_action_batch, true_state_batch): print('-------------------------------------------') print(f'Expert action was {true_action}') print(f'Policy action was {policy_action}') print(f'Estimated final position is {policy_action + true_state}') print(f'True final position is {true_state + true_action}') print('-------------------------------------------') totals.extend(policy_action_batch + true_state_batch) total_real.extend(true_state_batch + true_action_batch) p = torch.mean((output - inputs['label']) ** 2, dim=0) x_l.append(p[0] * self._batch_size(batch)) y_l.append(p[1] * self._batch_size(batch)) z_l.append(p[2] * self._batch_size(batch)) losses.append(loss * self._batch_size(batch)) if verbose: print(f'Mean pred final position: {np.mean(totals, axis=0)}') print(f'Mean real final position: {np.mean(total_real, axis=0)}') print(f'Std pred final position: {np.std(totals, axis=0)}') print(f'Std real final position: {np.std(total_real, axis=0)}') self.visualize_images(inputs, 'val') loss = sum(losses) / len(self.val_dataloader.dataset) x_l = sum(x_l) / len(self.val_dataloader.dataset) y_l = sum(y_l) / len(self.val_dataloader.dataset) z_l = sum(z_l) / len(self.val_dataloader.dataset) self.summary_writer.add_scalar('val/xloss', x_l, self.global_step) self.summary_writer.add_scalar('val/yloss', y_l, self.global_step) self.summary_writer.add_scalar('val/zloss', z_l, self.global_step) self.summary_writer.add_scalar('val/loss', loss, self.global_step)
def forward(self, x_in): if self.is_nested_pass: return x_in assert self.input is not None, "no input registered - activated?" # clear memory self.observed_acts = [] with autograd.no_grad(): # Pass input again and collect readout self.is_nested_pass = True self.model[0](self.input) self.is_nested_pass = False # Done with readout. Use it to obtain map return self.forward_augmented(x_in, self.observed_acts)
def val(self): print('Running Testing') if self.args.test_prediction: start = time.time() self.model_test.load_state_dict(self.model.state_dict()) losses_meter = RecursiveAverageMeter() self.model_test.eval() self.evaluator.reset() with autograd.no_grad(): for batch_idx, sample_batched in enumerate(self.val_loader): inputs = AttrDict( map_dict(lambda x: x.to(self.device), sample_batched)) # run evaluator with val-mode model with self.model_test.val_mode(): self.evaluator.eval(inputs, self.model_test) # run non-val-mode model (inference) to check overfitting output = self.model_test(inputs) losses = self.model_test.loss(output, inputs) losses_meter.update(losses) del losses if not self.args.dont_save: if self.evaluator is not None: self.evaluator.dump_results(self.global_step) self.model_test.log_outputs(output, inputs, losses_meter.avg, self.global_step, log_images=True, phase='val', **self._logging_kwargs) print(( '\nTest set: Average loss: {:.4f} in {:.2f}s\n'.format( losses_meter.avg.total.value.item(), time.time() - start))) del output
def val(self): print('Running Testing') if self.args.test_prediction: start = time.time() self.model_val.to(torch.device('cuda')) self.model_val.load_state_dict(self.model.state_dict()) if self._hp.model_test is not None: self.model_test.load_state_dict(self.model.state_dict()) losses_meter = RecursiveAverageMeter() with autograd.no_grad(): for batch_idx, sample_batched in enumerate(self.val_loader): inputs = AttrDict(map_dict(lambda x: x.to(self.device), sample_batched)) output = self.model_val(inputs) losses = self.model_val.loss(output) losses_meter.update(losses) del losses self.model_val.log_outputs( output, inputs, losses_meter.avg, self.global_step, log_images=True, phase='val') print(('\nTest set: Average loss: {:.4f} in {:.2f}s\n' .format(losses_meter.avg.total_loss.item(), time.time() - start))) del output self.model_val.to(torch.device('cpu'))
def detection(): """ :return: """ data = {'success': False} if flask.request.method == 'POST': if flask.request.files.get('image'): input_size = (config.image_shape, config.image_shape) max_object_num = config.max_objects class_names = config.class_names confidence_thresh = config.confidence_thresh iou_thresh = config.iou_thresh _img_bytes = flask.request.files['image'].read() _img_craft = np.frombuffer(_img_bytes, np.uint8) original_img = cv2.imdecode(_img_craft, cv2.IMREAD_COLOR) img, _ = dataset.transform_image(original_img, input_size=input_size, new_axis=True, augmentation=False) if net.use_cuda: img = img.cuda() with autograd.no_grad(): prediction = net(img) _, name_tuple = utils.parse_class_names(class_names) original_size = original_img.shape[:2] prediction_result = utils.transform_prediction( prediction, confidence_thresh, iou_thresh, max_object_num) res_dct = detect.get_detection_json(prediction_result[0], original_size, name_tuple, input_size) data['result'] = res_dct data['success'] = True return flask.jsonify(data)
def sample_from_pixels(self, pixels, cameras, mask_gt, c=None, **kwargs): """ IDR implementation """ batch_size, num_pixels = mask_gt.shape[:2] with autograd.no_grad(): cam_pos = cameras.get_camera_center() cam_ray = cameras.unproject_points(torch.cat( [-pixels, pixels.new_ones(pixels.shape[:-1] + (1,))], dim=-1), scaled_depth_input=False) - \ cam_pos.unsqueeze(1) cam_ray = F.normalize(cam_ray, dim=-1, p=2) points, mask_pred, dists = self.ray_tracing( sdf=lambda x: self.decode(x, **kwargs).sdf.squeeze(-1), cam_loc=cam_pos, object_mask=mask_gt.view(-1), ray_directions=cam_ray) dists = dists.view(batch_size, num_pixels) dists[dists == 0] = getattr(cameras, 'znear', 1.0) points = points.view(batch_size, num_pixels, 3) mask_pred = mask_pred.view(batch_size, num_pixels) iso_points = points.clone() if self.training: iso_points, _ = self.directional_sampling(self.decoder, iso_points.detach(), cam_ray, cam_pos.view( batch_size, 1, 3), return_eval=True) free_points = points[~mask_gt] occ_points = points[(~mask_pred) & mask_gt] num_free = (~mask_gt).view(batch_size, -1).sum(-1) num_occ = ((~mask_pred) & mask_gt).view(batch_size, -1).sum(-1) iso_points = iso_points.view(batch_size, num_pixels, 3) mask_pred = mask_pred.view(batch_size, num_pixels) return iso_points, free_points, occ_points, mask_pred, num_free, num_occ
def sample_world_points(self, p, cameras, n_points_per_ray, mask_gt, mask_pred, c=None): """ get freepsace and occupancy points in source coordinates (-1.0, 1.0) Args: p (tensor): (N, n_rays, 2) n_points_per_ray: number of 3D points per viewing ray mask_gt: (N, n_rays) mask_pred: (N, n_rays) Returns: p_freespace (N1*n_points_per_ray, 3) mask_freespace (N, n_rays, n_points_per_ray) p_occupancy (N2, 3) mask_occupancy (N, n_rays) """ batch_size, P = p.shape[:2] max_points = self.max_points_per_pass packed_to_cloud_idx = torch.arange(batch_size).view(-1, 1).expand( batch_size, P).to(device=p.device) with autograd.no_grad(): # 0. invalid points iso_incamera = (p[..., :2].abs() <= 1.0).all(dim=-1) # 1. find intersect with unit sphere, returns (N, n_rays, 3) and (N, n_rays) mask cam_pos = cameras.get_camera_center() cam_ray = cameras.unproject_points( torch.cat([-p, p.new_ones(p.shape[:-1] + (1, ))], dim=-1), scaled_depth_input=False) - cam_pos.unsqueeze(1) cam_ray = F.normalize(cam_ray, p=2, dim=-1, eps=1e-8) section0, section1, has_intersection = intersection_with_unit_cube( cam_pos.unsqueeze(1), cam_ray, side_length=2 * self.object_bounding_sphere) # section0, section1, has_intersection = intersection_with_unit_sphere( # cam_pos.unsqueeze(1), cam_ray, radius=self.object_bounding_sphere # ) # 2. sample n_points_per_ray uniformly between the intersections section1 = section1[has_intersection] section0 = section0[has_intersection] cam_ray = cam_ray[has_intersection] lengths = torch.norm(section1 - section0, dim=-1) # assert(not (section0 == cam_ray).all(dim=-1).any(dim=0)) lengths = torch.linspace( 0, 1.0, n_points_per_ray, device=lengths.device) * lengths.unsqueeze(-1) world_points = lengths.unsqueeze(-1) * \ cam_ray.unsqueeze(-2) + section0.unsqueeze(-2) # 3. sample freespace and occupancy points # NOTE: focus on rays that intersect the unit sphere to limit the sampling space # to the unit sphere. p_split_list = torch.split(world_points.view(-1, 3), max_points, dim=0) sdf_sampled = torch.cat([ self.decoder.forward(p_split).sdf for p_split in p_split_list ], dim=0) sdf_sampled = sdf_sampled.view(-1, n_points_per_ray) p_idx = torch.argmin(sdf_sampled, dim=-1, keepdim=True) world_points = world_points[torch.arange(world_points.shape[0]), p_idx.view(-1)] # pick the point between the two intersections that has the lowest sdf value mask_freespace = (mask_gt[has_intersection] == 0) & iso_incamera[has_intersection] # if mask_pred is not None: # mask_freespace = mask_freespace & (~mask_pred) n_free_per_batch = torch.stack([ x.sum() for x in mask_freespace.split( has_intersection.sum(-1).tolist(), dim=0) ]) p_freespace = world_points[mask_freespace].view(-1, 3) mask_occupancy = mask_gt[has_intersection] & iso_incamera[ has_intersection] if mask_pred is not None: mask_occupancy = mask_occupancy & ( ~mask_pred)[has_intersection] p_occupancy = world_points[mask_occupancy] n_occ_per_batch = torch.stack([ x.sum() for x in mask_occupancy.split( has_intersection.sum(-1).tolist(), dim=0) ]) return p_freespace, n_free_per_batch, p_occupancy, n_occ_per_batch
def pixels_to_world(self, pixels, cameras, c=None, it=0, **kwargs): """ Modified DVR implementation to find intersection via sphere-tracing. Ray-trace from both front and back like in IDR Args: pixels (B, P, 2) pixels with normalized coordinates (top-left is (-1, -1)) bottom right is (-1, -1) cameras (BaseCameras) c: latent code (B,*,C) Returns: p_world (B, P, 3) ray-traced points mask_pred (B, P): mask for successful ray-tracing """ with autograd.no_grad(): # First find initial ray0 as the intersection with the unit sphere cam_pos = cameras.get_camera_center() cam_ray = cameras.unproject_points(torch.cat( [-pixels, pixels.new_ones(pixels.shape[:-1] + (1,))], dim=-1), scaled_depth_input=False) - \ cam_pos.unsqueeze(1) cam_ray = F.normalize(cam_ray, dim=-1, p=2) # This returns an intersection between the two tangent plane if the ray # doesn't intersects with the sphere section0, section1, has_intersection = intersection_with_unit_cube( cam_pos.unsqueeze(1), cam_ray, side_length=self.object_bounding_sphere * 2) # Sphere tracing from the first unit-sphere intersection proj_result = self.sphere_tracing.project_points(section0, cam_ray, self.decoder, latent=c) mask_pred = proj_result['mask'] & has_intersection p_world = proj_result['levelset_points'].detach() p_world[~mask_pred] = section0[~mask_pred] # For failed sphere-tracing, attempt to find the intersection via # secant sign change (like DVR) proj_result_back = self.sphere_tracing.project_points(section1, -cam_ray, self.decoder, latent=c) p_world_back = proj_result_back['levelset_points'].detach() mask_pred_back = proj_result_back['mask'] iso_secant, mask_pred_secant = find_zero_crossing_between_point_pairs( p_world, p_world_back, self.decoder, c=c, is_occupancy=False) mask_pred_secant = (~mask_pred) & mask_pred_secant mask_pred = mask_pred_secant | mask_pred # merge two partions of iso intersections p_world = torch.where(mask_pred_secant.unsqueeze(-1), iso_secant, p_world) with autograd.enable_grad(): p_world = p_world.requires_grad_(True) p_world_val = self.decoder(p_world, c=c).sdf p_world_Dx = autograd.grad([p_world_val], [p_world], torch.ones_like(p_world_val), retain_graph=True)[0] # filter out p_world whose Dx is almost perpendicular mask = torch.sum(F.normalize(p_world_Dx, dim=-1) * cam_ray, dim=-1) < -1e-2 mask_pred = mask_pred & mask if self.training: p_world, _ = self.directional_sampling(self.decoder, p_world, cam_ray, cam_pos.unsqueeze(1), return_eval=True) return p_world, mask_pred
def dqn_learning( env, q_func, optimizer_spec, exploration, runname, frame_filter = None, stopping_criterion=None, replay_buffer_size=1000000, batch_size=32, gamma=0.99, learning_starts=50000, learning_freq=4, frame_history_len=4, target_update_freq=10000 ): """Run Deep Q-learning algorithm. You can specify your own convnet using q_func. All schedules are w.r.t. total number of steps taken in the environment. Parameters ---------- env: gym.Env gym environment to train on. q_func: function Model to use for computing the q function. It should accept the following named arguments: input_channel: int number of channel of input. num_actions: int number of actions optimizer_spec: OptimizerSpec Specifying the constructor and kwargs, as well as learning rate schedule for the optimizer exploration: Schedule (defined in utils.schedule) schedule for probability of choosing random action. stopping_criterion: (env) -> bool should return true when it's ok for the RL algorithm to stop. takes in env and the number of steps executed so far. replay_buffer_size: int How many memories to store in the replay buffer. batch_size: int How many transitions to sample each time experience is replayed. gamma: float Discount Factor learning_starts: int After how many environment steps to start replaying experiences learning_freq: int How many steps of environment to take between every experience replay frame_history_len: int How many past frames to include as input to the model. target_update_freq: int How many experience replay rounds (not steps!) to perform between each update to the target Q network """ assert type(env.observation_space) == gym.spaces.Box assert type(env.action_space) == gym.spaces.Discrete Statistic = { "t": [], "mean_episode_rewards": [], "best_mean_episode_rewards": [] } ############### # BUILD MODEL # ############### if len(env.observation_space.shape) == 1: # This means we are running on low-dimensional observations (e.g. RAM) input_arg = env.observation_space.shape[0] else: img_h, img_w, img_c = env.observation_space.shape input_arg = frame_history_len * img_c num_actions = env.action_space.n # Construct an epilson greedy policy with given exploration schedule def select_epilson_greedy_action(model, obs, t): sample = random.random() eps_threshold = exploration.value(t) if sample > eps_threshold: obs = torch.from_numpy(obs).type(dtype).unsqueeze(0) / 255.0 # Use volatile = True if variable is only used in inference mode, i.e. don’t save the history with torch.no_grad(): return model(Variable(obs)).data.max(1)[1].cpu() else: return torch.IntTensor([[random.randrange(num_actions)]]) # Initialize target q function and q function, i.e. build the model. # region Part 1 Q = q_func(input_arg, num_actions) Q_target = q_func(input_arg, num_actions) if USE_CUDA: Q = Q.cuda() Q_target = Q_target.cuda() # endregion # Construct Q network optimizer function optimizer = optimizer_spec.constructor(Q.parameters(), **optimizer_spec.kwargs) # Construct the replay buffer replay_buffer = ReplayBuffer(replay_buffer_size, frame_history_len) ############### # RUN ENV # ############### num_param_updates = 0 mean_episode_reward = -float('nan') best_mean_episode_reward = -float('inf') last_obs = env.reset() LOG_EVERY_N_STEPS = 10000 for t in count(): ### 1. Check stopping criterion if stopping_criterion is not None and stopping_criterion(t): break ### 2. Step the env and store the transition # At this point, "last_obs" contains the latest observation that was # recorded from the simulator. Here, your code needs to store this # observation and its outcome (reward, next observation, etc.) into # the replay buffer while stepping the simulator forward one step. # At the end of this block of code, the simulator should have been # advanced one step, and the replay buffer should contain one more # transition. # Specifically, last_obs must point to the new latest observation. # Useful functions you'll need to call: # obs, reward, done, info = env.step(action) # this steps the environment forward one step # obs = env.reset() # this resets the environment if you reached an episode boundary. # Don't forget to call env.reset() to get a new observation if done # is true!! # Note that you cannot use "last_obs" directly as input # into your network, since it needs to be processed to include context # from previous frames. You should check out the replay buffer # implementation in dqn_utils.py to see what functionality the replay # buffer exposes. The replay buffer has a function called # encode_recent_observation that will take the latest observation # that you pushed into the buffer and compute the corresponding # input that should be given to a Q network by appending some # previous frames. # Don't forget to include epsilon greedy exploration! # And remember that the first time you enter this loop, the model # may not yet have been initialized (but of course, the first step # might as well be random, since you haven't trained your net...) # region Part 2 if frame_filter is not None: frame_filter(last_obs) replay_buffer_idx = replay_buffer.store_frame(last_obs) network_obs = replay_buffer.encode_recent_observation() action = select_epilson_greedy_action(Q, network_obs, t) last_obs, reward, done, info = env.step(action) replay_buffer.store_effect(replay_buffer_idx, action, reward, done) if done: last_obs = env.reset() # endregion # at this point, the environment should have been advanced one step (and # reset if done was true), and last_obs should point to the new latest # observation ### 3. Perform experience replay and train the network. # Note that this is only done if the replay buffer contains enough samples # for us to learn something useful -- until then, the model will not be # initialized and random actions should be taken if t > learning_starts and t % learning_freq == 0 and replay_buffer.can_sample(batch_size): # Here, you should perform training. Training consists of four steps: # 3.a: use the replay buffer to sample a batch of transitions (see the # replay buffer code for function definition, each batch that you sample # should consist of current observations, current actions, rewards, # next observations, and done indicator). # Note: Move the variables to the GPU if avialable # 3.b: fill in your own code to compute the Bellman error. This requires # evaluating the current and next Q-values and constructing the corresponding error. # Note: don't forget to clip the error between [-1,1], multiply is by -1 (since pytorch minimizes) and # maskout post terminal status Q-values (see ReplayBuffer code). # 3.c: train the model. To do this, use the bellman error you calculated perviously. # Pytorch will differentiate this error for you, to backward the error use the following API: # current.backward(d_error.data.unsqueeze(1)) # Where "current" is the variable holding current Q Values and d_error is the clipped bellman error. # Your code should produce one scalar-valued tensor. # Note: don't forget to call optimizer.zero_grad() before the backward call and # optimizer.step() after the backward call. # 3.d: periodically update the target network by loading the current Q network weights into the # target_Q network. see state_dict() and load_state_dict() methods. # you should update every target_update_freq steps, and you may find the # variable num_param_updates useful for this (it was initialized to 0) # region Part 3 obs_batch, act_batch, rew_batch, next_obs_batch, done_mask = replay_buffer.sample(batch_size) # convert to the right tensors obs_batch = convert_to_dqn_input(obs_batch) act_batch = convert_to_tensor(act_batch) rew_batch = convert_to_tensor(rew_batch) next_obs_batch = convert_to_dqn_input(next_obs_batch) done_mask = convert_to_tensor(done_mask.astype(bool)) # calculate err curr_q = Q(obs_batch).gather(1, act_batch.long().unsqueeze(1)) next_q = rew_batch with autograd.no_grad(): max_next_q = Q_target(next_obs_batch).max(dim=1)[0][done_mask == False] next_q[done_mask == False] += (gamma * max_next_q) err = (next_q.unsqueeze(1) - curr_q).clamp(-1, 1) * -1.0 # back prop error optimizer.zero_grad() curr_q.backward(err.data) optimizer.step() # update target logic num_param_updates += 1 if num_param_updates % target_update_freq == 0: Q_target.load_state_dict(Q.state_dict()) # endregion ### 4. Log progress and keep track of statistics episode_rewards = get_wrapper_by_name(env, "Monitor").get_episode_rewards() if len(episode_rewards) > 0: mean_episode_reward = np.mean(episode_rewards[-100:]) if len(episode_rewards) > 100: best_mean_episode_reward = max(best_mean_episode_reward, mean_episode_reward) Statistic["t"].append(t) Statistic["mean_episode_rewards"].append(mean_episode_reward) Statistic["best_mean_episode_rewards"].append(best_mean_episode_reward) if t % LOG_EVERY_N_STEPS == 0 and t > learning_starts: print("Timestep %d" % (t,)) print("mean reward (100 episodes) %f" % mean_episode_reward) print("best mean reward %f" % best_mean_episode_reward) print("episodes %d" % len(episode_rewards)) print("exploration %f" % exploration.value(t)) sys.stdout.flush() write_statistics(runname, Statistic, [optimizer_spec, exploration, stopping_criterion, replay_buffer_size, batch_size, gamma, learning_starts, learning_freq, frame_history_len, target_update_freq])