def validation(model, dataloaders, criterion, phase, use_gpu, device): model.eval() pred = np.array([]) true = np.array([]) losses = np.array([]) with torch.no_grad(): for data in tqdm(dataloaders[phase]): inputs, labels = data if use_gpu: inputs = Variable(inputs).to(device) labels = Variable(labels).to(device) else: inputs, labels = Variable(inputs), Variable(labels) outputs = model(inputs) loss = criterion( outputs.type(torch.cuda.FloatTensor)[:, 1], labels.type(torch.cuda.FloatTensor)) #loss = nn.BCEWithLogitsLoss(reduction='sum', # weight=torch.ones(outputs.shape[0]).to(device) / 5)(outputs.type(torch.cuda.FloatTensor)[:, 1], # labels.type(torch.cuda.FloatTensor)) true = np.append(true, to_numpy(labels, use_gpu)) pred = np.append(pred, sigmoid(to_numpy(outputs[:, 1], use_gpu))) losses = np.append(losses, to_numpy(loss, use_gpu)) del inputs, labels, loss model.train() return true, pred, np.sum(losses)
def main(): utils.fix_randomness() opt = options.set_end2end_optim_options() assert opt.iou_space == 'part_and_whole' test_dataset = aligned_dataset.AlignedDatasetFactory.get_aligned_dataset( opt, 'test') test_loader = DataLoader( test_dataset, batch_size=opt.batch_size, shuffle=False, num_workers=0, ) e2e = end_2_end_optimization.End2EndOptimFactory.get_end_2_end_optimization_model( opt) iou = metrics.IOU(opt) orig_iou_list = [] optim_iou_list = [] original_homography_list = [] optim_homography_list = [] gt_homography_list = [] t0 = time.time() for i, data_batch in enumerate(test_loader): frame, _, gt_homography = data_batch orig_homography, optim_homography = e2e.optim(frame, test_dataset.template) orig_iou = iou(orig_homography, gt_homography) optim_iou = iou(optim_homography, gt_homography) orig_iou_list.append(orig_iou) optim_iou_list.append(optim_iou) original_homography_list.append(utils.to_numpy(orig_homography.data)) optim_homography_list.append(utils.to_numpy(optim_homography.data)) gt_homography_list.append(utils.to_numpy(gt_homography.data)) t1 = time.time() orig_iou_list = np.array(orig_iou_list) orig_iou_part_list = np.concatenate(orig_iou_list[:, 0]) orig_iou_whole_list = np.concatenate(orig_iou_list[:, 1]) print('----- Summary -----') print('original IOU part mean:', orig_iou_part_list.mean()) print('original IOU part median:', np.median(orig_iou_part_list)) print('original IOU whole mean:', orig_iou_whole_list.mean()) print('original IOU whole median:', np.median(orig_iou_whole_list)) optim_iou_list = np.array(optim_iou_list) optim_iou_part_list = np.concatenate(optim_iou_list[:, 0]) optim_iou_whole_list = np.concatenate(optim_iou_list[:, 1]) print('optimized IOU part mean:', optim_iou_part_list.mean()) print('optimized IOU part median:', np.median(optim_iou_part_list)) print('optimized IOU whole mean:', optim_iou_whole_list.mean()) print('optimized IOU whole median:', np.median(optim_iou_whole_list)) print('----- -----') print('spent {0} seconds for {1} images'.format( (t1 - t0), (optim_iou_whole_list.shape[0]))) print('{0} seconds per single image'.format( (t1 - t0) / (optim_iou_whole_list.shape[0]))) print('----- End -----')
def render_results(y_2d, y_2d_): y_2d = to_numpy(y_2d) y_2d_ = to_numpy(y_2d_) point_pairs = np.column_stack((y_2d, y_2d_)) im_to_show = point_pairs[1::4] res = np.concatenate(im_to_show, axis=1) res = np.expand_dims(res, axis=0) return res
def _eval_generator_and_save_images(self, epoch_idx): results = self._eval_generator() save_images( results, to_numpy(self.fixed_inp, False), to_numpy(self.fixed_target, False), path=os.path.join(self.test_images_path, "testimage_{:02d}.png".format(epoch_idx + 1)), )
def get_model_part_iou_rasterization(self, output, target): if output.shape == (3, 3): output = output[None] if target.shape == (3, 3): target = target[None] assert output.shape == target.shape, 'output shape does not match target shape' batch_size = output.shape[0] fake_frame = torch.ones( [batch_size, 1, self.frame_height, self.frame_width], device=output.device) fake_template = torch.ones( [batch_size, 1, self.template_height, self.template_width], device=output.device) output_mask = warp.warp_image(fake_frame, output.inverse(), out_shape=fake_template.shape[-2:]) target_mask = warp.warp_image(fake_frame, target.inverse(), out_shape=fake_template.shape[-2:]) output_mask[output_mask > 0] = 1 target_mask[target_mask > 0] = 1 intersection_mask = output_mask * target_mask output = output_mask.sum(dim=[1, 2, 3]) target = target_mask.sum(dim=[1, 2, 3]) intersection = intersection_mask.sum(dim=[1, 2, 3]) union = output + target - intersection iou = intersection / (union + self.epsilon) return utils.to_numpy(iou)
def main(args): test_x, test_y = load_image(args.image_path) test_inp = to_tensor(test_x.astype(np.float32)) test_target = to_tensor(test_y.astype(np.float32)) generator = Generator().to("cuda") start_t = time.time() pretrain_model = flow.load(args.model_path) generator.load_state_dict(pretrain_model) end_t = time.time() print("load params time : {}".format(end_t - start_t)) start_t = time.time() generator.eval() with flow.no_grad(): gout = to_numpy(generator(test_inp), False) end_t = time.time() print("infer time : {}".format(end_t - start_t)) # save images save_images( gout, test_inp.numpy(), test_target.numpy(), path=os.path.join("./testimage.png"), plot_size=1, )
def train_discriminator(self, input, target, label0, label1): g_out = self.netG(input) # Fake; stop backprop to the generator by detaching fake_B fake_AB = flow.cat([input, g_out.detach()], 1) pred_fake = self.netD(fake_AB) d_fake_loss = self.criterionGAN(pred_fake, label0) # Real real_AB = flow.cat([input, target], 1) pred_real = self.netD(real_AB) d_real_loss = self.criterionGAN(pred_real, label1) # combine loss and calculate gradients d_loss = (d_fake_loss + d_real_loss) * 0.5 d_loss.backward() self.optimizerD.step() self.optimizerD.zero_grad() return to_numpy(d_fake_loss), to_numpy(d_real_loss), to_numpy(d_loss)
def get_frame_space_scaling_homography(): src_4pts = utils.to_torch(utils.FULL_CANON4PTS_NP()) dest_4pts = utils.to_torch( np.array([[0, 0], [0, self.frame_h], [self.frame_w, self.frame_h], [self.frame_w, 0]], dtype=np.float32)) scaling_transformation = utils.get_perspective_transform( src_4pts[None], dest_4pts[None]) scaling_transformation = utils.to_numpy(scaling_transformation[0]) return scaling_transformation
def train_generator(self, input, target, label1): g_out = self.netG(input) # First, G(A) should fake the discriminator fake_AB = flow.cat([input, g_out], 1) pred_fake = self.netD(fake_AB) gan_loss = self.criterionGAN(pred_fake, label1) # Second, G(A) = B l1_loss = self.criterionL1(g_out, target) # combine loss and calculate gradients g_loss = gan_loss + self.LAMBDA * l1_loss g_loss.backward() self.optimizerG.step() self.optimizerG.zero_grad() return ( to_numpy(gan_loss), to_numpy(self.LAMBDA * l1_loss), to_numpy(g_loss), to_numpy(g_out, False), )
def squash_image(image): '''squash image to [0, 1] ''' min_val = image.min() max_val = image.max() if isinstance(image, np.ndarray): image = np.interp(image, (min_val, max_val), (0, 1)) return image elif isinstance(image, torch.Tensor): image_np = utils.to_numpy(image) image_np = np.interp(image_np, (min_val, max_val), (0, 1)) image_torch = utils.to_torch(image_np) return image_torch else: raise ValueError('unsupported data type: {0}'.format(type(image)))
def evaluate(self): for subset in ['val']: for i, _ in enumerate(tqdm(self.loaders["PIC"]["val"], 0)): for net_id in self.models.keys(): data_type = net_id[-3:] sample = self.get_samples(data_type, subset, i) mse_maps, uv_maps_ = self.get_mse_loss( self.models[net_id], sample) l1, mse = self.get_uv_errors(uv_maps_, sample) self.evaluation_results[subset][net_id]['mse_maps'].append( to_numpy(mse_maps)) self.evaluation_results[subset][net_id]['l1'].append( to_numpy(l1)) self.evaluation_results[subset][net_id]['mse'].append( to_numpy(mse)) saves_path = '/'.join( [self.save_to, f'evaluation_{subset}.pickle']) with open(saves_path, 'wb') as handle: pickle.dump(self.evaluation_results[subset], handle, protocol=pickle.HIGHEST_PROTOCOL)
def evaluate(self): for subset in self.subsets: for i, _ in enumerate(tqdm(self.loaders["PIC"]["val"], 0)): for net_id in self.models.keys(): is_vid = net_id.endswith('VID') data_type = net_id[-3:] sample = self.get_samples(data_type, subset) mse, l_1, renorm_loss = self.get_losses( self.models[net_id], sample, is_vid) self.evaluation_results[subset][net_id]['mse'].append( to_numpy(mse)) self.evaluation_results[subset][net_id]['l_1'].append( to_numpy(l_1)) self.evaluation_results[subset][net_id][ 'renorm_loss'].append(to_numpy(renorm_loss)) saves_path = '/'.join( [self.save_to, f'evaluation_{subset}.pickle']) with open(saves_path, 'wb') as handle: pickle.dump(self.evaluation_results[subset], handle, protocol=pickle.HIGHEST_PROTOCOL)
def _eval_agent(self): success_rate = 0 for _ in range(self.config.test_rollouts): observation = self.env.reset() obs = observation['observation'] goal = observation['desired_goal'] obs, goal = self._preproc_inputs(obs, goal) for _ in range(self.env_params['max_timesteps']): if self.config.render: self.env.render() with torch.no_grad(): action = self.actor(obs, goal) new_obs, _, _, info = self.env.step(to_numpy(action[0])) obs, goal = self._preproc_inputs(new_obs['observation'], new_obs['desired_goal']) success_rate += info['is_success'] return success_rate / self.config.test_rollouts
def get_model_whole_iou_rasterization(self, output, target): '''calculate the model whole iou ''' if output.shape == (3, 3): output = output[None] if target.shape == (3, 3): target = target[None] assert output.shape == target.shape, 'output shape does not match target shape' ZOOM_OUT_SCALE = 4 batch_size = output.shape[0] fake_template = torch.ones([ batch_size, 1, self.template_height * ZOOM_OUT_SCALE, self.template_width * ZOOM_OUT_SCALE ], device=output.device) scaling_mat = torch.eye(3, device=output.device).repeat(batch_size, 1, 1) scaling_mat[:, 0, 0] = scaling_mat[:, 1, 1] = ZOOM_OUT_SCALE target_mask = warp.warp_image(fake_template, scaling_mat, out_shape=fake_template.shape[-2:]) mapping_mat = torch.matmul(output, scaling_mat) mapping_mat = torch.matmul(target.inverse(), mapping_mat) output_mask = warp.warp_image(fake_template, mapping_mat, out_shape=fake_template.shape[-2:]) output_mask = (output_mask >= 0.5).float() target_mask = (target_mask >= 0.5).float() intersection_mask = output_mask * target_mask output = output_mask.sum(dim=[1, 2, 3]) target = target_mask.sum(dim=[1, 2, 3]) intersection = intersection_mask.sum(dim=[1, 2, 3]) union = output + target - intersection iou = intersection / (union + self.epsilon) return utils.to_numpy(iou)
def log_validation(self, losses_dict, model, target, prediction, iteration, target_alignments=None): self._log_scalar_dict(losses_dict, iteration, "val") # plot distribution of parameters for tag, value in model.named_parameters(): tag = tag.replace('.', '/') self.add_histogram(tag, value.data.cpu().numpy(), iteration) # plot alignment target and predicted, mel target and predicted, gate target and predicted log_image = partial(self.add_image, global_step=iteration, dataformats="HWC") idx = random.randint(0, prediction.alignments.size(0) - 1) if target_alignments is not None: log_image( "alignment/target", plot_alignment_to_numpy(to_numpy(target_alignments[idx]).T)) log_image( "alignment/predicted", plot_alignment_to_numpy(to_numpy(prediction.alignments[idx].T))) log_image("mel/target", plot_spectrogram_to_numpy(to_numpy(target.mels[idx]))) log_image("mel/predicted", plot_spectrogram_to_numpy(to_numpy(prediction.mels[idx]))) log_image( "gate", plot_gate_outputs_to_numpy( to_numpy(target.gate[idx]), to_numpy(torch.sigmoid(prediction.gate[idx]))))
def _eval_generator(self): self.netG.eval() with flow.no_grad(): g_out = self.netG(self.fixed_inp) return to_numpy(g_out, False)
def train_model(model, criterion, optimizer, scheduler, dataloaders, dataset_sizes, model_path, device, fold_name, use_gpu=True, num_epochs=25, plot_res=True, predict_test=True, save_val=True, best='loss'): since = time.time() torch.save(model.state_dict(), os.path.join(model_path, '_temp')) best_f1 = 0.0 best_loss = np.inf best_epoch = 0 train_metrics = [] val_metrics = [] print('=' * 10) print('VAL data {}'.format(fold_name)) print('=' * 10) for epoch in range(num_epochs): model.train(True) print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) running_loss = 0.0 pred = np.array([]) true = np.array([]) phase = 'train' for data in tqdm(dataloaders[phase]): optimizer.zero_grad() inputs, labels = data if use_gpu: inputs = Variable(inputs).to(device) labels = Variable(labels).to(device) else: inputs, labels = Variable(inputs), Variable(labels) outputs = model(inputs) loss = criterion( outputs.type(torch.cuda.FloatTensor)[:, 1], labels.type(torch.cuda.FloatTensor)) #loss = nn.BCEWithLogitsLoss(reduction='sum', # weight=torch.ones(outputs.shape[0]).to(device) / 5)(outputs.type(torch.cuda.FloatTensor)[:, 1], # labels.type(torch.cuda.FloatTensor)) loss.backward() optimizer.step() running_loss += loss.item() true = np.append(true, to_numpy(labels, use_gpu)) pred = np.append(pred, sigmoid(to_numpy(outputs[:, 1], use_gpu))) epoch_loss = running_loss / dataset_sizes[phase] pr_rec_auc, roc_auc, f1_05, f1_max, f1_auc, metr = calc_metrics( true, pred) val_true, val_pred, val_loss = validation(model, dataloaders, criterion, device=device, use_gpu=use_gpu, phase='val') pr_rec_auc_val, roc_auc_val, f1_05_val, f1_max_val, f1_auc_val, metr_val = calc_metrics( val_true, val_pred) if scheduler is not None: scheduler.step((val_loss) / dataset_sizes['val']) clear() clear_output() print('=' * 10) print('VAL data {}'.format(fold_name)) print('=' * 10) print('Epoch {}/{}'.format(epoch, num_epochs - 1)) print('-' * 10) print_results(phase=phase, loss=epoch_loss, roc_auc=roc_auc, pr_rec_auc=pr_rec_auc, f1_max=f1_max, f1_05=f1_05, f1_auc=f1_auc) print('\n') print_results(phase='val', loss=(val_loss) / dataset_sizes['val'], roc_auc=roc_auc_val, pr_rec_auc=pr_rec_auc_val, f1_max=f1_max_val, f1_05=f1_05_val, f1_auc=f1_auc_val) train_metrics.append(metr + [epoch_loss]) val_metrics.append(metr_val + [(val_loss) / dataset_sizes['val']]) if f1_05_val > best_f1: best_f1 = f1_05_val if best == 'metric': best_epoch = epoch torch.save(model.state_dict(), os.path.join(model_path, '_temp')) if (val_loss) / dataset_sizes['val'] < best_loss: best_loss = (val_loss) / dataset_sizes['val'] if best == 'loss': best_epoch = epoch torch.save(model.state_dict(), os.path.join(model_path, '_temp')) time_elapsed = time.time() - since print('Elapsed {:.0f}m {:.0f}s\n'.format(time_elapsed // 60, time_elapsed % 60)) np.save(os.path.join(model_path, 'val_metrics_' + fold_name), val_metrics) np.save(os.path.join(model_path, 'train_metrics_' + fold_name), train_metrics) print('Current best: {:4f}, epoch {}'.format(best_f1, best_epoch)) if plot_res: vizualize(model_path, fold_name, show=False, save=True, save_format='.png') time_elapsed = time.time() - since print('Training complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60)) print('Best val f1_05: {:4f}'.format(best_f1)) print('Best val loss: {:4f}'.format(best_loss)) if best is not None: model.load_state_dict(torch.load(os.path.join(model_path, '_temp'))) vizualize(model_path, fold_name, show=False, save=True, save_format='.pdf') if predict_test: results = pd.DataFrame() test_true, test_pred, _ = validation(model, dataloaders, criterion, device=device, use_gpu=use_gpu, phase='test') results['pred'] = test_pred results['pseudo_true'] = test_true results['file'] = [ sample[0] for sample in dataloaders['test'].dataset.samples ] results.to_csv(os.path.join(model_path, 'test_results_' + fold_name + '.csv'), index=False) if save_val: results = pd.DataFrame() val_true, val_pred, _ = validation(model, dataloaders, criterion, device=device, use_gpu=use_gpu, phase='val') results['true'] = val_true results['pred'] = val_pred results['file'] = [ sample[0] for sample in dataloaders['val'].dataset.samples ] results.to_csv(os.path.join(model_path, 'val_results_' + fold_name + '.csv'), index=False) del inputs, labels, loss return model, best_f1
import matplotlib.pyplot as plt from configs.config_m0 import CONFIG_M0 from datasets.frei_dataset_u0 import FreiHandDataset from transforms.to_tensor_u0 import ToTensor data_path = CONFIG_U0['dataset_path'] build_mode = CONFIG_U0['build_mode'] frames_generator = FramesGenerator(data_path, build_mode) oredered_sequence = frames_generator.order_frames('xyz', 2345, 340) build_id = CONFIG_M0['build_id'] dataset = FreiHandDataset(CONFIG_M0['dataset_path'], CONFIG_M0['data_version'], transform=ToTensor()) xyz_array = frames_generator.get_xyz_array() K_array = frames_generator.get_K_array() uv_array = frames_generator.get_uv_array(xyz_array, K_array) for idx in oredered_sequence: # uv_21x2 = np.array(uv_list).reshape(32560, 21, 2) img_tensor = dataset[idx]['img'] img_np = to_numpy(img_tensor).transpose((1, 2, 0)) im_rgb = cv2.cvtColor(img_np, cv2.COLOR_BGR2RGB) cv2.imshow('df', im_rgb) cv2.waitKey(-1)
dataset = FreiHandDataset(CONFIG_U0['dataset_path'], CONFIG_U0['data_version'], transform=ToTensor()) dataloader = DataLoader(dataset) model = ModelU0() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") checkpoint = torch.load(f'results/{build_id}.pt', map_location=torch.device(device)) model.load_state_dict(checkpoint['model_state_dict']) model.to(device) model.eval() sample = next(iter(dataloader)) img = sample['img'] y_2d = to_numpy(sample['y_2d']) uv = to_numpy(sample['uv']) y_2d_ = model(img) y_2d_ = to_numpy(y_2d_) img = to_numpy(img) img = img[0].transpose((1, 2, 0)) print(img.shape) fig, ax = plt.subplots() ax.imshow(img) plot_hand(ax, uv[0], order='uv') plt.show()
map_location=torch.device(device)) model.load_state_dict(checkpoint['model_state_dict']) g_step = checkpoint['g_step'] + 1 running_loss = checkpoint['running_loss'] print( f'Model "{build_id}" loaded. g_step: {g_step}; running_loss: {running_loss}' ) model.to(device) model.eval() # x = hand_joints[0].reshape([1, -1]) xyz_ = model(x) xyz_ = to_numpy(xyz_).reshape([21, 3]) # # print(xyz_) # # y = sample['xyz'] K = sample['K'] img = sample['img'] # # print("yb", y) # # y= y.reshape([CONFIG_M0['batch_size'], -1]) # # print("ya", y.reshape([21, 3])) # # y = to_numpy(y) # # print(y.shape) # MATRIX WAY # x = [[1.0692e-03, -2.9279e-04, 1.8849e-04, 3.7177e-01],