def save_circles(model, results_dir, img_paths, num_renders=50): circles_dir = os.path.join(results_dir, 'circles') util.cond_mkdir(circles_dir) print('Generating circle views around scene') for j, img_path in enumerate(img_paths): circle_dir = os.path.join(circles_dir, f'{j:03d}') util.cond_mkdir(circle_dir) # Copy reference image into directory shutil.copy(img_path, os.path.join(circle_dir, '0000_ref_image.png')) img = torch.from_numpy( data_util.load_rgb(img_path).transpose(2, 0, 1)).to(device).unsqueeze(0) split = img_path.split('/') split[-1] = split[-1].split('.')[0] + '.txt' split[-2] = 'pose' ref_pose = data_util.load_pose('/'.join(split)) sample_poses = data_util.gen_pose_circle(ref_pose, n_poses=num_renders, centre=[0., 0., 0.]) actions = np.zeros([num_renders, 12]) for i, target_pose in enumerate(sample_poses): actions[i] = (np.linalg.inv(target_pose) @ ref_pose).flatten()[:12] actions = torch.from_numpy(actions).float().to(device) with torch.no_grad(): state = model.encoder(img) state = state.repeat(num_renders, 1, 1) state = model.transition_model(state, actions) out = model.decoder(state.reshape(-1, model.embedding_dim)) masks, _, recs = model.compose_image(out) print(masks.shape) plt.imshow(masks[0, 0].cpu().detach().numpy()) plt.show() plt.imshow(masks[0, 1].cpu().detach().numpy()) plt.show() plt.imshow(masks[0, 2].cpu().detach().numpy()) plt.show() for i, rec in enumerate(recs): torchvision.utils.save_image(rec, os.path.join( circle_dir, f'{i+1:04d}.png'), normalize=True, range=(-1, 1)) print('Saved one circle view.')
def render(self, output_dir, blender_cam2world_matrices, write_cam_params=False): if write_cam_params: img_dir = os.path.join(output_dir, 'rgb') pose_dir = os.path.join(output_dir, 'pose') util.cond_mkdir(img_dir) util.cond_mkdir(pose_dir) else: img_dir = output_dir util.cond_mkdir(img_dir) if write_cam_params: K = util.get_calibration_matrix_K_from_blender(self.camera.data) with open(os.path.join(output_dir, 'intrinsics.txt'), 'w') as intrinsics_file: intrinsics_file.write('%f %f %f 0.\n' % (K[0][0], K[0][2], K[1][2])) intrinsics_file.write('0. 0. 0.\n') intrinsics_file.write('1.\n') intrinsics_file.write('%d %d\n' % (self.resolution, self.resolution)) for i in range(len(blender_cam2world_matrices)): self.camera.matrix_world = blender_cam2world_matrices[i] # Render the object if os.path.exists(os.path.join(img_dir, '%06d.png' % i)): continue # Render the color image self.blender_renderer.filepath = os.path.join( img_dir, '%06d.png' % i) bpy.ops.render.render(write_still=True) if write_cam_params: # Write out camera pose RT = util.get_world2cam_from_blender_cam(self.camera) cam2world = RT.inverted() with open(os.path.join(pose_dir, '%06d.txt' % i), 'w') as pose_file: matrix_flat = [] for j in range(4): for k in range(4): matrix_flat.append(cam2world[j][k]) pose_file.write(' '.join(map(str, matrix_flat)) + '\n') # Remember which meshes were just imported meshes_to_remove = [] for ob in bpy.context.selected_objects: meshes_to_remove.append(ob.data) bpy.ops.object.delete() # Remove the meshes from memory too for mesh in meshes_to_remove: bpy.data.meshes.remove(mesh)
def test(): if opt.specific_observation_idcs is not None: specific_observation_idcs = list( map(int, opt.specific_observation_idcs.split(','))) else: specific_observation_idcs = None dataset = dataio.SceneClassDataset( root_dir=opt.data_root, max_num_instances=opt.max_num_instances, specific_observation_idcs=specific_observation_idcs, max_observations_per_instance=-1, samples_per_instance=1, img_sidelength=opt.img_sidelength) dataset = DataLoader(dataset, collate_fn=dataset.collate_fn, batch_size=1, shuffle=False, drop_last=False) model = SRNsModel(num_instances=opt.num_instances, latent_dim=opt.embedding_size, has_params=opt.has_params, fit_single_srn=opt.fit_single_srn, use_unet_renderer=opt.use_unet_renderer, tracing_steps=opt.tracing_steps) assert (opt.checkpoint_path is not None), "Have to pass checkpoint!" print("Loading model from %s" % opt.checkpoint_path) util.custom_load(model, path=opt.checkpoint_path, discriminator=None, overwrite_embeddings=False) model.eval() model.cuda() # directory structure: month_day/ renderings_dir = os.path.join(opt.logging_root, 'renderings') gt_comparison_dir = os.path.join(opt.logging_root, 'gt_comparisons') util.cond_mkdir(opt.logging_root) util.cond_mkdir(gt_comparison_dir) util.cond_mkdir(renderings_dir) # Save command-line parameters to log directory. with open(os.path.join(opt.logging_root, "params.txt"), "w") as out_file: out_file.write('\n'.join( ["%s: %s" % (key, value) for key, value in vars(opt).items()])) print('Beginning evaluation...') with torch.no_grad(): instance_idx = 0 idx = 0 psnrs, ssims = list(), list() for model_input, ground_truth in dataset: model_outputs = model(model_input) psnr, ssim = model.get_psnr(model_outputs, ground_truth) psnrs.extend(psnr) ssims.extend(ssim) instance_idcs = model_input['instance_idx'] print("Object instance %d. Running mean PSNR %0.6f SSIM %0.6f" % (instance_idcs[-1], np.mean(psnrs), np.mean(ssims))) if instance_idx < opt.save_out_first_n: output_imgs = model.get_output_img(model_outputs).cpu().numpy() comparisons = model.get_comparisons(model_input, model_outputs, ground_truth) for i in range(len(output_imgs)): prev_instance_idx = instance_idx instance_idx = instance_idcs[i] if prev_instance_idx != instance_idx: idx = 0 img_only_path = os.path.join(renderings_dir, "%06d" % instance_idx) comp_path = os.path.join(gt_comparison_dir, "%06d" % instance_idx) util.cond_mkdir(img_only_path) util.cond_mkdir(comp_path) pred = util.convert_image(output_imgs[i].squeeze()) comp = util.convert_image(comparisons[i].squeeze()) util.write_img( pred, os.path.join(img_only_path, "%06d.png" % idx)) util.write_img(comp, os.path.join(comp_path, "%06d.png" % idx)) idx += 1 with open(os.path.join(opt.logging_root, "results.txt"), "w") as out_file: out_file.write("%0.6f, %0.6f" % (np.mean(psnrs), np.mean(ssims))) print("Final mean PSNR %0.6f SSIM %0.6f" % (np.mean(psnrs), np.mean(ssims)))
def train(): # Parses indices of specific observations from comma-separated list. if opt.specific_observation_idcs is not None: specific_observation_idcs = util.parse_comma_separated_integers( opt.specific_observation_idcs) else: specific_observation_idcs = None img_sidelengths = util.parse_comma_separated_integers(opt.img_sidelengths) batch_size_per_sidelength = util.parse_comma_separated_integers( opt.batch_size_per_img_sidelength) max_steps_per_sidelength = util.parse_comma_separated_integers( opt.max_steps_per_img_sidelength) train_dataset = dataio.SceneClassDataset( root_dir=opt.data_root, max_num_instances=opt.max_num_instances_train, max_observations_per_instance=opt.max_num_observations_train, img_sidelength=img_sidelengths[0], specific_observation_idcs=specific_observation_idcs, samples_per_instance=1) assert (len(img_sidelengths) == len(batch_size_per_sidelength)), \ "Different number of image sidelengths passed than batch sizes." assert (len(img_sidelengths) == len(max_steps_per_sidelength)), \ "Different number of image sidelengths passed than max steps." if not opt.no_validation: assert (opt.val_root is not None), "No validation directory passed." val_dataset = dataio.SceneClassDataset( root_dir=opt.val_root, max_num_instances=opt.max_num_instances_val, max_observations_per_instance=opt.max_num_observations_val, img_sidelength=img_sidelengths[0], samples_per_instance=1) collate_fn = val_dataset.collate_fn val_dataloader = DataLoader(val_dataset, batch_size=2, shuffle=False, drop_last=True, collate_fn=val_dataset.collate_fn) model = SRNsModel(num_instances=train_dataset.num_instances, latent_dim=opt.embedding_size, has_params=opt.has_params, fit_single_srn=opt.fit_single_srn, use_unet_renderer=opt.use_unet_renderer, tracing_steps=opt.tracing_steps, freeze_networks=opt.freeze_networks) model.train() model.cuda() if opt.checkpoint_path is not None: print("Loading model from %s" % opt.checkpoint_path) util.custom_load(model, path=opt.checkpoint_path, discriminator=None, optimizer=None, overwrite_embeddings=opt.overwrite_embeddings) ckpt_dir = os.path.join(opt.logging_root, 'checkpoints') events_dir = os.path.join(opt.logging_root, 'events') util.cond_mkdir(opt.logging_root) util.cond_mkdir(ckpt_dir) util.cond_mkdir(events_dir) # Save command-line parameters log directory. with open(os.path.join(opt.logging_root, "params.txt"), "w") as out_file: out_file.write('\n'.join( ["%s: %s" % (key, value) for key, value in vars(opt).items()])) # Save text summary of model into log directory. with open(os.path.join(opt.logging_root, "model.txt"), "w") as out_file: out_file.write(str(model)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) writer = SummaryWriter(events_dir) iter = opt.start_step epoch = iter // len(train_dataset) step = 0 print('Beginning training...') # This loop implements training with an increasing image sidelength. cum_max_steps = 0 # Tracks max_steps cumulatively over all image sidelengths. for img_sidelength, max_steps, batch_size in zip( img_sidelengths, max_steps_per_sidelength, batch_size_per_sidelength): print("\n" + "#" * 10) print("Training with sidelength %d for %d steps with batch size %d" % (img_sidelength, max_steps, batch_size)) print("#" * 10 + "\n") train_dataset.set_img_sidelength(img_sidelength) # Need to instantiate DataLoader every time to set new batch size. train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, drop_last=True, collate_fn=train_dataset.collate_fn, pin_memory=opt.preload) cum_max_steps += max_steps # Loops over epochs. while True: for model_input, ground_truth in train_dataloader: model_outputs = model(model_input) optimizer.zero_grad() dist_loss = model.get_image_loss(model_outputs, ground_truth) reg_loss = model.get_regularization_loss( model_outputs, ground_truth) latent_loss = model.get_latent_loss() weighted_dist_loss = opt.l1_weight * dist_loss weighted_reg_loss = opt.reg_weight * reg_loss weighted_latent_loss = opt.kl_weight * latent_loss total_loss = (weighted_dist_loss + weighted_reg_loss + weighted_latent_loss) total_loss.backward() optimizer.step() print( "Iter %07d Epoch %03d L_img %0.4f L_latent %0.4f L_depth %0.4f" % (iter, epoch, weighted_dist_loss, weighted_latent_loss, weighted_reg_loss)) model.write_updates(writer, model_outputs, ground_truth, iter) writer.add_scalar("scaled_distortion_loss", weighted_dist_loss, iter) writer.add_scalar("scaled_regularization_loss", weighted_reg_loss, iter) writer.add_scalar("scaled_latent_loss", weighted_latent_loss, iter) writer.add_scalar("total_loss", total_loss, iter) if iter % opt.steps_til_val == 0 and not opt.no_validation: print("Running validation set...") model.eval() with torch.no_grad(): psnrs = [] ssims = [] dist_losses = [] for model_input, ground_truth in val_dataloader: model_outputs = model(model_input) dist_loss = model.get_image_loss( model_outputs, ground_truth).cpu().numpy() psnr, ssim = model.get_psnr( model_outputs, ground_truth) psnrs.append(psnr) ssims.append(ssim) dist_losses.append(dist_loss) model.write_updates(writer, model_outputs, ground_truth, iter, prefix='val_') writer.add_scalar("val_dist_loss", np.mean(dist_losses), iter) writer.add_scalar("val_psnr", np.mean(psnrs), iter) writer.add_scalar("val_ssim", np.mean(ssims), iter) model.train() iter += 1 step += 1 if iter == cum_max_steps: break if iter % opt.steps_til_ckpt == 0: util.custom_save(model, os.path.join( ckpt_dir, 'epoch_%04d_iter_%06d.pth' % (epoch, iter)), discriminator=None, optimizer=optimizer) if iter == cum_max_steps: break epoch += 1 util.custom_save(model, os.path.join(ckpt_dir, 'epoch_%04d_iter_%06d.pth' % (epoch, iter)), discriminator=None, optimizer=optimizer)
def train(): # Parses indices of specific observations from comma-separated list. if opt.specific_observation_idcs is not None: specific_observation_idcs = util.parse_comma_separated_integers( opt.specific_observation_idcs) else: specific_observation_idcs = None img_sidelengths = util.parse_comma_separated_integers(opt.img_sidelengths) batch_size_per_sidelength = util.parse_comma_separated_integers( opt.batch_size_per_img_sidelength) max_steps_per_sidelength = util.parse_comma_separated_integers( opt.max_steps_per_img_sidelength) train_dataset = dataio.PBWDataset(train=True) assert (len(img_sidelengths) == len(batch_size_per_sidelength)), \ "Different number of image sidelengths passed than batch sizes." assert (len(img_sidelengths) == len(max_steps_per_sidelength)), \ "Different number of image sidelengths passed than max steps." if not opt.no_validation: assert (opt.val_root is not None), "No validation directory passed." val_dataset = dataio.PBWDataset(train=False) val_dataloader = DataLoader(val_dataset, batch_size=16, shuffle=False, drop_last=True, collate_fn=val_dataset.collate_fn) model = SRNsModel3(latent_dim=opt.embedding_size, has_params=opt.has_params, fit_single_srn=True, tracing_steps=opt.tracing_steps, freeze_networks=opt.freeze_networks) model.train() model.cuda() if opt.checkpoint_path is not None: print("Loading model from %s" % opt.checkpoint_path) util.custom_load(model, path=opt.checkpoint_path, discriminator=None, optimizer=None, overwrite_embeddings=opt.overwrite_embeddings) ckpt_dir = os.path.join(opt.logging_root, 'checkpoints') events_dir = os.path.join(opt.logging_root, 'events') util.cond_mkdir(opt.logging_root) util.cond_mkdir(ckpt_dir) util.cond_mkdir(events_dir) # Save command-line parameters log directory. with open(os.path.join(opt.logging_root, "params.txt"), "w") as out_file: out_file.write('\n'.join( ["%s: %s" % (key, value) for key, value in vars(opt).items()])) # Save text summary of model into log directory. with open(os.path.join(opt.logging_root, "model.txt"), "w") as out_file: out_file.write(str(model)) optimizer = torch.optim.Adam(model.parameters(), lr=opt.lr) writer = SummaryWriter(events_dir) iter = opt.start_step epoch = iter // len(train_dataset) step = 0 print('Beginning training...') # This loop implements training with an increasing image sidelength. cum_max_steps = 0 # Tracks max_steps cumulatively over all image sidelengths. for img_sidelength, max_steps, batch_size in zip( img_sidelengths, max_steps_per_sidelength, batch_size_per_sidelength): print("\n" + "#" * 10) print("Training with sidelength %d for %d steps with batch size %d" % (img_sidelength, max_steps, batch_size)) print("#" * 10 + "\n") # Need to instantiate DataLoader every time to set new batch size. train_dataloader = DataLoader( train_dataset, batch_size=batch_size, shuffle=True, drop_last=True, collate_fn=train_dataset.collate_fn, ) cum_max_steps += max_steps # Loops over epochs. while True: for batch in train_dataloader: rgb, ext_mat, info, rgb_mat = batch ground_truth = {"rgb": rgb} model_input = (ext_mat, rgb_mat, info ) # color, pix coord, location, box model_outputs = model(model_input) optimizer.zero_grad() total_loss = model.get_image_loss(model_outputs, ground_truth) total_loss.backward() optimizer.step() if iter % 100 == 0: print("Iter %07d Epoch %03d L_img %0.4f" % (iter, epoch, total_loss)) if iter % opt.steps_til_val == 0 and not opt.no_validation: print("Running validation set...") acc = test(model, val_dataloader, str(iter)) print("Accuracy:", acc) iter += 1 step += 1 if iter == cum_max_steps: break if iter == cum_max_steps: break epoch += 1 util.custom_save(model, os.path.join(ckpt_dir, 'epoch_%04d_iter_%06d.pth' % (epoch, iter)), discriminator=None, optimizer=optimizer)
now = datetime.datetime.now() timestamp = now.isoformat() if args.name == 'none': exp_name = timestamp else: exp_name = args.name exp_counter = 0 save_folder = '{}/{}/'.format(args.save_folder, exp_name) ckpt_dir = os.path.join(save_folder, 'checkpoints') events_dir = os.path.join(save_folder, 'events') util.cond_mkdir(save_folder) util.cond_mkdir(ckpt_dir) util.cond_mkdir(events_dir) # Save command-line parameters log directory. with open(os.path.join(save_folder, "params.txt"), "w") as out_file: out_file.write('\n'.join(["%s: %s" % (key, value) for key, value in vars(args).items()])) # Save text summary of model into log directory. with open(os.path.join(save_folder, "model.txt"), "w") as out_file: out_file.write(str(model)) writer = SummaryWriter(events_dir) # writer.add_graph(model)
def test(): test_dataset = dataio.TwoViewsDataset( data_dir=args.test_dir, num_pairs_per_scene=args.test_pairs_per_scene, num_scenes=args.num_test_scenes, sidelength=args.sidelength) test_loader = data.DataLoader(test_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4) print(f'Size of test dataset {len(test_dataset)}') obs = test_loader.__iter__().next() data_util.show_batch_pairs(obs) input_shape = obs['image1'].size()[1:] # Load training params with open(args.train_log_dir + '/params.txt', 'r') as f: train_params = yaml.safe_load(f) model = nod.NodModel(embedding_dim=train_params['embedding_dim'], input_dims=input_shape, hidden_dim=train_params['hidden_dim'], num_slots=train_params['num_slots'], encoder=train_params['encoder'], decoder=train_params['decoder']) print("Loading model from %s" % args.checkpoint_path) util.custom_load(model, path=args.checkpoint_path) print("Evaluation to be saved to %s" % args.results_dir) model.to(device) model.eval() gt_comparison_dir = os.path.join(args.results_dir, 'gt_comparisons') sv_comps_dir = os.path.join(args.results_dir, 'components_same_view') dv_comps_dir = os.path.join(args.results_dir, 'components_diff_view') util.cond_mkdir(args.results_dir) util.cond_mkdir(gt_comparison_dir) util.cond_mkdir(sv_comps_dir) util.cond_mkdir(dv_comps_dir) # Save command-line parameters to log directory. with open(os.path.join(args.results_dir, "params.txt"), "w") as out_file: out_file.write('\n'.join( ["%s: %s" % (key, value) for key, value in vars(args).items()])) l2_loss = nn.MSELoss(reduction="mean") print('Beginning evaluation...') with torch.no_grad(): same_view_losses = [] diff_view_losses = [] total_losses = [] for batch_idx, data_batch in enumerate(test_loader): img1, img2 = data_batch['image1'].to( device), data_batch['image2'].to(device) batch_size = img1.shape[0] imgs = torch.cat((img1, img2), dim=0) w, h = imgs.size(-2), imgs.size(-1) images_gt = torch.cat((img1.unsqueeze(1), img2.unsqueeze(1)), dim=1) action1, action2 = data_batch['transf21'].to( device), data_batch['transf12'].to(device) actions = torch.cat((action1, action2), dim=0) out = model(imgs, actions) masks, masked_comps, recs = model.compose_image(out) rec_views = recs[:batch_size * 2] novel_views = recs[batch_size * 2:] same_view_loss = l2_loss(rec_views, imgs) novel_view_loss = l2_loss(novel_views, imgs) total_loss = same_view_loss + novel_view_loss same_view_losses.append(same_view_loss.item()) diff_view_losses.append(novel_view_loss.item()) total_losses.append(total_loss.item()) print( f"Number input images {batch_idx * args.batch_size} | Running l2 loss: {np.mean(total_losses)}" ) break if batch_idx * args.batch_size < args.save_out_first_n: rec_views = rec_views.reshape(2, args.batch_size, 3, w, h).transpose(0, 1) novel_views = novel_views.reshape(2, args.batch_size, 3, w, h).transpose(0, 1) same_view_masked_comps = masked_comps[:args.batch_size * 2].reshape( 2, args.batch_size, model.num_slots, 3, w, h).transpose(0, 1) diff_view_masked_comps = masked_comps[args.batch_size * 2:].reshape( 2, args.batch_size, model.num_slots, 3, w, h).transpose(0, 1) same_view_masks = masks[args.batch_size * 2:].reshape( 2, args.batch_size, model.num_slots, w, h).transpose(0, 1) diff_view_masks = masks[args.batch_size * 2:].reshape( 2, args.batch_size, model.num_slots, w, h).transpose(0, 1) # Expand to have 3 channels so can concat with rgb images same_view_masks = same_view_masks.unsqueeze(3).repeat( 1, 1, 1, 3, 1, 1) diff_view_masks = diff_view_masks.unsqueeze(3).repeat( 1, 1, 1, 3, 1, 1) # Shift to be in range [-1, 1] like rgb same_view_masks = same_view_masks * 2 - 1 diff_view_masks = diff_view_masks * 2 - 1 for i in range(args.batch_size): gt = images_gt[i] same_view_rec = rec_views[i] diff_view_rec = novel_views[i] # Save ground truth reconstruction comparison gt_vs_rec_vs_nv = torch.cat( (gt, same_view_rec, diff_view_rec), dim=0) gt_comparison_imgs = torchvision.utils.make_grid( gt_vs_rec_vs_nv, nrow=2, scale_each=False, normalize=True, range=(-1, 1)).cpu().detach().numpy() plt.imsave( os.path.join( gt_comparison_dir, f'{i + batch_idx * args.batch_size:04d}.png'), np.transpose(gt_comparison_imgs, (1, 2, 0))) # Save components sv_images = torch.cat( (images_gt[i].unsqueeze(1), same_view_rec.unsqueeze(1), same_view_masked_comps[i], same_view_masks[i]), dim=1) dv_images = torch.cat( (images_gt[i].unsqueeze(1), diff_view_rec.unsqueeze(1), diff_view_masked_comps[i], diff_view_masks[i]), dim=1) comps_same_view_images = torchvision.utils.make_grid( sv_images.reshape(-1, 3, h, w), nrow=2 * model.num_slots + 2, scale_each=False, normalize=True, range=(-1, 1)).cpu().detach().numpy() comps_diff_view_images = torchvision.utils.make_grid( dv_images.reshape(-1, 3, h, w), nrow=2 * model.num_slots + 2, scale_each=False, normalize=True, range=(-1, 1)).cpu().detach().numpy() plt.imsave( os.path.join( sv_comps_dir, f'{i + batch_idx * args.batch_size:04d}.png'), np.transpose(comps_same_view_images, (1, 2, 0))) plt.imsave( os.path.join( dv_comps_dir, f'{i + batch_idx * args.batch_size:04d}.png'), np.transpose(comps_diff_view_images, (1, 2, 0))) save_circles(model, args.results_dir, args.circle_source_img_path.split()) with open(os.path.join(args.results_dir, "results.txt"), "w") as out_file: out_file.write("Evaluation Metric: score \n\n") out_file.write( f"Same view rec l2 loss: {np.mean(same_view_losses):10f} \n") out_file.write( f"Diff view rec l2 loss: {np.mean(diff_view_losses):10f} \n") out_file.write(f"Rec l2 loss: {np.mean(total_losses):10f} \n") print("\nFinal score: ")
def masks_eval(model, results_dir, img_paths, num_renders=50): circles_dir = os.path.join(results_dir, 'circles') util.cond_mkdir(circles_dir) print('Evaluating IoU of generated masks')