def test_G_adv(test_loader, N, G_adv, epsilon=0.3, iteration=10): totalNumber = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } N.eval() G_adv.eval() for i, sample_batched in enumerate(test_loader): image, depth = sample_batched['image'], sample_batched['depth'] image = torch.autograd.Variable(image).cuda() depth = torch.autograd.Variable(depth).cuda() img_clean = image.clone() count = 0 if 1: img_adv = image.clone() img_adv.requires_grad = True img_min = float(image.min()[0].data.cpu().numpy()) img_max = float(image.max()[0].data.cpu().numpy()) while count < iteration: output = N(img_adv) loss = torch.abs(output - depth).mean() N.zero_grad() loss.backward() img_adv.grad.sign_() img_adv = img_adv + img_adv.grad img_adv = where(img_adv > image + epsilon, image + epsilon, img_adv) img_adv = where(img_adv < image - epsilon, image - epsilon, img_adv) img_adv = torch.clamp(img_adv, img_min, img_max) img_adv = torch.autograd.Variable(img_adv.data, requires_grad=True) count += 1 mask_adv = G_adv(img_adv) output = N(img_adv * mask_adv) batchSize = depth.size(0) errors = util.evaluateError(output, depth) errorSum = util.addErrors(errorSum, errors, batchSize) totalNumber = totalNumber + batchSize averageError = util.averageErrors(errorSum, totalNumber) print('rmse:', np.sqrt(averageError['MSE'])) print(averageError)
def test(test_loader, model, args): losses = AverageMeter() model.eval() model.cuda() totalNumber = 0 errorSum = {'MSE': 0, 'RMSE': 0, 'MAE': 0,'SSIM':0} for i, sample_batched in enumerate(test_loader): image, depth = sample_batched['image'], sample_batched['depth'] depth = depth.cuda(async=True) image = image.cuda() output = model(image) output = torch.nn.functional.interpolate(output,size=(440,440),mode='bilinear') batchSize = depth.size(0) testing_loss(depth,output,losses,batchSize) totalNumber = totalNumber + batchSize errors = util.evaluateError(output, depth,i,batchSize) errorSum = util.addErrors(errorSum, errors, batchSize) averageError = util.averageErrors(errorSum, totalNumber) averageError['RMSE'] = np.sqrt(averageError['MSE']) loss = float((losses.avg).data.cpu().numpy()) print('Model Loss {loss:.4f}\t' 'MSE {mse:.4f}\t' 'RMSE {rmse:.4f}\t' 'MAE {mae:.4f}\t' 'SSIM {ssim:.4f}\t'.format(loss=loss,mse=averageError['MSE']\ ,rmse=averageError['RMSE'],mae=averageError['MAE'],\ ssim=averageError['SSIM']))
def test(train_loader, model, model2, dir): totalNumber = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } model.eval() model2.eval() # if not os.path.exists(dir): # os.mkdir(dir) for i, sample_batched in enumerate(train_loader): image, depth_ = sample_batched['image'], sample_batched['depth'] image = torch.autograd.Variable(image, volatile=True).cuda() depth_ = torch.autograd.Variable(depth_, volatile=True).cuda(async=True) depth = model(image) mask = model2(image) output = model(image * mask) batchSize = depth.size(0) errors = util.evaluateError(output, depth_) errorSum = util.addErrors(errorSum, errors, batchSize) totalNumber = totalNumber + batchSize averageError = util.averageErrors(errorSum, totalNumber) # mask = mask.squeeze().view(228,304).data.cpu().float().numpy() # matplotlib.image.imsave(dir+'/mask'+str(i)+'.png', mask) print('rmse:', np.sqrt(averageError['MSE']))
def test(test_loader, model, thre): model.eval() totalNumber = 0 Ae = 0 Pe = 0 Re = 0 Fe = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } for i, sample_batched in enumerate(test_loader): image, depth = sample_batched['image'], sample_batched['depth'] depth = depth.cuda(sync=True) image = image.cuda() image = torch.autograd.Variable(image, volatile=True) depth = torch.autograd.Variable(depth, volatile=True) output = model(image) output = torch.nn.functional.upsample( output, size=[depth.size(2), depth.size(3)], mode='bilinear') depth_edge = edge_detection(depth) output_edge = edge_detection(output) batchSize = depth.size(0) totalNumber = totalNumber + batchSize errors = util.evaluateError(output, depth) errorSum = util.addErrors(errorSum, errors, batchSize) averageError = util.averageErrors(errorSum, totalNumber) edge1_valid = (depth_edge > thre) edge2_valid = (output_edge > thre) nvalid = np.sum( torch.eq(edge1_valid, edge2_valid).float().data.cpu().numpy()) A = nvalid / (depth.size(2) * depth.size(3)) nvalid2 = np.sum( ((edge1_valid + edge2_valid) == 2).float().data.cpu().numpy()) P = nvalid2 / (np.sum(edge2_valid.data.cpu().numpy())) R = nvalid2 / (np.sum(edge1_valid.data.cpu().numpy())) F = (2 * P * R) / (P + R) Ae += A Pe += P Re += R Fe += F Av = Ae / totalNumber Pv = Pe / totalNumber Rv = Re / totalNumber Fv = Fe / totalNumber print('PV', Pv) print('RV', Rv) print('FV', Fv) averageError['RMSE'] = np.sqrt(averageError['MSE']) print(averageError)
def train(train_loader, model, model2, optimizer, epoch): batch_time = AverageMeter() losses = AverageMeter() totalNumber = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } model.eval() model2.train() cos = nn.CosineSimilarity(dim=1, eps=0) get_gradient = sobel.Sobel().cuda() end = time.time() for i, sample_batched in enumerate(train_loader): image, depth_ = sample_batched['image'], sample_batched['depth'] image = torch.autograd.Variable(image).cuda() depth_ = torch.autograd.Variable(depth_).cuda() ones = torch.ones(depth_.size(0), 1, depth_.size(2), depth_.size(3)).float().cuda() ones = torch.autograd.Variable(ones) depth = model(image.clone()).detach() optimizer.zero_grad() mask = model2(image) output = model(image * mask) depth_grad = get_gradient(depth) output_grad = get_gradient(output) depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth) depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth) output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth) output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth) depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1) output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1) loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean() loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) + 0.5).mean() loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) + 0.5).mean() loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean() loss_rec = loss_depth + loss_normal + (loss_dx + loss_dy) loss_sparse = mask.mean() loss = loss_rec + loss_sparse * 5 losses.update(loss_sparse.data[0], image.size(0)) loss.backward() optimizer.step() batch_time.update(time.time() - end) end = time.time() batchSize = depth.size(0) errors = util.evaluateError(output, depth) errorSum = util.addErrors(errorSum, errors, batchSize) totalNumber = totalNumber + batchSize averageError = util.averageErrors(errorSum, totalNumber) print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})'.format( epoch, i, len(train_loader), batch_time=batch_time, loss=losses)) print('errors: ', averageError)
def test(test_loader, model, epoch): model.eval() totalNumber = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } for i, sample_batched in enumerate(test_loader): image, depth, label, image_name = sample_batched['image'], sample_batched['depth'], sample_batched['label'], \ sample_batched['image_name'] image = image.cuda() depth = depth.cuda() print(image_name) # label = label.cuda() output = F.softmax(model(image)) if args.rebuild_strategy == 'soft_sum': depth_pred = soft_sum(output, args.discrete_strategy) if args.rebuild_strategy == 'max': depth_pred = max(output, args.discrete_strategy) depth_pred = F.interpolate(depth_pred.float(), size=[depth.size(2), depth.size(3)], mode='bilinear') t = depth_pred.squeeze().float().cpu() / args.range print(t.size()) results_imgs = ToPILImage()(depth_pred.squeeze().float().cpu() / args.range) if not os.path.exists( str(args.img_path) + '/' + str(epoch) + 'epochs_results/'): os.mkdir(str(args.img_path) + '/' + str(epoch) + 'epochs_results/') results_imgs.save( str(args.img_path) + '/' + str(epoch) + 'epochs_results/' + str(image_name).strip(str(['']))) batchSize = depth.size(0) totalNumber = totalNumber + batchSize errors = util.evaluateError(depth_pred, depth) errorSum = util.addErrors(errorSum, errors, batchSize) averageError = util.averageErrors(errorSum, totalNumber) averageError['RMSE'] = np.sqrt(averageError['MSE']) print('epoch %d testing' % epoch) print(averageError) with open(os.path.join(args.save_path, 'records_val.csv'), 'a') as f: f.write('%d,%f,%f,%f,%f,%f,%f,%f,%f\n' % (epoch, averageError['MSE'], averageError['RMSE'], averageError['ABS_REL'], averageError['LG10'], averageError['MAE'], averageError['DELTA1'], averageError['DELTA2'], averageError['DELTA3'])) return averageError['RMSE']
def test(test_loader, model, thre): model.eval() totalNumber = 0 Ae = 0 Pe = 0 Re = 0 Fe = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } with torch.no_grad(): for i, sample_batched in enumerate(test_loader): image, depth = sample_batched['image'], sample_batched['depth'] # depth = depth.cuda(async=True) #depth = depth.cuda() #image = image.cuda() #image = torch.autograd.Variable(image, volatile=True) #depth = torch.autograd.Variable(depth, volatile=True) output = model(image) output = torch.nn.functional.upsample( output, size=[depth.size(2), depth.size(3)], mode='bilinear') #print(output.size()) #print(depth.size()) depth_edge = edge_detection(depth) output_edge = edge_detection(output) batchSize = depth.size(0) totalNumber = totalNumber + batchSize errors = util.evaluateError(output, depth) errorSum = util.addErrors(errorSum, errors, batchSize) averageError = util.averageErrors(errorSum, totalNumber) edge1_valid = (depth_edge > thre) edge2_valid = (output_edge > thre) nvalid = np.sum( torch.eq(edge1_valid, edge2_valid).float().data.cpu().numpy()) A = nvalid / (depth.size(2) * depth.size(3)) nvalid2 = np.sum( ((edge1_valid + edge2_valid) == 2).float().data.cpu().numpy()) P = nvalid2 / (np.sum(edge2_valid.data.cpu().numpy())) R = nvalid2 / (np.sum(edge1_valid.data.cpu().numpy())) F = (2 * P * R) / (P + R) Ae += A Pe += P Re += R Fe += F print('Epoch: [{0}/{1}]\t'.format(i, len(test_loader))) Av = Ae / totalNumber Pv = Pe / totalNumber Rv = Re / totalNumber Fv = Fe / totalNumber print('PV', Pv) print('RV', Rv) print('FV', Fv) averageError['RMSE'] = np.sqrt(averageError['MSE']) print(averageError) if is_resnet: if pretrain_logical: save_name = 'resnet_pretrained' else: save_name = 'renet_untrained' elif is_densenet: if pretrain_logical: save_name = 'densenet_pretrained' else: save_name = 'densenet_untrained' else: if pretrain_logical: save_name = 'senet_pretrained' else: save_name = 'senet_untrained' dir_path = os.path.dirname(os.path.realpath(__file__)) result_out_path = Path(dir_path + '/csvs') if not result_out_path.exists(): result_out_path.mkdir() with open('csvs/' + save_name + '.csv', 'w') as sub: sub.write('RV' + str(Rv) + '\n') sub.write('FV' + str(Fv) + '\n') sub.write('RMSE' + str(averageError['RMSE']) + '\n') print('Done!')
def test_model(test_loader, rmodel, model, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) # logging.basicConfig(filename=output_dir + 'testing.log',filemode='w',format='%(message)s',level=logging.INFO) testfile = open(output_dir + 'testing.log', 'w') since = time.time() batch_time = AverageMeter() losses = AverageMeter() base_losses = AverageMeter() totalNumber = 0 base_totalNumber = 0 errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } base_errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } rmodel.eval() # Set model to evaluate mode model.eval() # Iterate over data. end = time.time() for i, sample_batched in enumerate(test_loader['val']): image, depth, mask, imagedepth, imagemask = \ sample_batched['image'].cuda(), sample_batched['depth'].cuda(), sample_batched['mask'].cuda(), sample_batched['imagedepth'].cuda(), sample_batched['imagemask'].cuda() bs = image.size(0) depth_mean = 0 #1.8698 depth_std = 10 # 1.6716 # mi, _ = depth.view(bs,-1).min(1) # mi = mi.view(-1,1,1,1) # ma, _ = depth.view(bs,-1).max(1) # ma = ma.view(-1,1,1,1) # depth = (depth - mi)/(ma-mi) depth = (depth - depth_mean) / depth_std zero_points = torch.zeros_like(imagemask) base_input = image #torch.cat((image,imagemask, zero_points),dim=1) with torch.set_grad_enabled(False): output = model(base_input) # output2x = nn.functional.interpolate(output, size=None, scale_factor=2) # output = output.squeeze().view(114,152).data.cpu().float().numpy() # matplotlib.image.imsave(output_dir+'base'+'.png', output) # # normalize # mi, _ = output.view(bs,-1).min(1) # mi = mi.view(-1,1,1,1) # ma, _ = output.view(bs,-1).max(1) # ma = ma.view(-1,1,1,1) # output = (output - mi)/(ma-mi) output = (output - depth_mean) / depth_std # imagemask = image[:,3,:,:].unsqueeze(1) P = torch.zeros_like(output) diff_map = (output / depth - 1) * mask diff_map_reverse = (depth / output - 1) * mask adj_diff_map = torch.max(torch.abs(diff_map), torch.abs(diff_map_reverse)) > 0.25 # # diff_map = output - depth # # adj_diff_map = torch.zeros_like(diff_map).byte() # # for j in range(bs): # # subdiff = diff_map[j,:,:,:] # # adj_diff_map[j,:,:,:] = torch.abs(subdiff)>0.1*(subdiff.max()) # selection_percent = 0.25 # r = (torch.rand_like(mask) <= selection_percent) * adj_diff_map # P = torch.sign(diff_map) * mask * r.float() D = diff_map.view(bs, -1) m = torch.max(torch.abs(D), torch.abs(diff_map_reverse).view(bs, -1)).argmax(1).view( -1, 1) # P is -1 at point where updates = torch.sign(D.gather(1, m)) P.view(bs, -1).scatter_(1, m, updates) new_input = torch.cat((output, mask, P), dim=1) #torch.cat((image, P),dim=1) refined_output = rmodel(new_input) masked_depth = depth[mask.byte()] masked_output = output[mask.byte()] masked_refined_output = refined_output[mask.byte()] # statistics batch_time.update(time.time() - end) end = time.time() # batchSize = depth.size(0) errors = util.evaluateError( depth_std * masked_refined_output + depth_mean, depth_std * masked_depth + depth_mean) errorSum = util.addErrors(errorSum, errors, bs) totalNumber = totalNumber + bs averageError = util.averageErrors(errorSum, totalNumber) base_totalNumber = base_totalNumber + bs # base error base_errors = util.evaluateError( depth_std * masked_output + depth_mean, depth_std * masked_depth + depth_mean) base_errorSum = util.addErrors(base_errorSum, base_errors, bs) base_averageError = util.averageErrors(base_errorSum, base_totalNumber) # # refined error # errors = util.evaluateError(routput_, depth_) # errorSum = util.addErrors(errorSum, errors, batchSize) # averageError = util.averageErrors(errorSum, totalNumber) # statistics batch_time.update(time.time() - end) end = time.time() if i % 100 == 0: out_str = ( 'Time {batch_time.val:.2f} ({batch_time.sum:.1f})\t' # 'N={n} ' # 'RL {loss.val:.1f} ({loss.avg:.1f})\t' 'RMSE {rmse:.3f} ({rmse_avg:.3f})' 'BASE RMSE {base_rmse:.3f} ({base_rmse_avg:.3f})'.format( batch_time=batch_time, loss=losses, rmse=np.sqrt(errors['MSE']), rmse_avg=np.sqrt(averageError['MSE']), base_rmse=np.sqrt(base_errors['MSE']), base_rmse_avg=np.sqrt(base_averageError['MSE']))) #,n=n, base_rmse=np.sqrt(base_errors['MSE']), base_rmse_avg=np.sqrt(base_averageError['MSE']))) print(out_str) testfile.write(out_str + '\n') # logging.info(out_str) # plot approx. 10 images # files = glob.glob(dir + '*') # for f in files: # os.remove(f) if i % (len(test_loader['val']) // 8) == 0: depth = depth.squeeze().view(114, 152).data.cpu().float().numpy() matplotlib.image.imsave( output_dir + str(i) + 'groundtruth' + '.png', depth) output = output.squeeze().view(114, 152).data.cpu().float().numpy() matplotlib.image.imsave(output_dir + str(i) + 'base' + '.png', output) refined_output = refined_output.squeeze().view( 114, 152).data.cpu().float().numpy() matplotlib.image.imsave(output_dir + str(i) + 'refine' + '.png', refined_output) mask = mask.squeeze().view(114, 152).data.cpu().float().numpy() matplotlib.image.imsave(output_dir + str(i) + 'mask' + '.png', mask) image = image.squeeze().data.cpu().numpy().transpose(1, 2, 0) image = image[:, :, [0, 1, 2]] # image = image.view(228,304,3).data.cpu().float().numpy() matplotlib.image.imsave(output_dir + str(i) + 'image' + '.png', image) time_elapsed = time.time() - since s1 = 'Testing complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60) s2 = 'rmse:' + str(np.sqrt(averageError['MSE'])) s3 = 'abs_rel:' + str(averageError['ABS_REL']) s4 = 'mae:' + str(averageError['MAE']) s5 = 'lg10:' + str(averageError['LG10']) s6 = 'delta1:' + str(averageError['DELTA1']) s7 = 'delta2:' + str(averageError['DELTA2']) s8 = 'delta3:' + str(averageError['DELTA3']) bs2 = 'base_rmse:' + str(np.sqrt(base_averageError['MSE'])) bs3 = 'base_abs_rel:' + str(base_averageError['ABS_REL']) bs4 = 'base_mae:' + str(base_averageError['MAE']) bs5 = 'lg10:' + str(base_averageError['LG10']) bs6 = 'delta1:' + str(base_averageError['DELTA1']) bs7 = 'delta2:' + str(base_averageError['DELTA2']) bs8 = 'delta3:' + str(base_averageError['DELTA3']) print(s1) print(s2) print(s3) print(s4) print(s5) print(s6) print(s7) print(s8) print(bs2) print(bs3) print(bs4) print(bs5) print(bs6) print(bs7) print(bs8) testfile.write(s1 + '\n') testfile.write(s2 + '\n') testfile.write(s3 + '\n') testfile.write(s4 + '\n') testfile.write(s5 + '\n') testfile.write(s6 + '\n') testfile.write(s7 + '\n') testfile.write(s8 + '\n') testfile.write(bs2 + '\n') testfile.write(bs3 + '\n') testfile.write(bs4 + '\n') testfile.write(bs5 + '\n') testfile.write(bs6 + '\n') testfile.write(bs7 + '\n') testfile.write(bs8 + '\n')
def train_model(dataloader, rmodel, model, optimizer, start_epoch, epochs, output_dir): # # initialize grid for point selection # step = 5 # grid = torch.zeros([114,152],dtype=torch.uint8) # x = np.arange(0,114,step) # y = np.arange(0,152,step) # X,Y = np.meshgrid(x,y) # grid[X.ravel(),Y.ravel()]=1 # grid = grid.view(-1).cuda() if not os.path.exists(output_dir): os.makedirs(output_dir) best_acc = np.inf valfile = open(output_dir + 'validation.log', 'w') trainfile = open(output_dir + 'training.log', 'w') resultsfile = open(output_dir + 'results.log', 'w') # d1 = 114 # d2 = 152 # xx, yy = torch.meshgrid([torch.arange(d1), torch.arange(d2)]) # xx = xx.cuda() # yy = yy.cuda() for epoch in range(start_epoch, epochs): # adjust_learning_rate(optimizer, epoch) for phase in ['train', 'val']: if phase == 'train': rmodel.train() # Set model to training mode model.eval() # Set model to training mode else: continue rmodel.eval() # Set model to evaluate mode model.eval() # Set model to evaluate mode since = time.time() threshold_percent = AverageMeter() batch_time = AverageMeter() losses = AverageMeter() totalNumber = 0 acc = AverageMeter() errorSum = { 'MSE': 0, 'RMSE': 0, 'ABS_REL': 0, 'LG10': 0, 'MAE': 0, 'DELTA1': 0, 'DELTA2': 0, 'DELTA3': 0 } print('Epoch {}/{}'.format(epoch, epochs - 1)) print('-' * 10) cos = nn.CosineSimilarity(dim=1, eps=0) get_gradient = sobel.Sobel().cuda() mu = 1 lamda = 1 alpha = 0.5 # Iterate over data. end = time.time() for i, sample_batched in enumerate(dataloader[phase]): image, depth, mask, imagedepth, imagemask = \ sample_batched['image'].cuda(), sample_batched['depth'].cuda(), sample_batched['mask'].cuda(), sample_batched['imagedepth'].cuda(), sample_batched['imagemask'].cuda() bs = image.size(0) depth_mean = 0 #1.8698 depth_std = 10 #1.6716 # mi, _ = depth.view(bs,-1).min(1) # mi = mi.view(-1,1,1,1) # ma, _ = depth.view(bs,-1).max(1) # ma = ma.view(-1,1,1,1) # depth = (depth - mi)/(ma-mi) depth = (depth - depth_mean) / depth_std # zero the parameter gradients optimizer.zero_grad() zero_points = torch.zeros_like(imagemask) base_input = image #torch.cat((image,imagemask, zero_points),dim=1) with torch.set_grad_enabled(False): output = model(base_input) # output2x = nn.functional.interpolate(output, size=None, scale_factor=2) # output = output.squeeze().view(114,152).data.cpu().float().numpy() # matplotlib.image.imsave(output_dir+'base'+'.png', output) # # normalize # mi, _ = output.view(bs,-1).min(1) # mi = mi.view(-1,1,1,1) # ma, _ = output.view(bs,-1).max(1) # ma = ma.view(-1,1,1,1) # output = (output - mi)/(ma-mi) output = (output - depth_mean) / depth_std # # imagemask = image[:,3,:,:].unsqueeze(1) P = torch.zeros_like(output) # diff_map = (output/depth - 1)*mask # < 0 if output point is relatively farther # adj_diff_map = torch.abs(diff_map)>0.5 # # 3 levels of ordinal relatinos # # abs_diff_map = torch.abs(diff_map) # # K = 3 diff_map = (output / depth - 1) * mask diff_map_reverse = (depth / output - 1) * mask adj_diff_map = torch.max( torch.abs(diff_map), torch.abs(diff_map_reverse)) > 0.25 threshold_percent.update(adj_diff_map.sum() / mask.sum(), 1) # # diff_map = output - depth # # adj_diff_map = torch.zeros_like(diff_map).byte() # # for j in range(bs): # # subdiff = diff_map[j,:,:,:] # # adj_diff_map[j,:,:,:] = torch.abs(subdiff)>0.1*(subdiff.max()) # selection_percent = 0.25 # r = (torch.rand_like(mask) <= selection_percent) * adj_diff_map # P = torch.sign(diff_map) * mask * r.float() # m = r.view(bs,-1).data.nonzero() # sig = 5 # d1 = r.size(2) # d2 = r.size(3) # # xx, yy = torch.meshgrid([torch.arange(d1), torch.arange(d2)]) # # xx = xx.cuda() # # yy = yy.cuda() # F = torch.zeros_like(r).float() # g = F[0,0,:,:] # old_idx = 0 # for idx in m: # mask_idx = idx[0] # if (mask_idx != old_idx): # F[mask_idx-1,0,:,:] = g/g.max() # g = F[mask_idx,0,:,:] # one_idx = idx[1] # x0 = one_idx // d2 # y0 = one_idx % d2 # t = -0.5 * ((xx-x0)**2 + (yy-y0)**2).float() / sig**2 # kernel = torch.exp(t) # g += kernel # F[mask_idx,0,:,:] = g/g.max() # r = F D = diff_map.view(bs, -1) m = torch.max(torch.abs(D), torch.abs(diff_map_reverse).view( bs, -1)).argmax(1).view(-1, 1) # P is -1 at point where updates = torch.sign(D.gather(1, m)) P.view(bs, -1).scatter_(1, m, updates) # new_input = torch.cat((image, P),dim=1) # d = torch.abs(diff_map[0,:,:,:]) # depth = d.squeeze().view(228,304).data.cpu().float().numpy() # implot = plt.imshow(depth) # d2 = diff_map.size(3) # indices = torch.cat((m // d2, m % d2), dim=1).data.cpu() # plt.scatter(indices[0,1],indices[0,0],c='r') # plt.savefig(output_dir+'point'+'.png') # mask = mask[0,:,:,:] # mask = mask.squeeze().view(114,152).data.cpu().float().numpy() # matplotlib.image.imsave(output_dir+str(i)+'mask'+'.png', mask) # for j in range(bs): # batch size # # submask = mask[j,:,:,:].view(-1).byte() # submask = mask[j,:,:,:].view(-1).byte() # z = output[j,:,:,:].view(-1) # gt_depth = depth[j,:,:,:].view(-1) # subP = P[j,:,:,:].view(-1) # num_sampled = 1 # number of randomly sample points pairs in mask # NZ = submask.nonzero() # sample_idx = torch.multinomial(torch.ones(submask.nonzero().size(0)),num_sampled*2) # randomly_sampled = NZ[sample_idx,:] # J = randomly_sampled.view(num_sampled,2) # # # choose # # for ik in randomly_sampled: # # diff = gt_depth[ik]-z[i] # # select point-pair with greatest discrepancy # best_pair = None # max_diff = 0 # for (ik,jk) in J: #combinations(randomly_sampled, 2): # gt_diff = gt_depth[ik]/gt_depth[jk] # z_diff = z[ik]/z[jk] # diff = gt_diff - z_diff # if torch.abs(diff) > max_diff: # best_pair = (ik,jk) # max_diff = torch.abs(diff) # ik = best_pair[0] # jk = best_pair[1] # if max_diff<0: # predicted P1 should be relatively closer, P2 relatively further # subP[ik] = 1 # subP[jk] = -1 # else: # subP[ik] = -1 # subP[jk] = 1 new_input = torch.cat((output, mask, P), dim=1) # forward # track history if only in train with torch.set_grad_enabled(phase == 'train'): # output = model(image).detach() # P = torch.zeros_like(output).cuda() refined_output = rmodel(new_input) # P.zero_() # diff_map = (refined_output.detach()/depth - 1)*mask # < 0 if output point is relatively farther # D = diff_map.view(bs, -1) # m = torch.abs(D).argmax(1).view(-1,1) # updates = torch.sign(D.gather(1,m)) # P.view(bs,-1).scatter_(1,m,updates) # new_input = torch.cat((refined_output, mask, P),dim=1) # refined_output = rmodel(new_input) # P.zero_() # diff_map = (refined_output.detach()/depth - 1)*mask # < 0 if output point is relatively farther # D = diff_map.view(bs, -1) # m = torch.abs(D).argmax(1).view(-1,1) # updates = torch.sign(D.gather(1,m)) # P.view(bs,-1).scatter_(1,m,updates) # new_input = torch.cat((refined_output, mask, P),dim=1) # refined_output = rmodel(new_input) # routput_im = refined_output.squeeze().view(114,152).data.cpu().float().numpy() # matplotlib.image.imsave(output_dir+'refined'+'.png', routput_im) # output_im = refined_output.squeeze().view(114,152).data.cpu().float().numpy() # matplotlib.image.imsave(output_dir+'output'+'.png', output_im) byte_mask = mask.byte() masked_depth = depth[byte_mask] masked_output = output[byte_mask] masked_refined_output = refined_output[byte_mask] depth_ = depth output_ = refined_output ones = torch.ones(depth_.size(0), 1, depth_.size(2), depth_.size(3)).float().cuda() depth_grad = get_gradient(depth_) output_grad = get_gradient(output_) depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as( depth_) depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as( depth_) output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as( depth_) output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as( depth_) depth_normal = torch.cat( (-depth_grad_dx, -depth_grad_dy, ones), 1) output_normal = torch.cat( (-output_grad_dx, -output_grad_dy, ones), 1) loss_depth = torch.log(torch.abs(output_ - depth_) + alpha) loss_dx = torch.log( torch.abs(output_grad_dx - depth_grad_dx) + alpha) loss_dy = torch.log( torch.abs(output_grad_dy - depth_grad_dy) + alpha) loss_grad = loss_dx + loss_dy loss_normal = torch.abs(1 - cos(output_normal, depth_normal)) loss_normal.unsqueeze_(1) loss_depth_masked = loss_depth[byte_mask].mean() loss_grad_masked = loss_grad[byte_mask].mean() loss_normal_masked = loss_normal[byte_mask].mean() loss = loss_depth_masked + mu * loss_normal_masked + lamda * loss_grad_masked # loss_point_depth_masked = loss_depth.view(bs,-1).gather(1,m).mean() # loss_point_grad_masked = loss_grad.view(bs,-1).gather(1,m).mean() # loss_point_normal_masked = loss_normal.view(bs,-1).gather(1,m).mean() # loss_point_depth_masked = (loss_depth)[byte_mask*r].mean() # loss_point_grad_masked = (loss_grad)[byte_mask* r].mean() # loss_point_normal_masked = (loss_normal)[byte_mask* r].mean() # point_loss = loss_point_depth_masked + mu*loss_point_grad_masked + lamda*loss_point_normal_masked # loss += 0.5*point_loss # # #berHu loss (Deeper Depth Prediction with Fully Convolutional Residual Networks) # diff = torch.abs(masked_refined_output - masked_depth) # c = torch.max(diff)/5 # bh = diff.where(diff<=c, (diff**2+c**2)/(2*c)) # bh_loss = bh.sum()/mask.sum() # # point loss # diff_map = torch.abs((refined_output/depth - 1)*mask) # D = diff_map.view(bs, -1) # point_loss = torch.sum(D.gather(1,m))/bs # loss = bh_loss + 3*point_loss # loss = diff[diff<=c] + ((diff[diff>c])**2)/(2*c) # # loss function # rankingloss = 0 # for j in range(bs): # batch size # submask = mask[j,:,:,:].view(-1).byte() # z = refined_output[j,:,:,:].view(-1) # gt_depth = depth[j,:,:,:].view(-1) # # selection_points = torch.mul(submask,grid) # # NZ = selection_points.nonzero() # NZ = submask.nonzero() # M = 10 # number of pairs of points selected k = 1..M-1 # sample_idx = torch.multinomial(torch.ones(NZ.size(0)),2*M) # # if NZ.size(0) < 2*M: # # sample_idx = torch.multinomial(torch.ones(submask.nonzero().size(0)),2*M) # # # M = NZ.size(0)//2 # # else: # # sample_idx = torch.multinomial(torch.ones(NZ.size(0)),2*M) # J = NZ[sample_idx,:] # J = J.view(M,2) # # r = torch.zeros(M,).cuda() # # k = 0 # tau = 0.02 # rloss = 0 # for ik, jk in J: # M*(M-1)/2 loop iterations (so keep M small!) # if (gt_depth[ik]/gt_depth[jk] > 1 + tau): # rloss += torch.log(1+torch.exp(-z[ik]+z[jk])) # jk closer than ik # # r[k] = 1 # elif (gt_depth[jk]/gt_depth[ik] > 1 + tau): # ik closer than jk # rloss += torch.log(1+torch.exp(z[ik]-z[jk])) # # r[k] = -1 # else: # equal # rloss += (z[ik]-z[jk])**2 # # r[k] = 0 # # k = k + 1 # rankingloss += rloss/M # rl = rankingloss/bs # loss += 2*rl losses.update(loss.item(), bs) if phase == 'train': loss.backward() optimizer.step() # masked_depth = depth[mask.byte()] # masked_output = output[mask.byte()] # if phase == 'val': # depth_val = masked_depth.data.cpu().numpy() # output_val = masked_output.data.cpu().numpy() # indices = np.argsort(depth_val,kind='stable') # idx = np.argsort(output_val[indices],kind='stable') # n = idx.size # num_swaps = countSwaps(idx, n) # acc.update(num_swaps/n) # statistics batch_time.update(time.time() - end) end = time.time() errors = util.evaluateError(masked_refined_output, masked_output) errorSum = util.addErrors(errorSum, errors, bs) totalNumber = totalNumber + bs averageError = util.averageErrors(errorSum, totalNumber) if i % 100 == 0: out_str = ( 'Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t' 'Loss {loss.val:.4f} ({loss.avg:.4f})'.format( epoch, i, len(dataloader[phase]), batch_time=batch_time, loss=losses)) print(out_str) trainfile.write(out_str + '\n') # get accuracy as RMSE # epoch_acc = np.sqrt(averageError['MSE']) # epoch_acc = acc.avg # deep copy the model if phase == 'val': epoch_acc = np.sqrt(averageError['MSE']) valfile.write('epoch: ' + str(epoch) + ', rmse: ' + str(epoch_acc) + '\n') if epoch_acc < best_acc: best_acc = epoch_acc is_best = True else: is_best = False save_checkpoint(rmodel, optimizer, loss, False, epoch, output_dir) time_elapsed = time.time() - since s1 = 'Testing complete in {:.0f}m {:.0f}s'.format( time_elapsed // 60, time_elapsed % 60) s2 = 'rmse:' + str(np.sqrt(averageError['MSE'])) s3 = 'abs_rel:' + str(averageError['ABS_REL']) s4 = 'mae:' + str(averageError['MAE']) print(s1) print(s2) print(s3) print(s4) resultsfile.write(phase + '\n') resultsfile.write(s1 + '\n') resultsfile.write(s2 + '\n') resultsfile.write(s3 + '\n') resultsfile.write(s4 + '\n') print(threshold_percent.avg) valfile.write('\n') trainfile.write('\n') resultsfile.write('\n') # print('avg. num swaps:',epoch_acc) print() print('errors: ', averageError)