def test_G_adv(test_loader, N, G_adv, epsilon=0.3, iteration=10):
totalNumber = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
N.eval()
G_adv.eval()
for i, sample_batched in enumerate(test_loader):
image, depth = sample_batched['image'], sample_batched['depth']
image = torch.autograd.Variable(image).cuda()
depth = torch.autograd.Variable(depth).cuda()
img_clean = image.clone()
count = 0
if 1:
img_adv = image.clone()
img_adv.requires_grad = True
img_min = float(image.min()[0].data.cpu().numpy())
img_max = float(image.max()[0].data.cpu().numpy())
while count < iteration:
output = N(img_adv)
loss = torch.abs(output - depth).mean()
N.zero_grad()
loss.backward()
img_adv.grad.sign_()
img_adv = img_adv + img_adv.grad
img_adv = where(img_adv > image + epsilon, image + epsilon,
img_adv)
img_adv = where(img_adv < image - epsilon, image - epsilon,
img_adv)
img_adv = torch.clamp(img_adv, img_min, img_max)
img_adv = torch.autograd.Variable(img_adv.data,
requires_grad=True)
count += 1
mask_adv = G_adv(img_adv)
output = N(img_adv * mask_adv)
batchSize = depth.size(0)
errors = util.evaluateError(output, depth)
errorSum = util.addErrors(errorSum, errors, batchSize)
totalNumber = totalNumber + batchSize
averageError = util.averageErrors(errorSum, totalNumber)
print('rmse:', np.sqrt(averageError['MSE']))
print(averageError)
def test(test_loader, model, args):
losses = AverageMeter()
model.eval()
model.cuda()
totalNumber = 0
errorSum = {'MSE': 0, 'RMSE': 0, 'MAE': 0,'SSIM':0}
for i, sample_batched in enumerate(test_loader):
image, depth = sample_batched['image'], sample_batched['depth']
depth = depth.cuda(async=True)
image = image.cuda()
output = model(image)
output = torch.nn.functional.interpolate(output,size=(440,440),mode='bilinear')
batchSize = depth.size(0)
testing_loss(depth,output,losses,batchSize)
totalNumber = totalNumber + batchSize
errors = util.evaluateError(output, depth,i,batchSize)
errorSum = util.addErrors(errorSum, errors, batchSize)
averageError = util.averageErrors(errorSum, totalNumber)
averageError['RMSE'] = np.sqrt(averageError['MSE'])
loss = float((losses.avg).data.cpu().numpy())
print('Model Loss {loss:.4f}\t'
'MSE {mse:.4f}\t'
'RMSE {rmse:.4f}\t'
'MAE {mae:.4f}\t'
'SSIM {ssim:.4f}\t'.format(loss=loss,mse=averageError['MSE']\
,rmse=averageError['RMSE'],mae=averageError['MAE'],\
ssim=averageError['SSIM']))
def test(train_loader, model, model2, dir):
totalNumber = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
model.eval()
model2.eval()
# if not os.path.exists(dir):
# os.mkdir(dir)
for i, sample_batched in enumerate(train_loader):
image, depth_ = sample_batched['image'], sample_batched['depth']
image = torch.autograd.Variable(image, volatile=True).cuda()
depth_ = torch.autograd.Variable(depth_,
volatile=True).cuda(async=True)
depth = model(image)
mask = model2(image)
output = model(image * mask)
batchSize = depth.size(0)
errors = util.evaluateError(output, depth_)
errorSum = util.addErrors(errorSum, errors, batchSize)
totalNumber = totalNumber + batchSize
averageError = util.averageErrors(errorSum, totalNumber)
# mask = mask.squeeze().view(228,304).data.cpu().float().numpy()
# matplotlib.image.imsave(dir+'/mask'+str(i)+'.png', mask)
print('rmse:', np.sqrt(averageError['MSE']))
def test(test_loader, model, thre):
model.eval()
totalNumber = 0
Ae = 0
Pe = 0
Re = 0
Fe = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
for i, sample_batched in enumerate(test_loader):
image, depth = sample_batched['image'], sample_batched['depth']
depth = depth.cuda(sync=True)
image = image.cuda()
image = torch.autograd.Variable(image, volatile=True)
depth = torch.autograd.Variable(depth, volatile=True)
output = model(image)
output = torch.nn.functional.upsample(
output, size=[depth.size(2), depth.size(3)], mode='bilinear')
depth_edge = edge_detection(depth)
output_edge = edge_detection(output)
batchSize = depth.size(0)
totalNumber = totalNumber + batchSize
errors = util.evaluateError(output, depth)
errorSum = util.addErrors(errorSum, errors, batchSize)
averageError = util.averageErrors(errorSum, totalNumber)
edge1_valid = (depth_edge > thre)
edge2_valid = (output_edge > thre)
nvalid = np.sum(
torch.eq(edge1_valid, edge2_valid).float().data.cpu().numpy())
A = nvalid / (depth.size(2) * depth.size(3))
nvalid2 = np.sum(
((edge1_valid + edge2_valid) == 2).float().data.cpu().numpy())
P = nvalid2 / (np.sum(edge2_valid.data.cpu().numpy()))
R = nvalid2 / (np.sum(edge1_valid.data.cpu().numpy()))
F = (2 * P * R) / (P + R)
Ae += A
Pe += P
Re += R
Fe += F
Av = Ae / totalNumber
Pv = Pe / totalNumber
Rv = Re / totalNumber
Fv = Fe / totalNumber
print('PV', Pv)
print('RV', Rv)
print('FV', Fv)
averageError['RMSE'] = np.sqrt(averageError['MSE'])
print(averageError)
def train(train_loader, model, model2, optimizer, epoch):
batch_time = AverageMeter()
losses = AverageMeter()
totalNumber = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
model.eval()
model2.train()
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
end = time.time()
for i, sample_batched in enumerate(train_loader):
image, depth_ = sample_batched['image'], sample_batched['depth']
image = torch.autograd.Variable(image).cuda()
depth_ = torch.autograd.Variable(depth_).cuda()
ones = torch.ones(depth_.size(0), 1, depth_.size(2),
depth_.size(3)).float().cuda()
ones = torch.autograd.Variable(ones)
depth = model(image.clone()).detach()
optimizer.zero_grad()
mask = model2(image)
output = model(image * mask)
depth_grad = get_gradient(depth)
output_grad = get_gradient(output)
depth_grad_dx = depth_grad[:, 0, :, :].contiguous().view_as(depth)
depth_grad_dy = depth_grad[:, 1, :, :].contiguous().view_as(depth)
output_grad_dx = output_grad[:, 0, :, :].contiguous().view_as(depth)
output_grad_dy = output_grad[:, 1, :, :].contiguous().view_as(depth)
depth_normal = torch.cat((-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat((-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(output - depth) + 0.5).mean()
loss_dx = torch.log(torch.abs(output_grad_dx - depth_grad_dx) +
0.5).mean()
loss_dy = torch.log(torch.abs(output_grad_dy - depth_grad_dy) +
0.5).mean()
loss_normal = torch.abs(1 - cos(output_normal, depth_normal)).mean()
loss_rec = loss_depth + loss_normal + (loss_dx + loss_dy)
loss_sparse = mask.mean()
loss = loss_rec + loss_sparse * 5
losses.update(loss_sparse.data[0], image.size(0))
loss.backward()
optimizer.step()
batch_time.update(time.time() - end)
end = time.time()
batchSize = depth.size(0)
errors = util.evaluateError(output, depth)
errorSum = util.addErrors(errorSum, errors, batchSize)
totalNumber = totalNumber + batchSize
averageError = util.averageErrors(errorSum, totalNumber)
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(train_loader),
batch_time=batch_time,
loss=losses))
print('errors: ', averageError)
def test(test_loader, model, epoch):
model.eval()
totalNumber = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
for i, sample_batched in enumerate(test_loader):
image, depth, label, image_name = sample_batched['image'], sample_batched['depth'], sample_batched['label'], \
sample_batched['image_name']
image = image.cuda()
depth = depth.cuda()
print(image_name)
# label = label.cuda()
output = F.softmax(model(image))
if args.rebuild_strategy == 'soft_sum':
depth_pred = soft_sum(output, args.discrete_strategy)
if args.rebuild_strategy == 'max':
depth_pred = max(output, args.discrete_strategy)
depth_pred = F.interpolate(depth_pred.float(),
size=[depth.size(2),
depth.size(3)],
mode='bilinear')
t = depth_pred.squeeze().float().cpu() / args.range
print(t.size())
results_imgs = ToPILImage()(depth_pred.squeeze().float().cpu() /
args.range)
if not os.path.exists(
str(args.img_path) + '/' + str(epoch) + 'epochs_results/'):
os.mkdir(str(args.img_path) + '/' + str(epoch) + 'epochs_results/')
results_imgs.save(
str(args.img_path) + '/' + str(epoch) + 'epochs_results/' +
str(image_name).strip(str([''])))
batchSize = depth.size(0)
totalNumber = totalNumber + batchSize
errors = util.evaluateError(depth_pred, depth)
errorSum = util.addErrors(errorSum, errors, batchSize)
averageError = util.averageErrors(errorSum, totalNumber)
averageError['RMSE'] = np.sqrt(averageError['MSE'])
print('epoch %d testing' % epoch)
print(averageError)
with open(os.path.join(args.save_path, 'records_val.csv'), 'a') as f:
f.write('%d,%f,%f,%f,%f,%f,%f,%f,%f\n' %
(epoch, averageError['MSE'], averageError['RMSE'],
averageError['ABS_REL'], averageError['LG10'],
averageError['MAE'], averageError['DELTA1'],
averageError['DELTA2'], averageError['DELTA3']))
return averageError['RMSE']
def test(test_loader, model, thre):
model.eval()
totalNumber = 0
Ae = 0
Pe = 0
Re = 0
Fe = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
with torch.no_grad():
for i, sample_batched in enumerate(test_loader):
image, depth = sample_batched['image'], sample_batched['depth']
# depth = depth.cuda(async=True)
#depth = depth.cuda()
#image = image.cuda()
#image = torch.autograd.Variable(image, volatile=True)
#depth = torch.autograd.Variable(depth, volatile=True)
output = model(image)
output = torch.nn.functional.upsample(
output, size=[depth.size(2), depth.size(3)], mode='bilinear')
#print(output.size())
#print(depth.size())
depth_edge = edge_detection(depth)
output_edge = edge_detection(output)
batchSize = depth.size(0)
totalNumber = totalNumber + batchSize
errors = util.evaluateError(output, depth)
errorSum = util.addErrors(errorSum, errors, batchSize)
averageError = util.averageErrors(errorSum, totalNumber)
edge1_valid = (depth_edge > thre)
edge2_valid = (output_edge > thre)
nvalid = np.sum(
torch.eq(edge1_valid, edge2_valid).float().data.cpu().numpy())
A = nvalid / (depth.size(2) * depth.size(3))
nvalid2 = np.sum(
((edge1_valid + edge2_valid) == 2).float().data.cpu().numpy())
P = nvalid2 / (np.sum(edge2_valid.data.cpu().numpy()))
R = nvalid2 / (np.sum(edge1_valid.data.cpu().numpy()))
F = (2 * P * R) / (P + R)
Ae += A
Pe += P
Re += R
Fe += F
print('Epoch: [{0}/{1}]\t'.format(i, len(test_loader)))
Av = Ae / totalNumber
Pv = Pe / totalNumber
Rv = Re / totalNumber
Fv = Fe / totalNumber
print('PV', Pv)
print('RV', Rv)
print('FV', Fv)
averageError['RMSE'] = np.sqrt(averageError['MSE'])
print(averageError)
if is_resnet:
if pretrain_logical:
save_name = 'resnet_pretrained'
else:
save_name = 'renet_untrained'
elif is_densenet:
if pretrain_logical:
save_name = 'densenet_pretrained'
else:
save_name = 'densenet_untrained'
else:
if pretrain_logical:
save_name = 'senet_pretrained'
else:
save_name = 'senet_untrained'
dir_path = os.path.dirname(os.path.realpath(__file__))
result_out_path = Path(dir_path + '/csvs')
if not result_out_path.exists():
result_out_path.mkdir()
with open('csvs/' + save_name + '.csv', 'w') as sub:
sub.write('RV' + str(Rv) + '\n')
sub.write('FV' + str(Fv) + '\n')
sub.write('RMSE' + str(averageError['RMSE']) + '\n')
print('Done!')
def test_model(test_loader, rmodel, model, output_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# logging.basicConfig(filename=output_dir + 'testing.log',filemode='w',format='%(message)s',level=logging.INFO)
testfile = open(output_dir + 'testing.log', 'w')
since = time.time()
batch_time = AverageMeter()
losses = AverageMeter()
base_losses = AverageMeter()
totalNumber = 0
base_totalNumber = 0
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
base_errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
rmodel.eval() # Set model to evaluate mode
model.eval()
# Iterate over data.
end = time.time()
for i, sample_batched in enumerate(test_loader['val']):
image, depth, mask, imagedepth, imagemask = \
sample_batched['image'].cuda(), sample_batched['depth'].cuda(), sample_batched['mask'].cuda(), sample_batched['imagedepth'].cuda(), sample_batched['imagemask'].cuda()
bs = image.size(0)
depth_mean = 0 #1.8698
depth_std = 10 # 1.6716
# mi, _ = depth.view(bs,-1).min(1)
# mi = mi.view(-1,1,1,1)
# ma, _ = depth.view(bs,-1).max(1)
# ma = ma.view(-1,1,1,1)
# depth = (depth - mi)/(ma-mi)
depth = (depth - depth_mean) / depth_std
zero_points = torch.zeros_like(imagemask)
base_input = image #torch.cat((image,imagemask, zero_points),dim=1)
with torch.set_grad_enabled(False):
output = model(base_input)
# output2x = nn.functional.interpolate(output, size=None, scale_factor=2)
# output = output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'base'+'.png', output)
# # normalize
# mi, _ = output.view(bs,-1).min(1)
# mi = mi.view(-1,1,1,1)
# ma, _ = output.view(bs,-1).max(1)
# ma = ma.view(-1,1,1,1)
# output = (output - mi)/(ma-mi)
output = (output - depth_mean) / depth_std
# imagemask = image[:,3,:,:].unsqueeze(1)
P = torch.zeros_like(output)
diff_map = (output / depth - 1) * mask
diff_map_reverse = (depth / output - 1) * mask
adj_diff_map = torch.max(torch.abs(diff_map),
torch.abs(diff_map_reverse)) > 0.25
# # diff_map = output - depth
# # adj_diff_map = torch.zeros_like(diff_map).byte()
# # for j in range(bs):
# # subdiff = diff_map[j,:,:,:]
# # adj_diff_map[j,:,:,:] = torch.abs(subdiff)>0.1*(subdiff.max())
# selection_percent = 0.25
# r = (torch.rand_like(mask) <= selection_percent) * adj_diff_map
# P = torch.sign(diff_map) * mask * r.float()
D = diff_map.view(bs, -1)
m = torch.max(torch.abs(D),
torch.abs(diff_map_reverse).view(bs,
-1)).argmax(1).view(
-1, 1)
# P is -1 at point where
updates = torch.sign(D.gather(1, m))
P.view(bs, -1).scatter_(1, m, updates)
new_input = torch.cat((output, mask, P),
dim=1) #torch.cat((image, P),dim=1)
refined_output = rmodel(new_input)
masked_depth = depth[mask.byte()]
masked_output = output[mask.byte()]
masked_refined_output = refined_output[mask.byte()]
# statistics
batch_time.update(time.time() - end)
end = time.time()
# batchSize = depth.size(0)
errors = util.evaluateError(
depth_std * masked_refined_output + depth_mean,
depth_std * masked_depth + depth_mean)
errorSum = util.addErrors(errorSum, errors, bs)
totalNumber = totalNumber + bs
averageError = util.averageErrors(errorSum, totalNumber)
base_totalNumber = base_totalNumber + bs
# base error
base_errors = util.evaluateError(
depth_std * masked_output + depth_mean,
depth_std * masked_depth + depth_mean)
base_errorSum = util.addErrors(base_errorSum, base_errors, bs)
base_averageError = util.averageErrors(base_errorSum, base_totalNumber)
# # refined error
# errors = util.evaluateError(routput_, depth_)
# errorSum = util.addErrors(errorSum, errors, batchSize)
# averageError = util.averageErrors(errorSum, totalNumber)
# statistics
batch_time.update(time.time() - end)
end = time.time()
if i % 100 == 0:
out_str = (
'Time {batch_time.val:.2f} ({batch_time.sum:.1f})\t'
# 'N={n} '
# 'RL {loss.val:.1f} ({loss.avg:.1f})\t'
'RMSE {rmse:.3f} ({rmse_avg:.3f})'
'BASE RMSE {base_rmse:.3f} ({base_rmse_avg:.3f})'.format(
batch_time=batch_time,
loss=losses,
rmse=np.sqrt(errors['MSE']),
rmse_avg=np.sqrt(averageError['MSE']),
base_rmse=np.sqrt(base_errors['MSE']),
base_rmse_avg=np.sqrt(base_averageError['MSE'])))
#,n=n, base_rmse=np.sqrt(base_errors['MSE']), base_rmse_avg=np.sqrt(base_averageError['MSE'])))
print(out_str)
testfile.write(out_str + '\n')
# logging.info(out_str)
# plot approx. 10 images
# files = glob.glob(dir + '*')
# for f in files:
# os.remove(f)
if i % (len(test_loader['val']) // 8) == 0:
depth = depth.squeeze().view(114, 152).data.cpu().float().numpy()
matplotlib.image.imsave(
output_dir + str(i) + 'groundtruth' + '.png', depth)
output = output.squeeze().view(114, 152).data.cpu().float().numpy()
matplotlib.image.imsave(output_dir + str(i) + 'base' + '.png',
output)
refined_output = refined_output.squeeze().view(
114, 152).data.cpu().float().numpy()
matplotlib.image.imsave(output_dir + str(i) + 'refine' + '.png',
refined_output)
mask = mask.squeeze().view(114, 152).data.cpu().float().numpy()
matplotlib.image.imsave(output_dir + str(i) + 'mask' + '.png',
mask)
image = image.squeeze().data.cpu().numpy().transpose(1, 2, 0)
image = image[:, :, [0, 1, 2]]
# image = image.view(228,304,3).data.cpu().float().numpy()
matplotlib.image.imsave(output_dir + str(i) + 'image' + '.png',
image)
time_elapsed = time.time() - since
s1 = 'Testing complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60,
time_elapsed % 60)
s2 = 'rmse:' + str(np.sqrt(averageError['MSE']))
s3 = 'abs_rel:' + str(averageError['ABS_REL'])
s4 = 'mae:' + str(averageError['MAE'])
s5 = 'lg10:' + str(averageError['LG10'])
s6 = 'delta1:' + str(averageError['DELTA1'])
s7 = 'delta2:' + str(averageError['DELTA2'])
s8 = 'delta3:' + str(averageError['DELTA3'])
bs2 = 'base_rmse:' + str(np.sqrt(base_averageError['MSE']))
bs3 = 'base_abs_rel:' + str(base_averageError['ABS_REL'])
bs4 = 'base_mae:' + str(base_averageError['MAE'])
bs5 = 'lg10:' + str(base_averageError['LG10'])
bs6 = 'delta1:' + str(base_averageError['DELTA1'])
bs7 = 'delta2:' + str(base_averageError['DELTA2'])
bs8 = 'delta3:' + str(base_averageError['DELTA3'])
print(s1)
print(s2)
print(s3)
print(s4)
print(s5)
print(s6)
print(s7)
print(s8)
print(bs2)
print(bs3)
print(bs4)
print(bs5)
print(bs6)
print(bs7)
print(bs8)
testfile.write(s1 + '\n')
testfile.write(s2 + '\n')
testfile.write(s3 + '\n')
testfile.write(s4 + '\n')
testfile.write(s5 + '\n')
testfile.write(s6 + '\n')
testfile.write(s7 + '\n')
testfile.write(s8 + '\n')
testfile.write(bs2 + '\n')
testfile.write(bs3 + '\n')
testfile.write(bs4 + '\n')
testfile.write(bs5 + '\n')
testfile.write(bs6 + '\n')
testfile.write(bs7 + '\n')
testfile.write(bs8 + '\n')
def train_model(dataloader, rmodel, model, optimizer, start_epoch, epochs,
output_dir):
# # initialize grid for point selection
# step = 5
# grid = torch.zeros([114,152],dtype=torch.uint8)
# x = np.arange(0,114,step)
# y = np.arange(0,152,step)
# X,Y = np.meshgrid(x,y)
# grid[X.ravel(),Y.ravel()]=1
# grid = grid.view(-1).cuda()
if not os.path.exists(output_dir):
os.makedirs(output_dir)
best_acc = np.inf
valfile = open(output_dir + 'validation.log', 'w')
trainfile = open(output_dir + 'training.log', 'w')
resultsfile = open(output_dir + 'results.log', 'w')
# d1 = 114
# d2 = 152
# xx, yy = torch.meshgrid([torch.arange(d1), torch.arange(d2)])
# xx = xx.cuda()
# yy = yy.cuda()
for epoch in range(start_epoch, epochs):
# adjust_learning_rate(optimizer, epoch)
for phase in ['train', 'val']:
if phase == 'train':
rmodel.train() # Set model to training mode
model.eval() # Set model to training mode
else:
continue
rmodel.eval() # Set model to evaluate mode
model.eval() # Set model to evaluate mode
since = time.time()
threshold_percent = AverageMeter()
batch_time = AverageMeter()
losses = AverageMeter()
totalNumber = 0
acc = AverageMeter()
errorSum = {
'MSE': 0,
'RMSE': 0,
'ABS_REL': 0,
'LG10': 0,
'MAE': 0,
'DELTA1': 0,
'DELTA2': 0,
'DELTA3': 0
}
print('Epoch {}/{}'.format(epoch, epochs - 1))
print('-' * 10)
cos = nn.CosineSimilarity(dim=1, eps=0)
get_gradient = sobel.Sobel().cuda()
mu = 1
lamda = 1
alpha = 0.5
# Iterate over data.
end = time.time()
for i, sample_batched in enumerate(dataloader[phase]):
image, depth, mask, imagedepth, imagemask = \
sample_batched['image'].cuda(), sample_batched['depth'].cuda(), sample_batched['mask'].cuda(), sample_batched['imagedepth'].cuda(), sample_batched['imagemask'].cuda()
bs = image.size(0)
depth_mean = 0 #1.8698
depth_std = 10 #1.6716
# mi, _ = depth.view(bs,-1).min(1)
# mi = mi.view(-1,1,1,1)
# ma, _ = depth.view(bs,-1).max(1)
# ma = ma.view(-1,1,1,1)
# depth = (depth - mi)/(ma-mi)
depth = (depth - depth_mean) / depth_std
# zero the parameter gradients
optimizer.zero_grad()
zero_points = torch.zeros_like(imagemask)
base_input = image #torch.cat((image,imagemask, zero_points),dim=1)
with torch.set_grad_enabled(False):
output = model(base_input)
# output2x = nn.functional.interpolate(output, size=None, scale_factor=2)
# output = output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'base'+'.png', output)
# # normalize
# mi, _ = output.view(bs,-1).min(1)
# mi = mi.view(-1,1,1,1)
# ma, _ = output.view(bs,-1).max(1)
# ma = ma.view(-1,1,1,1)
# output = (output - mi)/(ma-mi)
output = (output - depth_mean) / depth_std
# # imagemask = image[:,3,:,:].unsqueeze(1)
P = torch.zeros_like(output)
# diff_map = (output/depth - 1)*mask # < 0 if output point is relatively farther
# adj_diff_map = torch.abs(diff_map)>0.5
# # 3 levels of ordinal relatinos
# # abs_diff_map = torch.abs(diff_map)
# # K = 3
diff_map = (output / depth - 1) * mask
diff_map_reverse = (depth / output - 1) * mask
adj_diff_map = torch.max(
torch.abs(diff_map),
torch.abs(diff_map_reverse)) > 0.25
threshold_percent.update(adj_diff_map.sum() / mask.sum(),
1)
# # diff_map = output - depth
# # adj_diff_map = torch.zeros_like(diff_map).byte()
# # for j in range(bs):
# # subdiff = diff_map[j,:,:,:]
# # adj_diff_map[j,:,:,:] = torch.abs(subdiff)>0.1*(subdiff.max())
# selection_percent = 0.25
# r = (torch.rand_like(mask) <= selection_percent) * adj_diff_map
# P = torch.sign(diff_map) * mask * r.float()
# m = r.view(bs,-1).data.nonzero()
# sig = 5
# d1 = r.size(2)
# d2 = r.size(3)
# # xx, yy = torch.meshgrid([torch.arange(d1), torch.arange(d2)])
# # xx = xx.cuda()
# # yy = yy.cuda()
# F = torch.zeros_like(r).float()
# g = F[0,0,:,:]
# old_idx = 0
# for idx in m:
# mask_idx = idx[0]
# if (mask_idx != old_idx):
# F[mask_idx-1,0,:,:] = g/g.max()
# g = F[mask_idx,0,:,:]
# one_idx = idx[1]
# x0 = one_idx // d2
# y0 = one_idx % d2
# t = -0.5 * ((xx-x0)**2 + (yy-y0)**2).float() / sig**2
# kernel = torch.exp(t)
# g += kernel
# F[mask_idx,0,:,:] = g/g.max()
# r = F
D = diff_map.view(bs, -1)
m = torch.max(torch.abs(D),
torch.abs(diff_map_reverse).view(
bs, -1)).argmax(1).view(-1, 1)
# P is -1 at point where
updates = torch.sign(D.gather(1, m))
P.view(bs, -1).scatter_(1, m, updates)
# new_input = torch.cat((image, P),dim=1)
# d = torch.abs(diff_map[0,:,:,:])
# depth = d.squeeze().view(228,304).data.cpu().float().numpy()
# implot = plt.imshow(depth)
# d2 = diff_map.size(3)
# indices = torch.cat((m // d2, m % d2), dim=1).data.cpu()
# plt.scatter(indices[0,1],indices[0,0],c='r')
# plt.savefig(output_dir+'point'+'.png')
# mask = mask[0,:,:,:]
# mask = mask.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+str(i)+'mask'+'.png', mask)
# for j in range(bs): # batch size
# # submask = mask[j,:,:,:].view(-1).byte()
# submask = mask[j,:,:,:].view(-1).byte()
# z = output[j,:,:,:].view(-1)
# gt_depth = depth[j,:,:,:].view(-1)
# subP = P[j,:,:,:].view(-1)
# num_sampled = 1 # number of randomly sample points pairs in mask
# NZ = submask.nonzero()
# sample_idx = torch.multinomial(torch.ones(submask.nonzero().size(0)),num_sampled*2)
# randomly_sampled = NZ[sample_idx,:]
# J = randomly_sampled.view(num_sampled,2)
# # # choose
# # for ik in randomly_sampled:
# # diff = gt_depth[ik]-z[i]
# # select point-pair with greatest discrepancy
# best_pair = None
# max_diff = 0
# for (ik,jk) in J: #combinations(randomly_sampled, 2):
# gt_diff = gt_depth[ik]/gt_depth[jk]
# z_diff = z[ik]/z[jk]
# diff = gt_diff - z_diff
# if torch.abs(diff) > max_diff:
# best_pair = (ik,jk)
# max_diff = torch.abs(diff)
# ik = best_pair[0]
# jk = best_pair[1]
# if max_diff<0: # predicted P1 should be relatively closer, P2 relatively further
# subP[ik] = 1
# subP[jk] = -1
# else:
# subP[ik] = -1
# subP[jk] = 1
new_input = torch.cat((output, mask, P), dim=1)
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
# output = model(image).detach()
# P = torch.zeros_like(output).cuda()
refined_output = rmodel(new_input)
# P.zero_()
# diff_map = (refined_output.detach()/depth - 1)*mask # < 0 if output point is relatively farther
# D = diff_map.view(bs, -1)
# m = torch.abs(D).argmax(1).view(-1,1)
# updates = torch.sign(D.gather(1,m))
# P.view(bs,-1).scatter_(1,m,updates)
# new_input = torch.cat((refined_output, mask, P),dim=1)
# refined_output = rmodel(new_input)
# P.zero_()
# diff_map = (refined_output.detach()/depth - 1)*mask # < 0 if output point is relatively farther
# D = diff_map.view(bs, -1)
# m = torch.abs(D).argmax(1).view(-1,1)
# updates = torch.sign(D.gather(1,m))
# P.view(bs,-1).scatter_(1,m,updates)
# new_input = torch.cat((refined_output, mask, P),dim=1)
# refined_output = rmodel(new_input)
# routput_im = refined_output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'refined'+'.png', routput_im)
# output_im = refined_output.squeeze().view(114,152).data.cpu().float().numpy()
# matplotlib.image.imsave(output_dir+'output'+'.png', output_im)
byte_mask = mask.byte()
masked_depth = depth[byte_mask]
masked_output = output[byte_mask]
masked_refined_output = refined_output[byte_mask]
depth_ = depth
output_ = refined_output
ones = torch.ones(depth_.size(0), 1, depth_.size(2),
depth_.size(3)).float().cuda()
depth_grad = get_gradient(depth_)
output_grad = get_gradient(output_)
depth_grad_dx = depth_grad[:,
0, :, :].contiguous().view_as(
depth_)
depth_grad_dy = depth_grad[:,
1, :, :].contiguous().view_as(
depth_)
output_grad_dx = output_grad[:,
0, :, :].contiguous().view_as(
depth_)
output_grad_dy = output_grad[:,
1, :, :].contiguous().view_as(
depth_)
depth_normal = torch.cat(
(-depth_grad_dx, -depth_grad_dy, ones), 1)
output_normal = torch.cat(
(-output_grad_dx, -output_grad_dy, ones), 1)
loss_depth = torch.log(torch.abs(output_ - depth_) + alpha)
loss_dx = torch.log(
torch.abs(output_grad_dx - depth_grad_dx) + alpha)
loss_dy = torch.log(
torch.abs(output_grad_dy - depth_grad_dy) + alpha)
loss_grad = loss_dx + loss_dy
loss_normal = torch.abs(1 -
cos(output_normal, depth_normal))
loss_normal.unsqueeze_(1)
loss_depth_masked = loss_depth[byte_mask].mean()
loss_grad_masked = loss_grad[byte_mask].mean()
loss_normal_masked = loss_normal[byte_mask].mean()
loss = loss_depth_masked + mu * loss_normal_masked + lamda * loss_grad_masked
# loss_point_depth_masked = loss_depth.view(bs,-1).gather(1,m).mean()
# loss_point_grad_masked = loss_grad.view(bs,-1).gather(1,m).mean()
# loss_point_normal_masked = loss_normal.view(bs,-1).gather(1,m).mean()
# loss_point_depth_masked = (loss_depth)[byte_mask*r].mean()
# loss_point_grad_masked = (loss_grad)[byte_mask* r].mean()
# loss_point_normal_masked = (loss_normal)[byte_mask* r].mean()
# point_loss = loss_point_depth_masked + mu*loss_point_grad_masked + lamda*loss_point_normal_masked
# loss += 0.5*point_loss
# # #berHu loss (Deeper Depth Prediction with Fully Convolutional Residual Networks)
# diff = torch.abs(masked_refined_output - masked_depth)
# c = torch.max(diff)/5
# bh = diff.where(diff<=c, (diff**2+c**2)/(2*c))
# bh_loss = bh.sum()/mask.sum()
# # point loss
# diff_map = torch.abs((refined_output/depth - 1)*mask)
# D = diff_map.view(bs, -1)
# point_loss = torch.sum(D.gather(1,m))/bs
# loss = bh_loss + 3*point_loss
# loss = diff[diff<=c] + ((diff[diff>c])**2)/(2*c)
# # loss function
# rankingloss = 0
# for j in range(bs): # batch size
# submask = mask[j,:,:,:].view(-1).byte()
# z = refined_output[j,:,:,:].view(-1)
# gt_depth = depth[j,:,:,:].view(-1)
# # selection_points = torch.mul(submask,grid)
# # NZ = selection_points.nonzero()
# NZ = submask.nonzero()
# M = 10 # number of pairs of points selected k = 1..M-1
# sample_idx = torch.multinomial(torch.ones(NZ.size(0)),2*M)
# # if NZ.size(0) < 2*M:
# # sample_idx = torch.multinomial(torch.ones(submask.nonzero().size(0)),2*M)
# # # M = NZ.size(0)//2
# # else:
# # sample_idx = torch.multinomial(torch.ones(NZ.size(0)),2*M)
# J = NZ[sample_idx,:]
# J = J.view(M,2)
# # r = torch.zeros(M,).cuda()
# # k = 0
# tau = 0.02
# rloss = 0
# for ik, jk in J: # M*(M-1)/2 loop iterations (so keep M small!)
# if (gt_depth[ik]/gt_depth[jk] > 1 + tau):
# rloss += torch.log(1+torch.exp(-z[ik]+z[jk])) # jk closer than ik
# # r[k] = 1
# elif (gt_depth[jk]/gt_depth[ik] > 1 + tau): # ik closer than jk
# rloss += torch.log(1+torch.exp(z[ik]-z[jk]))
# # r[k] = -1
# else: # equal
# rloss += (z[ik]-z[jk])**2
# # r[k] = 0
# # k = k + 1
# rankingloss += rloss/M
# rl = rankingloss/bs
# loss += 2*rl
losses.update(loss.item(), bs)
if phase == 'train':
loss.backward()
optimizer.step()
# masked_depth = depth[mask.byte()]
# masked_output = output[mask.byte()]
# if phase == 'val':
# depth_val = masked_depth.data.cpu().numpy()
# output_val = masked_output.data.cpu().numpy()
# indices = np.argsort(depth_val,kind='stable')
# idx = np.argsort(output_val[indices],kind='stable')
# n = idx.size
# num_swaps = countSwaps(idx, n)
# acc.update(num_swaps/n)
# statistics
batch_time.update(time.time() - end)
end = time.time()
errors = util.evaluateError(masked_refined_output,
masked_output)
errorSum = util.addErrors(errorSum, errors, bs)
totalNumber = totalNumber + bs
averageError = util.averageErrors(errorSum, totalNumber)
if i % 100 == 0:
out_str = (
'Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.sum:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})'.format(
epoch,
i,
len(dataloader[phase]),
batch_time=batch_time,
loss=losses))
print(out_str)
trainfile.write(out_str + '\n')
# get accuracy as RMSE
# epoch_acc = np.sqrt(averageError['MSE'])
# epoch_acc = acc.avg
# deep copy the model
if phase == 'val':
epoch_acc = np.sqrt(averageError['MSE'])
valfile.write('epoch: ' + str(epoch) + ', rmse: ' +
str(epoch_acc) + '\n')
if epoch_acc < best_acc:
best_acc = epoch_acc
is_best = True
else:
is_best = False
save_checkpoint(rmodel, optimizer, loss, False, epoch, output_dir)
time_elapsed = time.time() - since
s1 = 'Testing complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60)
s2 = 'rmse:' + str(np.sqrt(averageError['MSE']))
s3 = 'abs_rel:' + str(averageError['ABS_REL'])
s4 = 'mae:' + str(averageError['MAE'])
print(s1)
print(s2)
print(s3)
print(s4)
resultsfile.write(phase + '\n')
resultsfile.write(s1 + '\n')
resultsfile.write(s2 + '\n')
resultsfile.write(s3 + '\n')
resultsfile.write(s4 + '\n')
print(threshold_percent.avg)
valfile.write('\n')
trainfile.write('\n')
resultsfile.write('\n')
# print('avg. num swaps:',epoch_acc)
print()
print('errors: ', averageError)
