# Predict
outputs = model(image)
# Compute the loss
l_depth = l1_criterion(outputs, depth_n)
l_ssim = torch.clamp(
(1 - ssim(outputs, depth_n, val_range=1000.0 / 10.0)) * 0.5, 0, 1)
loss = (1.0 * l_ssim) + (0.1 * l_depth)
running_loss += loss.item()
# Save output images, one at a time, to results
inputs_tensor = image.detach().cpu()
output_tensor = outputs.detach().cpu()
label_tensor = depth_n.detach().cpu()
depth_metric = depth * (config.train.max_depth / 1000.0)
outputs_tmp = DepthNorm(outputs)
outputs_metric = outputs_tmp * (config.train.max_depth / 1000.0)
metrics = compute_errors(depth_metric, outputs_metric)
# print(metrics)
for keys, values in metrics.items():
print(str(keys) + ': ' + str(values))
# Extract each tensor within batch and save results
for iii, sample_batched in enumerate(
zip(inputs_tensor, output_tensor, label_tensor)):
input, output, label = sample_batched
if key == 'real':
def test_model(save_dir, save_img=False, evaluate=True):
if not os.path.exists('%s/testimg' % save_dir):
os.makedirs('%s/testimg' % save_dir)
# load saved model
model = Model_rgbd().cuda()
model.load_state_dict(
torch.load(os.path.join(save_dir, 'model_1up_TXrefixed/epoch-5.pth')))
# model.load_state_dict(torch.load(os.path.join(save_dir, 'epoch-19.pth')))
model.eval()
print('model loaded for evaluation.')
# Load data
test_loader = getTestingDataOnly(batch_size=1)
train_loader_l, test_loader_l = getTranslucentData(batch_size=1)
with torch.cuda.device(0):
model.eval()
tot_len = len(
test_loader_l) # min(len(test_loader), len(test_loader_l))
testiter = iter(test_loader)
testiter_l = iter(test_loader_l)
for i in range(tot_len):
# print("Iteration "+str(i)+". loop start:")
try:
sample_batched = next(testiter)
sample_batched_l = next(testiter_l)
except StopIteration:
print(' (almost) end of iteration: %d.' % i)
break
print('/=/=/=/=/=/ iter %02d /=/=/=/=/' % i)
# (1) Pretext task : test and save
image_nyu = torch.autograd.Variable(sample_batched['image'].cuda())
depth_nyu = torch.autograd.Variable(
sample_batched['depth'].cuda(non_blocking=True))
mask_raw = torch.autograd.Variable(sample_batched_l['mask'].cuda())
depth_nyu_n = DepthNorm(depth_nyu)
# # Apply random mask to it
ordered_index = list(
range(depth_nyu.shape[0])
) # NOTE: NYU test batch size shouldn't be bigger than lucent's.
mask_new = mask_raw[ordered_index, :, :, :]
depth_nyu_masked = resize2d(depth_nyu_n, (480, 640)) * mask_new
# if i <= 1:
# print('====/ %02d /====' % i)
# print(image_nyu.shape)
# print(" " + str(torch.max(image_nyu)) + " " + str(torch.min(image_nyu)))
# print(depth_nyu.shape)
# print(" " + str(torch.max(depth_nyu)) + " " + str(torch.min(depth_nyu)))
# print(mask_new.shape)
# print(" " + str(torch.max(mask_new)) + " " + str(torch.min(mask_new)))
# Predict
(htped_out_t1, _) = model(image_nyu, depth_nyu_masked)
depth_out_t1 = DepthNorm(htped_out_t1)
dn_resized = resize2d(depth_nyu, (240, 320))
if save_img:
# Save image
vutils.save_image(depth_out_t1,
'%s/testimg/1out_%02d.png' % (save_dir, i),
normalize=True,
range=(0, 1000))
if not os.path.exists('%s/testimg/1in_000000_%02d.png' %
(save_dir, i)):
vutils.save_image(depth_nyu_masked,
'%s/testimg/1in_%02d.png' %
(save_dir, i),
normalize=True,
range=(0, 1000))
save_error_image(depth_out_t1 - dn_resized,
'%s/testimg/1diff_%02d.png' % (save_dir, i),
normalize=True,
range=(-300, 300))
del image_nyu, depth_nyu, htped_out_t1, depth_out_t1, dn_resized
# (2) Main task : test and save
image = torch.autograd.Variable(sample_batched_l['image'].cuda())
depth_in = torch.autograd.Variable(
sample_batched_l['depth_raw'].cuda())
htped_in = DepthNorm(depth_in)
depth_gt = torch.autograd.Variable(
sample_batched_l['depth_truth'].cuda(non_blocking=True))
(_, htped_out_t2) = model(image, htped_in)
depth_out_t2 = DepthNorm(htped_out_t2)
mask_small = resize2dmask(mask_raw, (240, 320))
obj_mask = thresh_mask(depth_gt, resize2d(depth_in, (240, 320)))
# print(" " + str(torch.max(depth_out_t2)) + " " + str(torch.min(depth_out_t2)))
# print(" " + str(torch.max(depth_gt)) + " " + str(torch.min(depth_gt)))
# print(" " + str(torch.max(depth_in)) + " " + str(torch.min(depth_in)))
if i == 0:
(s0, s1, s2, s3) = depth_out_t2.size()
# https://stackoverflow.com/questions/22392497/how-to-add-a-new-row-to-an-empty-numpy-array
true_y = np.empty((0, s1, s2, s3), float)
raw_y = np.empty((0, s1, s2, s3), float)
pred_y = np.empty((0, s1, s2, s3), float)
mask_y = np.empty((0, s1, s2, s3), float)
objmask_y = np.empty((0, s1, s2, s3), float)
if evaluate:
true_y = np.append(true_y, depth_gt.cpu().numpy(), axis=0)
raw_y = np.append(raw_y,
resize2d(depth_in, (240, 320)).cpu().numpy(),
axis=0)
pred_y = np.append(pred_y,
depth_out_t2.detach().cpu().numpy(),
axis=0)
mask_y = np.append(mask_y, mask_small.cpu().numpy(), axis=0)
objmask_y = np.append(objmask_y,
obj_mask.cpu().numpy(),
axis=0)
# dl = depth_in.cpu().numpy()
# hl = htped_in.cpu().numpy()
# dr = resize2d(depth_in, (240, 320)).cpu().numpy()
# hr = resize2d(htped_in, (240, 320)).cpu().numpy()
# do = depth_out_t2.cpu().detach().numpy()
# gr = depth_gt.cpu().numpy()
#
# print(" Depth input (original size):" + str(np.min(dl)) + "~" + str(np.max(dl)) + " (" + str(np.mean(dl)) + ")")
# print(" Depth Normed (original size):" + str(np.min(hl)) + "~" + str(np.max(hl)) + " (" + str(np.mean(hl)) + ")")
#
# print(" Depth input (resized):" + str(np.min(dr)) + "~" + str(np.max(dr)) + " (" + str(np.mean(dr)) + ")")
# print(" Depth Normed (resized):" + str(np.min(hr)) + "~" + str(np.max(hr)) + " (" + str(np.mean(hr)) + ")")
#
# print(" Output converted to depth:" + str(np.min(do)) + "~" + str(np.max(do)) + " (" + str(np.mean(do)) + ")")
# print(" GT depth (original size):" + str(np.min(gr)) + "~" + str(np.max(gr)) + " (" + str(np.mean(gr)) + ")")
if save_img:
if not os.path.exists('%s/testimg/2truth_000000_%02d.png' %
(save_dir, i)):
vutils.save_image(depth_in,
'%s/testimg/2in_%02d.png' %
(save_dir, i),
normalize=True,
range=(0, 500))
vutils.save_image(resize2d(depth_in, (240, 320)),
'%s/testimg/2in_s_%02d.png' %
(save_dir, i),
normalize=True,
range=(0, 500))
vutils.save_image(depth_gt,
'%s/testimg/2truth_%02d.png' %
(save_dir, i),
normalize=True,
range=(0, 500))
vutils.save_image(depth_out_t2,
'%s/testimg/2out_%02d.png' % (save_dir, i),
normalize=True,
range=(0, 500))
save_error_image(resize2d(depth_out_t2, (480, 640)) - depth_in,
'%s/testimg/2corr_%02d.png' % (save_dir, i),
normalize=True,
range=(-50, 50),
mask=mask_raw)
save_error_image(depth_out_t2 - depth_gt,
'%s/testimg/2diff_%02d.png' % (save_dir, i),
normalize=True,
range=(-50, 50),
mask=mask_small)
vutils.save_image(mask_small,
'%s/testimg/2_mask_%02d.png' % (save_dir, i),
normalize=True,
range=(-0.5, 1.5))
vutils.save_image(obj_mask,
'%s/testimg/2_objmask_%02d.png' %
(save_dir, i),
normalize=True,
range=(-0.5, 1.5))
del image, htped_in, depth_in, depth_gt, depth_out_t2, mask_raw, mask_small
if evaluate:
eo = eo_r = 0
print(
'# \ta1 \ta2 \ta3 \tabsrel\trmse \tlog10 | \timprovements--> '
)
for j in range(len(true_y)):
# errors = compute_errors(true_y[j], pred_y[j], mask_y[j])
errors_object = compute_errors(true_y[j], pred_y[j],
mask_y[j] * objmask_y[j])
# errors_r = compute_errors(true_y[j], raw_y[j], mask_y[j])
errors_object_r = compute_errors(true_y[j], raw_y[j],
mask_y[j] * objmask_y[j])
eo = eo + errors_object
eo_r = eo_r + errors_object_r
print('{j:2d} | \t'
'{e[1]:.4f}\t'
'{e[2]:.4f}\t'
'{e[3]:.4f}\t'
'{e[4]:.4f}\t'
'{e[5]:.3f}\t'
'{e[6]:.4f} | \t'
'{f1[1]:+.3f}\t'
'{f1[2]:+.3f}\t'
'{f1[3]:+.3f}\t'
'{f2[4]:+.3f}\t'
'{f2[5]:+.3f}\t'
'{f2[6]:+.3f}'.format(
j=j,
e=errors_object,
f1=(1 - errors_object_r) / (1 - errors_object) - 1,
f2=errors_object_r / errors_object - 1))
eo = eo / len(true_y)
eo_r = eo_r / len(true_y)
print('\ntotal \t'
'{e[1]:.4f}\t'
'{e[2]:.4f}\t'
'{e[3]:.4f}\t'
'{e[4]:.4f}\t'
'{e[5]:.3f}\t'
'{e[6]:.4f} | \t'
'{f1[1]:+.3f}\t'
'{f1[2]:+.3f}\t'
'{f1[3]:+.3f}\t'
'{f2[4]:+.3f}\t'
'{f2[5]:+.3f}\t'
'{f2[6]:+.3f}'.format(e=eo,
f1=(1 - eo_r) / (1 - eo) - 1,
f2=eo_r / eo - 1))
def main():
# Arguments
parser = argparse.ArgumentParser(
description=
'High Quality Monocular Depth Estimation via Transfer Learning')
parser.add_argument('--epochs',
default=config.TRAIN_EPOCH,
type=int,
help='number of total epochs to run')
parser.add_argument('--lr',
'--learning-rate',
default=1e-4,
type=float,
help='initial learning rate')
parser.add_argument('--bs', default=4, type=int, help='batch size')
args = parser.parse_args()
# Create model
model = Model().cuda()
print('Model created.')
# Training parameters
optimizer = torch.optim.Adam(model.parameters(), args.lr)
batch_size = 1
prefix = 'densenet_' + str(batch_size)
# Load data
train_loader, test_loader = getTrainingTestingData(batch_size=batch_size)
print(len(train_loader), len(test_loader))
train_loader = None
# Logging
# writer = SummaryWriter(comment='{}-lr{}-e{}-bs{}'.format(prefix, args.lr, args.epochs, args.bs), flush_secs=30)
niter = 0
if config.load_model and config.model_name != '':
model.load_state_dict(torch.load('Checkpoints\\%s' %
config.model_name))
niter = int(config.model_name.split('_')[-3])
print('load success -> %s' % config.model_name)
bicubic_total_loss = 0
our_total_loss = 0
bicubic_total_loss_rel = 0
our_total_loss_rel = 0
bicubic_total_loss_rms = 0
our_total_loss_rms = 0
bicubic_total_loss = 0
our_total_loss = 0
count = 0
model.eval()
for i, sample_batched in enumerate(test_loader):
if i > 1500:
break
# optimizer.zero_grad()
print(i)
# Prepare sample and target
image = torch.autograd.Variable(sample_batched['image'].cuda())
depth = torch.autograd.Variable(
sample_batched['depth'].cuda(non_blocking=True))
rgb = sample_batched['image'][:, :3, :, :].numpy().squeeze().transpose(
(1, 2, 0)) * 255.
rgb = rgb.astype(np.uint8)
raw = sample_batched['image'][:, 3:, :, :].numpy().squeeze()
raw = raw * 1000.
# raw = raw.astype(np.uint8)
# plt.subplot(141), plt.imshow(rgb)
# plt.show()
# Normalize depth
depth_n = DepthNorm(depth)
gt = depth_n.detach().cpu().numpy().squeeze()
gt = DepthNorm(gt)
# Predict
output = model(image)
output = output.detach().cpu().numpy().squeeze()
output = DepthNorm(output)
# raw = DepthNorm(raw)
raw = raw / 100.
output = output / 100.
gt = gt / 100.
our_diff = np.mean(np.abs(gt - output))
bic_diff = np.mean(np.abs(gt - raw))
our_total_loss_rel += our_diff / gt
bicubic_total_loss_rel += bic_diff / gt
our_total_loss_rms += np.mean(np.square(our_diff))
bicubic_total_loss_rms += np.mean(np.square(bic_diff))
count += 1
# print(np.max(raw), np.max(output), np.max(gt))
res = np.concatenate([raw, output, gt], axis=1)
res = depth2color(res, cmap='plasma')
res = np.split(res, 3, axis=1)
res = make_grid([rgb, res[0], res[1], res[2]])
BASE_DIR = 'E:\Experimrnt\\SR'
# cv2.imwrite('%s/%04d-pseudo.png' % (BASE_DIR, i), res[:, :, (2,1,0)])
# plt.subplot(142), plt.imshow(raw, cmap='plasma')
# plt.subplot(143), plt.imshow(output, cmap='plasma')
# plt.subplot(144), plt.imshow(gt, cmap='plasma')
# plt.show()
print(our_total_loss_rel / count, bicubic_total_loss_rel / count)
print(np.sqrt(our_total_loss_rms / count),
np.sqrt(bicubic_total_loss_rms / count))