def visualize_error_map(test_loader, model, out_dir):
model.eval()
with torch.no_grad():
for i, sample_batched in enumerate(tqdm(test_loader)):
f_num = str(i + 1).zfill(5)
image, depth = sample_batched['image'], sample_batched['depth']
depth = depth.cuda(async=True)
image = image.cuda()
image = torch.autograd.Variable(image, volatile=True)
depth = torch.autograd.Variable(depth, volatile=True)
#out_mff, out_semff = model(image)
start = time.time()
out_mff, out_semff = model(image)
end = time.time()
time_elapsed = end - start
print(time_elapsed)
out_mff = torch.nn.functional.upsample(
out_mff, size=[depth.size(2), depth.size(3)], mode='bilinear')
out_semff = torch.nn.functional.upsample(
out_semff,
size=[depth.size(2), depth.size(3)],
mode='bilinear')
os.makedirs(osp.join(out_dir, "mff", "npys"), exist_ok=True)
np.save(osp.join(out_dir, "mff", "npys", "%s.npy" % f_num),
out_mff.data.cpu().numpy())
os.makedirs(osp.join(out_dir, "pcamff", "npys"), exist_ok=True)
np.save(osp.join(out_dir, "pcamff", "npys", "%s.npy" % f_num),
out_semff.data.cpu().numpy())
def test(nyu2_loader, model):
for i, image in enumerate(nyu2_loader):
image = torch.autograd.Variable(image, volatile=True)
if torch.cuda.is_available():
image = image.cuda()
out = model(image)
out = out.view(out.size(2), out.size(3)).data.cpu().numpy()
input_shape = image.data.cpu().numpy().shape[2:4]
out = imresize(arr=out, size=input_shape)
Image.fromarray(out.astype(np.uint8)).save('data/demo/out.png')
def test(model, rgb_path, output_path):
nyu2_loader = loaddata.readNyu2(rgb_path)
path, file = os.path.split(rgb_path)
file = f"{file.split('.')[0]}.png"
depth_path = os.path.join(output_path,
file) if output_path else os.path.join(
path, f"out_{file}")
print(f"{rgb_path} -> {depth_path}")
for i, image in enumerate(nyu2_loader):
image = image.cuda()
out = model(image)
matplotlib.image.imsave(
depth_path,
out.view(out.size(2), out.size(3)).data.cpu().numpy())
test_folder + '/model_epoch_' + str(epoch) + '.pth')
torch.save(discriminator.state_dict(),
test_folder + '/D_epoch_' + str(epoch) + '.pth')
####testing
model.eval()
skip = len(testloader) // 9 # save images every skip iters
with torch.no_grad():
for i, batch in enumerate(testloader):
if i == 0:
image, depth = batch['image'], batch['depth']
real_image, real_depth = image.cuda(), depth.cuda()
rand_image, rand_depth = image.cuda(), depth.cuda()
rgb0 = model(real_image, real_depth, real_depth)
rgb1 = model(real_image, real_depth, rand_depth)
rgb2 = model(rand_image, rand_depth, real_depth)
rgb_ori = utils.output_rgb(real_image)
dep_ori = utils.output_depth(real_depth)
out_ori = utils.output_rgb(rgb0)
dep_ran = utils.output_depth(rand_depth)
out_dep = utils.output_rgb(rgb1)
rgb_ran = utils.output_rgb(rand_image)
out_rgb = utils.output_rgb(rgb2)
img_merge = utils.merge_into_row([
torch.cuda.empty_cache()
G = G.cuda()
G.eval()
testloader = loaddata.getTestingData(opt.num_of_ref)
refloader = loaddata.getReferanceData(opt.num_of_ref)
for i, batch in enumerate(refloader):
rand_image, rand_depth = batch['image'].cuda() , batch['depth'].cuda()
break
for i, batch in enumerate(testloader):
image, depth = batch['image'], batch['depth']
real_image = image.cuda()
real_depth = depth.cuda()
for i_s in range(real_image.shape[0]):
# reconstruction
rgb = G(real_image[i_s:i_s+1], real_depth[i_s:i_s+1], real_depth[i_s:i_s+1])
rgb = 255*np.transpose(np.squeeze(rgb.cpu().detach().numpy()),(1,2,0))
filename = test_folder + '/' + str(i*opt.num_of_ref + i_s) + '_recon.png'
utils.save_image(rgb, filename)
# with reference pair
for i_t in range(opt.num_of_ref):
rgb = G(rand_image[i_t:i_t+1], rand_depth[i_t:i_t+1], real_depth[i_s:i_s+1])
rgb = 255*np.transpose(np.squeeze(rgb.cpu().detach().numpy()),(1,2,0))
filename = test_folder + '/' + str(i*opt.num_of_ref + i_s) + '_' + str(i_t + 1) + '.png'
utils.save_image(rgb, filename)
def visualize_error_map(test_loader, out_dir):
#model.eval()
hu_paths = sorted(
glob.glob(
"/home/takagi.kazunari/projects/TBDP-Net/experimental_results/output_npys/hu_1116/*"
))
tsm_paths = sorted(
glob.glob(
"/home/takagi.kazunari/projects/TBDP-Net/experimental_results/output_npys/1231_tsm_pcamff_v2_e16/mff/*"
))
mff_out_dir = "/home/takagi.kazunari/projects/TBDP-Net/hu_vs_tsm_1231_winter/hu"
tsm_out_dir = "/home/takagi.kazunari/projects/TBDP-Net/hu_vs_tsm_1231_winter/tsm"
makedirs_for_outputs(mff_out_dir)
makedirs_for_outputs(tsm_out_dir)
errors = None
with torch.no_grad():
for i, sample_batched in enumerate(tqdm(test_loader)):
f_num = str(i + 1).zfill(5)
image, depth = sample_batched['image'], sample_batched['depth']
depth = depth.cuda(async=True)
image = image.cuda()
image = torch.autograd.Variable(image, volatile=True)
depth = torch.autograd.Variable(depth, volatile=True)
#out_r = model(image)
#out_r = torch.nn.functional.upsample(out_r, size=[depth.size(2), depth.size(3)], mode='bilinear')
mff_out = torch.Tensor(np.load(hu_paths[i])).cuda()
tsm_out = torch.Tensor(np.load(tsm_paths[i])).cuda()
_, _target, Mask, ValidElement = util.setZeroToNum(mff_out, depth)
mff_error = torch.abs(mff_out - depth)
tsm_error = torch.abs(tsm_out - depth)
mff_error = mff_error * Mask.float()
tsm_error = tsm_error * Mask.float()
mff_rms = np.sqrt(mff_error.sum().data.cpu().numpy() /
ValidElement)
tsm_rms = np.sqrt(tsm_error.sum().data.cpu().numpy() /
ValidElement)
if errors is None:
errors = np.array([[mff_rms, tsm_rms]])
else:
errors = np.concatenate((errors, [[mff_rms, tsm_rms]]))
mff_delta = torch.max(depth / mff_out, mff_out / depth)
tsm_delta = torch.max(depth / tsm_out, tsm_out / depth)
mff_delta = torch.where(mff_delta < 1.25,
torch.zeros(mff_delta.shape).cuda(),
torch.ones(mff_delta.shape).cuda())
tsm_delta = torch.where(tsm_delta < 1.25,
torch.zeros(tsm_delta.shape).cuda(),
torch.ones(tsm_delta.shape).cuda())
image = image.data.cpu().numpy()[0]
mean = np.expand_dims(np.expand_dims(imagenet_stats['mean'],
axis=1),
axis=1)
std = np.expand_dims(np.expand_dims(imagenet_stats['std'], axis=1),
axis=1)
image = np.clip(np.floor((image * std + mean) * 255), 0,
255).transpose(1, 2, 0)
# save image
matplotlib.image.imsave(
osp.join(mff_out_dir, "images", "%s.png" % f_num),
np.array(image, dtype=np.uint8))
d_vmin = min(depth.min(), mff_out.min(), tsm_out.min())
d_vmax = max(depth.max(), mff_out.max(), tsm_out.max())
# save depth
matplotlib.image.imsave(
osp.join(mff_out_dir, "depths", "%s.png" % f_num),
depth.view(depth.size(2), depth.size(3)).data.cpu().numpy(),
vmin=d_vmin,
vmax=d_vmax,
cmap='winter')
# save refinement's outputs
matplotlib.image.imsave(
osp.join(mff_out_dir, "preds", "%s.png" % f_num),
mff_out.view(mff_out.size(2),
mff_out.size(3)).data.cpu().numpy(),
vmin=d_vmin,
vmax=d_vmax,
cmap='winter')
df_name = osp.join(tsm_out_dir, "preds", "%s.png" % f_num)
save_fig_with_colorbar(tsm_out.data.cpu().numpy()[0, 0],
d_vmin,
d_vmax,
f_name=df_name)
#matplotlib.image.imsave(osp.join(tsm_out_dir, "preds", "%s.png" % f_num),
# tsm_out.view(tsm_out.size(2), tsm_out.size(3)).data.cpu().numpy(),
# vmin=d_vmin, vmax=d_vmax, cmap='winter')
e_vmin = min(mff_error.min(), tsm_error.min())
e_vmax = max(mff_error.max(), tsm_error.max())
# save error maps
matplotlib.image.imsave(
osp.join(mff_out_dir, "error_maps", "%s.png" % f_num),
mff_error.view(mff_error.size(2),
mff_error.size(3)).data.cpu().numpy(),
vmin=e_vmin,
vmax=e_vmax,
cmap='hot')
ef_name = osp.join(tsm_out_dir, "error_maps", "%s.png" % f_num)
save_fig_with_colorbar(tsm_error.data.cpu().numpy()[0, 0],
e_vmin,
e_vmax,
f_name=ef_name,
cmap="hot")
#matplotlib.image.imsave(osp.join(tsm_out_dir, "error_maps", "%s.png" % f_num),
# tsm_error.view(tsm_error.size(2), tsm_error.size(3)).data.cpu().numpy(),
# vmin=e_vmin, vmax=e_vmax, cmap='hot')
"""
try:
matplotlib.image.imsave(osp.join(mff_out_dir, "refinement", "delta_1", "%s.png" % f_num),
mff_delta.view(mff_delta.size(2), mff_delta.size(3)).data.cpu().numpy(),
vmin=1.25, vmax=max(mff_delta.max(), semff_delta.max()), cmap='hot')
matplotlib.image.imsave(osp.join(semff_out_dir, "refinement", "delta_1", "%s.png" % f_num),
semff_delta.view(semff_delta.size(2), semff_delta.size(3)).data.cpu().numpy(),
vmin=1.25, vmax=max(mff_delta.max(), semff_delta.max()), cmap='hot')
except:
matplotlib.image.imsave(osp.join(mff_out_dir, "refinement", "delta_1", "%s.png" % f_num),
mff_delta.view(mff_delta.size(2), mff_delta.size(3)).data.cpu().numpy(),
vmin=min(mff_delta.min(), semff_delta.min()),
vmax=max(mff_delta.max(), semff_delta.max()), cmap='hot')
matplotlib.image.imsave(osp.join(semff_out_dir, "refinement", "delta_1", "%s.png" % f_num),
semff_delta.view(semff_delta.size(2), semff_delta.size(3)).data.cpu().numpy(),
vmin=min(mff_delta.min(), semff_delta.min()),
vmax=max(mff_delta.max(), semff_delta.max()), cmap='hot')
"""
matplotlib.image.imsave(
osp.join(mff_out_dir, "delta_1", "%s.png" % f_num),
mff_delta.view(mff_delta.size(2),
mff_delta.size(3)).data.cpu().numpy(),
cmap='gray')
matplotlib.image.imsave(
osp.join(tsm_out_dir, "delta_1", "%s.png" % f_num),
tsm_delta.view(tsm_delta.size(2),
tsm_delta.size(3)).data.cpu().numpy(),
cmap='gray')
"""
matplotlib.image.imsave(osp.join(mff_out_dir, "refinement", "rel", "%s.png" % f_num),
mff_rel.view(mff_rel.size(2), mff_rel.size(3)).data.cpu().numpy(),
vmin=0.0, vmax=max(mff_rel.max(), semff_rel.max()), cmap='hot')
matplotlib.image.imsave(osp.join(semff_out_dir, "refinement", "rel", "%s.png" % f_num),
semff_rel.view(semff_rel.size(2), semff_rel.size(3)).data.cpu().numpy(),
vmin=0.0, vmax=max(mff_rel.max(), semff_rel.max()), cmap='hot')
"""
#np.save(osp.join(out_dir, "npys", "%s.npy" % f_num), out_r.data.cpu().numpy())
np.savetxt("1231_tsm_mff_pcamff.csv", errors, delimiter=",")