def test_synthesize_batch_view():
"""
gt depth와 gt pose를 입력했을 때 스케일 별로 복원되는 이미지를 정성적으로 확인
실제 target image와 복원된 "single" scale target image를 눈으로 비교
"""
dataset = TfrecordGenerator(op.join(opts.DATAPATH_TFR,
"kitti_raw_test")).get_generator()
print("\n===== start test_synthesize_batch_view")
scale_idx = 1
for i, features in enumerate(dataset):
stacked_image = features['image']
intrinsic = features['intrinsic']
depth_gt = features['depth_gt']
pose_gt = features['pose_gt']
source_image, target_image = uf.split_into_source_and_target(
stacked_image)
depth_gt_ms = uf.multi_scale_depths(depth_gt, [1, 2, 4, 8])
batch, height, width, _ = target_image.get_shape().as_list()
# check only 1 scale
depth_scaled = depth_gt_ms[scale_idx]
width_ori = source_image.get_shape().as_list()[2]
batch, height_sc, width_sc, _ = depth_scaled.get_shape().as_list()
scale = int(width_ori // width_sc)
# create synthesizer
synthesizer = SynthesizeBatchBasic(shape=(batch, height_sc, width_sc),
num_src=4,
scale=scale)
# adjust intrinsic upto scale
intrinsic_sc = layers.Lambda(
lambda intrin: synthesizer.scale_intrinsic(intrin, scale),
name=f"scale_intrin_sc{scale}")(intrinsic)
# reorganize source images: [batch, 4, height, width, 3]
srcimg_scaled = layers.Lambda(
lambda image: synthesizer.reshape_source_images(image),
name=f"reorder_source_sc{scale}")(source_image)
# EXECUTE
recon_image_sc = synthesizer.synthesize_batch_view(srcimg_scaled,
depth_scaled,
pose_gt,
intrinsic_sc,
suffix=f"sc{scale}")
print("reconstructed image shape:", recon_image_sc.get_shape())
# convert single target image in batch
target_image = uf.to_uint8_image(target_image[0]).numpy()
recon_image = uf.to_uint8_image(recon_image_sc[0]).numpy()
recon_image = recon_image.reshape((4 * height_sc, width_sc, 3))
recon_image = cv2.resize(recon_image, (width, height * 4),
interpolation=cv2.INTER_NEAREST)
view = np.concatenate([target_image, recon_image], axis=0)
cv2.imshow("synthesize_batch", view)
cv2.waitKey(WAIT_KEY)
if i >= 3:
break
cv2.destroyAllWindows()
def test_synthesize_batch_multi_scale():
"""
gt depth와 gt pose를 입력했을 때 스케일 별로 복원되는 이미지를 정성적으로 확인
실제 target image와 복원된 "multi" scale target image를 눈으로 비교
"""
print("===== start test_synthesize_batch_multi_scale")
dataset = TfrecordGenerator(op.join(opts.DATAPATH_TFR,
"kitti_raw_test")).get_generator()
for i, features in enumerate(dataset):
print("----- test_synthesize_batch_multi_scale")
stacked_image = features['image']
intrinsic = features['intrinsic']
depth_gt = features['depth_gt']
pose_gt = features['pose_gt']
source_image, target_image = uf.split_into_source_and_target(
stacked_image)
depth_gt_ms = uf.multi_scale_depths(depth_gt, [1, 2, 4, 8])
pred_pose = cp.pose_matr2rvec_batch(pose_gt)
# EXECUTE
synth_target_ms = SynthesizeMultiScale()(source_image, intrinsic,
depth_gt_ms, pred_pose)
# compare target image and reconstructed images
# recon_img0[0, 0]: reconstructed from the first image
target_image = uf.to_uint8_image(target_image).numpy()[0]
source_image = uf.to_uint8_image(source_image).numpy()[
0, 0:opts.IM_HEIGHT]
recon_img0 = uf.to_uint8_image(synth_target_ms[0]).numpy()[0, 0]
recon_img1 = uf.to_uint8_image(synth_target_ms[2]).numpy()[0, 0]
recon_img1 = cv2.resize(recon_img1, (opts.IM_WIDTH, opts.IM_HEIGHT),
cv2.INTER_NEAREST)
view = np.concatenate(
[source_image, target_image, recon_img0, recon_img1], axis=0)
print("Check if all the images are the same")
cv2.imshow("source, target, and reconstructed", view)
cv2.waitKey(WAIT_KEY)
if i >= 3:
break
cv2.destroyAllWindows()
print("!!! test_synthesize_batch_multi_scale passed")
def test_photometric_loss_quantity(suffix=""):
"""
gt depth와 gt pose를 입력했을 때 나오는 photometric loss와
gt pose에 노이즈를 추가하여 나오는 photometric loss를 비교
두 가지 pose로 복원된 영상을 눈으로 확인하고 gt 데이터의 loss가 더 낮음을 확인 (assert)
"""
print("\n===== start test_photometric_loss_quantity")
dataset = TfrecordGenerator(op.join(opts.DATAPATH_TFR,
"kitti_raw_test")).get_generator()
for i, features in enumerate(dataset):
print("\n--- fetch a batch data")
stacked_image = features["image" + suffix]
intrinsic = features["intrinsic" + suffix]
depth_gt = features["depth_gt" + suffix]
pose_gt = features["pose_gt" + suffix]
source_image, target_image = uf.split_into_source_and_target(
stacked_image)
depth_gt_ms = uf.multi_scale_depths(depth_gt, [1, 2, 4, 8])
pose_gt = cp.pose_matr2rvec_batch(pose_gt)
target_ms = uf.multi_scale_like(target_image, depth_gt_ms)
# EXECUTE
batch_loss_right, scale_loss_right, recon_image_right = \
test_photo_loss(source_image, intrinsic, depth_gt_ms, pose_gt, target_ms)
print("\ncorrupt poses")
pose_gt = pose_gt.numpy()
pose_gt = pose_gt + np.random.uniform(-0.2, 0.2, pose_gt.shape)
pose_gt = tf.constant(pose_gt, dtype=tf.float32)
# EXECUTE
batch_loss_wrong, scale_loss_wrong, recon_image_wrong = \
test_photo_loss(source_image, intrinsic, depth_gt_ms, pose_gt, target_ms)
# TEST
print("loss diff: wrong - right =",
batch_loss_wrong - batch_loss_right)
# Due to randomness, allow minority of frames to fail to the test
assert (np.sum(batch_loss_right.numpy() < batch_loss_wrong.numpy()) >
opts.BATCH_SIZE // 4)
assert (np.sum(scale_loss_right.numpy() < scale_loss_wrong.numpy()) >
opts.BATCH_SIZE // 4)
target = uf.to_uint8_image(target_image).numpy()[0]
view = np.concatenate([target, recon_image_right, recon_image_wrong],
axis=0)
cv2.imshow("pose corruption", view)
cv2.waitKey(WAIT_KEY)
if i > 3:
break
cv2.destroyAllWindows()
print("!!! test_photometric_loss_quantity passed")
def test_reshape_source_images():
"""
위 아래로 쌓인 원본 이미지를 batch 아래 한 차원을 더 만들어서 reshape이 잘 됐는지 확인(assert)
"""
print("===== start test_reshape_source_images")
dataset = TfrecordGenerator(op.join(opts.DATAPATH_TFR,
"kitti_raw_test")).get_generator()
dataset = iter(dataset)
features = next(dataset)
stacked_image = features['image']
source_image, target_image = uf.split_into_source_and_target(stacked_image)
print("batch source image shape", source_image.shape)
# create synthesizer
batch, height, width, _ = target_image.get_shape().as_list()
synthesizer = SynthesizeBatchBasic(
(batch, int(height / 2), int(width / 2)), 4, 2)
# EXECUTE
reshaped_image = synthesizer.reshape_source_images(source_image)
print("reorganized source image shape",
reshaped_image.get_shape().as_list())
reshaped_image = uf.to_uint8_image(reshaped_image).numpy()
imgidx = 2
scsize = (int(opts.IM_HEIGHT / 2), int(opts.IM_WIDTH / 2))
scaled_image = tf.image.resize(source_image,
size=(scsize[0] * 4, scsize[1]),
method="bilinear")
scaled_image = uf.to_uint8_image(scaled_image).numpy()
scaled_image = scaled_image[0, scsize[0] * imgidx:scsize[0] * (imgidx + 1)]
# compare second image in the stacked images
assert np.isclose(scaled_image, reshaped_image[0, imgidx]).all()
view = np.concatenate([scaled_image, reshaped_image[0, 1]], axis=0)
cv2.imshow("original and reshaped", view)
cv2.waitKey(WAIT_KEY)
print("!!! test_reshape_source_images passed")
cv2.destroyAllWindows()
def test_photo_loss(source_image, intrinsic, depth_gt_ms, pose_gt, target_ms):
synth_target_ms = SynthesizeMultiScale()(source_image, intrinsic,
depth_gt_ms, pose_gt)
losses = []
recon_image = 0
for scale, synt_target, orig_target in zip([1, 2, 4, 8], synth_target_ms,
target_ms):
# EXECUTE
loss = ls.photometric_loss_l1(synt_target, orig_target)
losses.append(loss)
if scale == 1:
recon_target = uf.to_uint8_image(synt_target).numpy()
recon_image = cv2.resize(recon_target[0, 0],
(opts.IM_WIDTH, opts.IM_HEIGHT),
interpolation=cv2.INTER_NEAREST)
losses = tf.stack(losses, axis=2) # [batch, num_src, num_scales]
batch_loss = tf.reduce_sum(losses, axis=[1, 2])
scale_loss = tf.reduce_sum(losses, axis=[0, 1])
print("all photometric loss:", losses)
print("batch mean photometric loss:", batch_loss)
print("scale mean photometric loss:", scale_loss)
return batch_loss, scale_loss, recon_image
def test_photometric_loss_quality(suffix=""):
"""
gt depth와 gt pose를 입력했을 때 스케일 별로 복원되는 이미지를 정성적으로 확인하고
복원된 이미지로부터 계산되는 photometric loss를 확인
assert 없음
"""
print("\n===== start test_photometric_loss_quality")
dataset = TfrecordGenerator(op.join(opts.DATAPATH_TFR,
"kitti_raw_test")).get_generator()
for i, features in enumerate(dataset):
print("\n--- fetch a batch data")
stacked_image = features["image" + suffix]
intrinsic = features["intrinsic" + suffix]
depth_gt = features["depth_gt" + suffix]
pose_gt = features["pose_gt" + suffix]
# identity pose results in NaN data
pose_gt_np = pose_gt.numpy()
for pose_seq in pose_gt_np:
for pose in pose_seq:
assert not np.isclose(np.identity(4, dtype=np.float),
pose).all()
source_image, target_image = uf.split_into_source_and_target(
stacked_image)
depth_gt_ms = uf.multi_scale_depths(depth_gt, [1, 2, 4, 8])
pose_gt = cp.pose_matr2rvec_batch(pose_gt)
target_ms = uf.multi_scale_like(target_image, depth_gt_ms)
batch, height, width, _ = target_image.get_shape().as_list()
synth_target_ms = SynthesizeMultiScale()(source_image, intrinsic,
depth_gt_ms, pose_gt)
srcimgs = uf.to_uint8_image(source_image).numpy()[0]
srcimg0 = srcimgs[0:height]
srcimg3 = srcimgs[height * 3:height * 4]
losses = []
for scale, synt_target, orig_target in zip([1, 2, 4, 8],
synth_target_ms, target_ms):
# EXECUTE
loss = ls.photometric_loss_l1(synt_target, orig_target)
losses.append(loss)
recon_target = uf.to_uint8_image(synt_target).numpy()
recon0 = cv2.resize(recon_target[0, 0], (width, height),
interpolation=cv2.INTER_NEAREST)
recon3 = cv2.resize(recon_target[0, 3], (width, height),
interpolation=cv2.INTER_NEAREST)
target = uf.to_uint8_image(orig_target).numpy()[0]
target = cv2.resize(target, (width, height),
interpolation=cv2.INTER_NEAREST)
view = np.concatenate([target, srcimg0, recon0, srcimg3, recon3],
axis=0)
print(f"1/{scale} scale, photo loss:", tf.reduce_sum(loss, axis=1))
cv2.imshow("photo loss", view)
cv2.waitKey(WAIT_KEY)
losses = tf.stack(losses, axis=2) # [batch, num_src, num_scales]
print("all photometric loss:", tf.reduce_sum(losses, axis=1))
print("batch mean photometric loss:", tf.reduce_sum(losses,
axis=[1, 2]))
print("scale mean photometric loss:", tf.reduce_sum(losses,
axis=[0, 1]))
if i > 3:
break
cv2.destroyAllWindows()
print("!!! test_photometric_loss_quality passed")