def process_frame(params):
image0_path, image1_path, image2_path, \
sparse_depth_path, semi_dense_depth_path = params
# Read images and concatenate together
image0 = cv2.imread(image0_path)
image1 = cv2.imread(image1_path)
image2 = cv2.imread(image2_path)
image = np.concatenate([image1, image0, image2], axis=1)
sz, vm = data_utils.load_depth_with_validity_map(sparse_depth_path)
iz = data_utils.interpolate_depth(sz, vm)
# Create validity map and image output path
interp_depth_output_path = sparse_depth_path \
.replace(KITTI_DEPTH_COMPLETION_DIRPATH, KITTI_DEPTH_COMPLETION_OUTPUT_DIRPATH) \
.replace('sparse_depth', 'interp_depth')
validity_map_output_path = sparse_depth_path \
.replace(KITTI_DEPTH_COMPLETION_DIRPATH, KITTI_DEPTH_COMPLETION_OUTPUT_DIRPATH) \
.replace('sparse_depth', 'validity_map')
image_output_path = validity_map_output_path \
.replace(os.path.join(os.sep+'proj_depth', 'velodyne_raw'), '') \
.replace('validity_map', 'image')
# Create output directories
for output_path in [
image_output_path, interp_depth_output_path,
validity_map_output_path
]:
output_dirpath = os.path.dirname(output_path)
if not os.path.exists(output_dirpath):
try:
os.makedirs(output_dirpath)
except FileExistsError:
pass
# Write to disk
data_utils.save_depth(iz, interp_depth_output_path)
data_utils.save_validity_map(vm, validity_map_output_path)
cv2.imwrite(image_output_path, image)
return (image_output_path, sparse_depth_path, interp_depth_output_path,
validity_map_output_path, semi_dense_depth_path)
def process_frame(args):
image_path1, image_path0, image_path2, \
sparse_depth_path, validity_map_path, ground_truth_path = args
# Create image composite of triplets
im1 = cv2.imread(image_path1)
im0 = cv2.imread(image_path0)
im2 = cv2.imread(image_path2)
imc = np.concatenate([im1, im0, im2], axis=1)
# Create interpolated depth
sz, vm = data_utils.load_depth_with_validity_map(sparse_depth_path)
iz = data_utils.interpolate_depth(sz, vm)
image_ref_path = os.path.join(*image_path0.split(os.sep)[2:])
sparse_depth_ref_path = os.path.join(*sparse_depth_path.split(os.sep)[2:])
# Set output paths
image_output_path = os.path.join(VOID_OUT_DIRPATH, image_ref_path)
sparse_depth_output_path = sparse_depth_path
interp_depth_output_path = os.path.join(VOID_OUT_DIRPATH, sparse_depth_ref_path) \
.replace('sparse_depth', 'interp_depth')
validity_map_output_path = validity_map_path
ground_truth_output_path = ground_truth_path
# Verify that all filenames match
image_out_dirpath, image_filename = os.path.split(image_output_path)
sparse_depth_filename = os.path.basename(sparse_depth_output_path)
validity_map_filename = os.path.basename(validity_map_output_path)
ground_truth_filename = os.path.basename(ground_truth_output_path)
assert (image_filename == sparse_depth_filename)
assert (image_filename == validity_map_filename)
assert (image_filename == ground_truth_filename)
# Write to disk
cv2.imwrite(image_output_path, imc)
data_utils.save_depth(iz, interp_depth_output_path)
return (image_ref_path, image_output_path, sparse_depth_output_path,
interp_depth_output_path, validity_map_output_path,
ground_truth_output_path)
def process_frame(args):
sparse_depth_path, validity_map_path = args
# Create interpolated depth
sz, vm = data_utils.load_depth_with_validity_map(sparse_depth_path)
iz = data_utils.interpolate_depth(sz, vm)
sparse_depth_ref_path = os.path.join(*sparse_depth_path.split(os.sep)[5:])
# Set output paths
sparse_depth_output_path = sparse_depth_path
interp_depth_output_path = os.path.join(VOID_OUT_DIRPATH, sparse_depth_ref_path) \
.replace('sparse_depth', 'interp_depth')
validity_map_output_path = validity_map_path
sparse_depth_filename = os.path.basename(sparse_depth_output_path)
validity_map_filename = os.path.basename(validity_map_output_path)
# Write to disk
data_utils.save_depth(iz, interp_depth_output_path)
return (sparse_depth_ref_path, sparse_depth_output_path, interp_depth_output_path, validity_map_output_path)
def process_frame(inputs):
'''
Processes a single frame
Args:
inputs : tuple
image path, ground truth path
Returns:
str : output sparse depth path
str : output validity map path
str : output semi-dense depth (convex hull of sparse points) path
str : output ground truth path
'''
image_path, ground_truth_path = inputs
# Load image (for corner detection) to generate validity map
image = cv2.resize(cv2.imread(image_path), (args.n_width, args.n_height))
image = np.float32(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))
if args.sparse_depth_distro_type == 'corner':
# Run Harris corner detector
corners = cv2.cornerHarris(image, blockSize=5, ksize=3, k=0.04)
# Extract specified corner locations:
corners = corners.ravel()
corner_locs = np.argsort(corners)[0:N_INIT_CORNER]
corner_map = np.zeros_like(corners)
corner_map[corner_locs] = 1
corner_locs = np.unravel_index(corner_locs,
(image.shape[0], image.shape[1]))
corner_locs = np.transpose(np.array([corner_locs[0], corner_locs[1]]))
kmeans = MiniBatchKMeans(n_clusters=args.n_points,
max_iter=2,
n_init=1,
init_size=None,
random_state=RANDOM_SEED,
reassignment_ratio=1e-11)
kmeans.fit(corner_locs)
# k-Means means as corners
corner_locs = kmeans.cluster_centers_.astype(np.uint16)
validity_map = np.zeros_like(image).astype(np.int16)
validity_map[corner_locs[:, 0], corner_locs[:, 1]] = 1
elif args.sparse_depth_distro_type == 'uniform':
indices = \
np.array([[h, w] for h in range(args.n_height) for w in range(args.n_width)])
selected_indices = \
np.random.permutation(range(args.n_height * args.n_width))[0:args.n_points]
selected_indices = indices[selected_indices]
validity_map = np.zeros_like(image).astype(np.int16)
validity_map[selected_indices[:, 0], selected_indices[:, 1]] = 1.0
ground_truth = data_utils.load_depth(ground_truth_path, multiplier=448)
ground_truth = cv2.resize(ground_truth, (args.n_width, args.n_height),
interpolation=cv2.INTER_NEAREST)
sparse_depth = validity_map * ground_truth
semi_dense_depth = ground_truth * np.where(
skmorph.convex_hull_image(validity_map), 1, 0)
# Shape check
error_flag = False
if np.squeeze(sparse_depth).shape != (args.n_height, args.n_width):
error_flag = True
print(
'FAILED: sparse depth height and width do not match specified values'
)
if np.squeeze(semi_dense_depth).shape != (args.n_height, args.n_width):
error_flag = True
print(
'FAILED: semi dense depth height and width do not match specified values'
)
# Validity map check
if not np.array_equal(np.unique(validity_map), np.array([0, 1])):
error_flag = True
print('FAILED: validity map contains values ({}) other than 0 or 1'.
format(np.unique(validity_map)))
if np.sum(validity_map) < args.min_points:
error_flag = True
print(
'FAILED: validity map contains fewer ({}) points than miniumum point threshold'
.format(np.sum(validity_map)))
# Depth value check
if np.min(ground_truth) < 0.0 or np.max(ground_truth) > 256.0:
error_flag = True
print('FAILED: min ground truth value less than 0 or greater than 256')
if np.sum(np.where(sparse_depth > 0.0, 1.0, 0.0)) < args.min_points:
error_flag = True
print(
'FAILED: valid sparse depth is less than minimum point threshold',
np.sum(np.where(sparse_depth > 0.0, 1.0, 0.0)))
if np.sum(np.where(semi_dense_depth > 0.0, 1.0, 0.0)) < args.min_points:
error_flag = True
print(
'FAILED: valid semi dense depth is less than minimum point threshold',
np.sum(np.where(semi_dense_depth > 0.0, 1.0, 0.0)))
if np.sum(np.where(ground_truth > 0.0, 1.0, 0.0)) < args.min_points:
error_flag = True
print(
'FAILED: valid ground truth is less than minimum point threshold',
np.sum(np.where(ground_truth > 0.0, 1.0, 0.0)))
# NaN check
if np.any(np.isnan(sparse_depth)):
error_flag = True
print('FAILED: found NaN in sparse depth')
if np.any(np.isnan(semi_dense_depth)):
error_flag = True
print('FAILED: found NaN in semi dense depth')
if np.any(np.isnan(ground_truth)):
error_flag = True
print('FAILED: found NaN in sparse depth or semi dense depth')
if not error_flag:
# Generate paths
derived_ground_truth_path = ground_truth_path \
.replace(SCENENET_ROOT_DIRPATH, SCENENET_OUT_DIRPATH) \
.replace('depth', 'ground_truth')
derived_sparse_depth_path = ground_truth_path \
.replace(SCENENET_ROOT_DIRPATH, SCENENET_OUT_DIRPATH) \
.replace('depth', 'sparse_depth')
derived_validity_map_path = ground_truth_path \
.replace(SCENENET_ROOT_DIRPATH, SCENENET_OUT_DIRPATH) \
.replace('depth', 'validity_map')
derived_semi_dense_depth_path = ground_truth_path \
.replace(SCENENET_ROOT_DIRPATH, SCENENET_OUT_DIRPATH) \
.replace('depth', 'semi_dense_depth')
# Write to file
data_utils.save_validity_map(validity_map, derived_validity_map_path)
data_utils.save_depth(sparse_depth, derived_sparse_depth_path)
data_utils.save_depth(semi_dense_depth, derived_semi_dense_depth_path)
data_utils.save_depth(ground_truth, derived_ground_truth_path)
else:
print('Found error in {}'.format(ground_truth_path))
derived_ground_truth_path = 'error'
derived_sparse_depth_path = 'error'
derived_validity_map_path = 'error'
derived_semi_dense_depth_path = 'error'
return (derived_sparse_depth_path, derived_validity_map_path,
derived_semi_dense_depth_path, derived_ground_truth_path)
log(
'Total time: {:.2f} min Average time per sample: {:.2f} ms'.format(
time_elapse / 60.0, time_elapse_per_sample * 1000.0), log_path)
if ground_truth_available:
# Run evaluation metrics
eval_utils.evaluate(output_depths,
ground_truths,
log_path=log_path,
min_evaluate_depth=args.min_evaluate_depth,
max_evaluate_depth=args.max_evaluate_depth)
# Store output depth as images
if args.save_outputs:
output_dirpath = os.path.join(args.output_path, 'saved')
print('Storing output depth as PNG into {}'.format(output_dirpath))
if not os.path.exists(output_dirpath):
os.makedirs(output_dirpath)
for idx in range(n_sample):
output_depth = np.squeeze(output_depths[idx, ...])
if args.keep_input_filenames:
filename = os.path.basename(sparse_depth_paths[idx])
else:
filename = '{:010d}.png'.format(idx)
output_path = os.path.join(output_dirpath, filename)
data_utils.save_depth(output_depth, output_path)
def process_frame(inputs):
'''
Processes a single depth frame
Args:
inputs : tuple
KITTI sparse depth path,
Virtual KITTI ground truth path,
output directory paths in order of:
sparse depth, validity map, semi-dense depth, dense depth, groundtruth
Returns:
str : Virtual KITTI output sparse depth path
str : Virtual KITTI output validity map path
str : Virtual KITTI output semi-dense depth (convex hull of sparse points) path
str : Virtual KITTI output dense depth path (ground truth without sky)
str : Virtual KITTI output ground truth path
'''
# Separate arguments into individual variables
kitti_sparse_depth_path, vkitti_ground_truth_path, output_dirpaths = inputs
# Extract validity map from KITTI sparse depth
_, kitti_validity_map = data_utils.load_depth_with_validity_map(kitti_sparse_depth_path)
# Load Virtual KITTI ground truth
vkitti_ground_truth = \
cv2.imread(vkitti_ground_truth_path, cv2.IMREAD_ANYCOLOR | cv2.IMREAD_ANYDEPTH)
# Convert Virtual KITTI ground truth to meters
vkitti_ground_truth = vkitti_ground_truth / 100.0
if kitti_validity_map.shape != vkitti_ground_truth.shape:
# Resize KITTI validity map to VKITTI size
kitti_validity_map = cv2.resize(
kitti_validity_map,
dsize=(vkitti_ground_truth.shape[1], vkitti_ground_truth.shape[0]),
interpolation=cv2.INTER_NEAREST)
assert(np.all(np.unique(kitti_validity_map) == [0, 1]))
# Get Virtual KITTI dense depth without sky
vkitti_validity_map = np.ones(vkitti_ground_truth.shape)
vkitti_validity_map[vkitti_ground_truth > 600.0] = 0.0
vkitti_dense_depth = vkitti_validity_map * vkitti_ground_truth
# Get Virtual KITTI sparse depth
vkitti_sparse_depth = kitti_validity_map * vkitti_dense_depth
# Get Virtual KITTI semi-dense depth (convex hull of sparse points)
vkitti_semi_dense_depth = \
np.where(skmorph.convex_hull_image(kitti_validity_map), 1, 0) * vkitti_dense_depth
# Create output filepaths
filename = os.path.basename(vkitti_ground_truth_path)
output_sparse_depth_dirpath, \
output_validity_map_dirpath, \
output_semi_dense_depth_dirpath, \
output_dense_depth_dirpath, \
output_ground_truth_dirpath = output_dirpaths
output_sparse_depth_path = os.path.join(output_sparse_depth_dirpath, filename)
output_validity_map_path = os.path.join(output_validity_map_dirpath, filename)
output_semi_dense_depth_path = os.path.join(output_semi_dense_depth_dirpath, filename)
output_dense_depth_path = os.path.join(output_dense_depth_dirpath, filename)
output_ground_truth_path = os.path.join(output_ground_truth_dirpath, filename)
# Write to disk
data_utils.save_depth(vkitti_sparse_depth, output_sparse_depth_path)
data_utils.save_validity_map(kitti_validity_map, output_validity_map_path)
data_utils.save_depth(vkitti_semi_dense_depth, output_semi_dense_depth_path)
data_utils.save_depth(vkitti_dense_depth, output_dense_depth_path)
data_utils.save_depth(vkitti_ground_truth, output_ground_truth_path)
return (output_sparse_depth_path,
output_validity_map_path,
output_semi_dense_depth_path,
output_dense_depth_path,
output_ground_truth_path)
.replace('sparse_depth', 'interp_depth')
validity_map_output_path = path \
.replace('kitti_depth_completion', 'kitti_depth_completion_voiced') \
.replace('sparse_depth', 'validity_map')
sparse_depth_output_paths.append(path)
interp_depth_output_paths.append(interp_depth_output_path)
validity_map_output_paths.append(validity_map_output_path)
for output_path in [
interp_depth_output_path, validity_map_output_path
]:
output_dirpath = os.path.dirname(output_path)
if not os.path.exists(output_dirpath):
os.makedirs(output_dirpath)
# Write to disk
data_utils.save_depth(iz, interp_depth_output_path)
data_utils.save_validity_map(vm, validity_map_output_path)
elif refdir == 'ground_truth':
semi_dense_depth_output_paths.append(path)
sys.stdout.write('Processed {}/{} {} {} samples \r'.format(
idx + 1, len(filepaths), mode_type, refdir))
sys.stdout.flush()
print('Completed generating {} {} {} samples'.format(
len(filepaths), mode_type, refdir))
print('Storing validation image file paths into: %s' %
VAL_IMAGE_OUTPUT_FILEPATH)
with open(VAL_IMAGE_OUTPUT_FILEPATH, "w") as o: