def _compute_pair_reconstructability(args):
log.setup()
im1, im2, p1, p2, camera1, camera2, threshold = args
R, inliers = two_view_reconstruction_rotation_only(
p1, p2, camera1, camera2, threshold)
r = pairwise_reconstructability(len(p1), len(inliers))
return (im1, im2, r)
def prune_depthmap(arguments):
"""Prune depthmap to remove redundant points."""
log.setup()
data: UndistortedDataSet = arguments[0]
neighbors = arguments[1]
shot = arguments[2]
if data.pruned_depthmap_exists(shot.id):
logger.info("Using precomputed pruned depthmap {}".format(shot.id))
return
logger.info("Pruning depthmap for image {}".format(shot.id))
dp = pydense.DepthmapPruner()
dp.set_same_depth_threshold(data.config["depthmap_same_depth_threshold"])
add_views_to_depth_pruner(data, neighbors, dp)
points, normals, colors, labels, detections = dp.prune()
# Save and display results
data.save_pruned_depthmap(shot.id, points, normals, colors, labels,
detections)
if data.config["depthmap_save_debug_files"]:
data.save_point_cloud(points, normals, colors, labels, detections,
"pruned.npz.ply")
def prune_depthmap(arguments):
"""Prune depthmap to remove redundant points."""
log.setup()
data, udata, neighbors, shot = arguments
# if udata.pruned_depthmap_exists(shot.id):
# logger.info("Using precomputed pruned depthmap {}".format(shot.id))
# return
logger.info("Pruning depthmap for image {}".format(shot.id))
dp = pydense.DepthmapPruner()
dp.set_same_depth_threshold(udata.config['depthmap_same_depth_threshold'])
add_views_to_depth_pruner(data,udata, neighbors, dp)
points, normals, colors, labels, detections = dp.prune()
# Save and display results
udata.save_pruned_depthmap(shot.id, points, normals, colors, labels, detections)
depthmap={}
depthmap.update({'points':points})
depthmap.update({'normals':normals})
depthmap.update({'colors':colors})
depthmap.update({'labels':labels})
depthmap.update({'detections':detections})
data.save_pruned_depthmap(shot.id, depthmap)
#pruned_depthmap.update({shot.id:depthmap})
if udata.config['depthmap_save_debug_files']:
ply_line=pruned_point_cloud_to_ply(points, normals, colors, labels, detections)
data.save_ply_line(shot.id, ply_line)
#pruned_ply.update({shot.id:ply_line})
with io.open_wt(udata._depthmap_file(shot.id, 'pruned.npz.ply')) as fp:
ply_line=point_cloud_to_ply(points, normals, colors, labels, detections, fp)
def match_unwrap_args(args):
"""Wrapper for parallel processing of pair matching.
Compute all pair matchings of a given image and save them.
"""
log.setup()
im1, candidates, ctx = args
print(args)
im1_matches = {}
p1, f1, _ = feature_loader.instance.load_points_features_colors(
ctx.data, im1)
camera1 = ctx.cameras[ctx.exifs[im1]['camera']]
for im2 in candidates:
p2, f2, _ = feature_loader.instance.load_points_features_colors(
ctx.data, im2)
camera2 = ctx.cameras[ctx.exifs[im2]['camera']]
im1_matches[im2] = match(im1, im2, camera1, camera2, ctx.data)
num_matches = sum(1 for m in im1_matches.values() if len(m) > 0)
logger.debug('Image {} matches: {} out of {}'.format(
im1, num_matches, len(candidates)))
return im1, im1_matches
def match_unwrap_args(
args: Tuple[str, str, Dict[str, pygeometry.Camera], Dict[str, Any],
DataSetBase, Dict[str, Any], Optional[Dict[str,
pygeometry.Pose]], ]
) -> Tuple[str, str, np.ndarray]:
"""Wrapper for parallel processing of pair matching.
Compute all pair matchings of a given image and save them.
"""
log.setup()
im1 = args[0]
im2 = args[1]
cameras = args[2]
exifs = args[3]
data: DataSetBase = args[4]
config_override = args[5]
poses = args[6]
if poses:
pose1 = poses[im1]
pose2 = poses[im2]
pose = pose2.relative_to(pose1)
else:
pose = None
camera1 = cameras[exifs[im1]["camera"]]
camera2 = cameras[exifs[im2]["camera"]]
matches = match(im1, im2, camera1, camera2, data, config_override, pose)
return im1, im2, matches
def clean_depthmap(arguments):
"""Clean depthmap by checking consistency with neighbors."""
log.setup()
data, neighbors, shot = arguments
if data.clean_depthmap_exists(shot.id):
logger.info("Using precomputed clean depthmap {}".format(shot.id))
return
logger.info("Cleaning depthmap for image {}".format(shot.id))
dc = csfm.DepthmapCleaner()
dc.set_same_depth_threshold(data.config['depthmap_same_depth_threshold'])
dc.set_min_consistent_views(data.config['depthmap_min_consistent_views'])
add_views_to_depth_cleaner(data, neighbors, dc)
depth = dc.clean()
# Save and display results
raw_depth, raw_plane, raw_score, raw_nghbr, nghbrs = data.load_raw_depthmap(
shot.id)
data.save_clean_depthmap(shot.id, depth, raw_plane, raw_score)
if data.config['depthmap_save_debug_files']:
image = data.load_undistorted_image(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with io.open_wt(data._depthmap_file(shot.id, 'clean.npz.ply')) as fout:
fout.write(ply)
def undistort_image(arguments):
log.setup()
shot, undistorted_shots, data = arguments
logger.debug('Undistorting image {}'.format(shot.id))
if shot.camera.projection_type == 'perspective':
image = data.image_as_array(shot.id)
undistorted = undistort_perspective_image(image, shot.camera)
data.save_undistorted_image(shot.id, undistorted)
elif shot.camera.projection_type == 'brown':
image = data.image_as_array(shot.id)
new_camera = undistorted_shots[0].camera
undistorted = undistort_brown_image(image, shot.camera, new_camera)
data.save_undistorted_image(shot.id, undistorted)
elif shot.camera.projection_type == 'fisheye':
image = data.image_as_array(shot.id)
undistorted = undistort_fisheye_image(image, shot.camera)
data.save_undistorted_image(shot.id, undistorted)
elif shot.camera.projection_type in ['equirectangular', 'spherical']:
original = data.image_as_array(shot.id)
subshot_width = int(data.config['depthmap_resolution'])
width = 4 * subshot_width
height = width / 2
image = cv2.resize(original, (width, height),
interpolation=cv2.INTER_AREA)
for subshot in undistorted_shots:
undistorted = render_perspective_view_of_a_panorama(
image, shot, subshot)
data.save_undistorted_image(subshot.id, undistorted)
else:
raise NotImplementedError(
'Undistort not implemented for projection type: {}'.format(
shot.camera.projection_type))
def undistort_image_and_masks(arguments) -> None:
shot, undistorted_shots, data, udata = arguments
log.setup()
logger.debug("Undistorting image {}".format(shot.id))
max_size = data.config["undistorted_image_max_size"]
# Undistort image
image = data.load_image(shot.id, unchanged=True, anydepth=True)
if image is not None:
undistorted = undistort_image(
shot, undistorted_shots, image, cv2.INTER_AREA, max_size
)
for k, v in undistorted.items():
udata.save_undistorted_image(k, v)
# Undistort mask
mask = data.load_mask(shot.id)
if mask is not None:
undistorted = undistort_image(
shot, undistorted_shots, mask, cv2.INTER_NEAREST, max_size
)
for k, v in undistorted.items():
udata.save_undistorted_mask(k, v)
# Undistort segmentation
segmentation = data.load_segmentation(shot.id)
if segmentation is not None:
undistorted = undistort_image(
shot, undistorted_shots, segmentation, cv2.INTER_NEAREST, max_size
)
for k, v in undistorted.items():
udata.save_undistorted_segmentation(k, v)
def undistort_image(arguments):
log.setup()
shot, undistorted_shots, data = arguments
logger.debug('Undistorting image {}'.format(shot.id))
if shot.camera.projection_type == 'perspective':
image = data.image_as_array(shot.id)
undistorted = undistort_perspective_image(image, shot.camera)
data.save_undistorted_image(shot.id, undistorted)
elif shot.camera.projection_type == 'brown':
image = data.image_as_array(shot.id)
new_camera = undistorted_shots[0].camera
undistorted = undistort_brown_image(image, shot.camera, new_camera)
data.save_undistorted_image(shot.id, undistorted)
elif shot.camera.projection_type == 'fisheye':
image = data.image_as_array(shot.id)
undistorted = undistort_fisheye_image(image, shot.camera)
data.save_undistorted_image(shot.id, undistorted)
elif shot.camera.projection_type in ['equirectangular', 'spherical']:
original = data.image_as_array(shot.id)
subshot_width = int(data.config['depthmap_resolution'])
width = 4 * subshot_width
height = width // 2
image = cv2.resize(original, (width, height), interpolation=cv2.INTER_AREA)
for subshot in undistorted_shots:
undistorted = render_perspective_view_of_a_panorama(
image, shot, subshot)
data.save_undistorted_image(subshot.id, undistorted)
else:
raise NotImplementedError(
'Undistort not implemented for projection type: {}'.format(
shot.camera.projection_type))
def command_runner(all_commands_types, dataset_factory):
""" Main entry point for running the passed SfM commands types."""
log.setup()
# Create the top-level parser
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(help="Command to run",
dest="command",
metavar="command")
command_objects = [c.Command() for c in all_commands_types]
for command in command_objects:
subparser = subparsers.add_parser(command.name, help=command.help)
command.add_arguments(subparser)
# Parse arguments
args = parser.parse_args()
# Instanciate datast
data = dataset_factory(args.dataset)
# Run the selected subcommand
for command in command_objects:
if args.command == command.name:
command.run(data, args)
def compute_depthmap(arguments):
"""Compute depthmap for a single shot."""
log.setup()
data, neighbors, min_depth, max_depth, shot = arguments
method = data.config['depthmap_method']
if data.raw_depthmap_exists(shot.id):
logger.info("Using precomputed raw depthmap {}".format(shot.id))
return
logger.info("Computing depthmap for image {0} with {1}".format(shot.id, method))
de = csfm.DepthmapEstimator()
de.set_depth_range(min_depth, max_depth, 100)
de.set_patchmatch_iterations(data.config['depthmap_patchmatch_iterations'])
de.set_min_patch_sd(data.config['depthmap_min_patch_sd'])
add_views_to_depth_estimator(data, neighbors, de)
if (method == 'BRUTE_FORCE'):
depth, plane, score, nghbr = de.compute_brute_force()
elif (method == 'PATCH_MATCH'):
depth, plane, score, nghbr = de.compute_patch_match()
elif (method == 'PATCH_MATCH_SAMPLE'):
depth, plane, score, nghbr = de.compute_patch_match_sample()
else:
raise ValueError('Unknown depthmap method type ' \
'(must be BRUTE_FORCE, PATCH_MATCH or PATCH_MATCH_SAMPLE)')
good_score = score > data.config['depthmap_min_correlation_score']
depth = depth * (depth < max_depth) * good_score
# Save and display results
neighbor_ids = [i.id for i in neighbors[1:]]
data.save_raw_depthmap(shot.id, depth, plane, score, nghbr, neighbor_ids)
if data.config['depthmap_save_debug_files']:
image = data.undistorted_image_as_array(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with open(data._depthmap_file(shot.id, 'raw.npz.ply'), 'w') as fout:
fout.write(ply)
if data.config.get('interactive'):
import matplotlib.pyplot as plt
plt.figure()
plt.suptitle("Shot: " + shot.id + ", neighbors: " + ', '.join(neighbor_ids))
plt.subplot(2, 3, 1)
plt.imshow(image)
plt.subplot(2, 3, 2)
plt.imshow(color_plane_normals(plane))
plt.subplot(2, 3, 3)
plt.imshow(depth)
plt.colorbar()
plt.subplot(2, 3, 4)
plt.imshow(score)
plt.colorbar()
plt.subplot(2, 3, 5)
plt.imshow(nghbr)
plt.colorbar()
plt.show()
def detect(args):
log.setup()
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
mask = data.mask_as_array(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config.get('preemptive_max', 200)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config, mask)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config.get('matcher_type', 'FLANN') == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
def match_unwrap_args(args):
"""Wrapper for parallel processing of pair matching.
Compute all pair matchings of a given image and save them.
"""
log.setup()
im1 = args[0]
candidates = args[1]
data: DataSetBase = args[2]
config_override = args[3]
cameras = args[4]
exifs = args[5]
im1_matches = {}
camera1 = cameras[exifs[im1]["camera"]]
for im2 in candidates:
camera2 = cameras[exifs[im2]["camera"]]
im1_matches[im2] = match(im1, im2, camera1, camera2, data, config_override)
num_matches = sum(1 for m in im1_matches.values() if len(m) > 0)
logger.debug(
"Image {} matches: {} out of {}".format(im1, num_matches, len(candidates))
)
return im1, im1_matches
def command_runner(all_commands_types: List[Any], dataset_factory: Callable,
dataset_choices: List[str]) -> None:
""" Main entry point for running the passed SfM commands types."""
log.setup()
# Create the top-level parser
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(help="Command to run",
dest="command",
metavar="command")
command_objects = [c.Command() for c in all_commands_types]
for command in command_objects:
subparser = subparsers.add_parser(command.name, help=command.help)
command.add_arguments(subparser)
subparser.add_argument(
"--dataset-type",
type=str,
required=False,
default="opensfm",
choices=dataset_choices,
)
# Parse arguments
args = parser.parse_args()
# Instanciate datast
with dataset_factory(args.dataset, args.dataset_type) as data:
# Run the selected subcommand
for command in command_objects:
if args.command == command.name:
command.run(data, args)
def detect(args):
image, data = args
log.setup()
need_words = data.config[
'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
need_flann = data.config['matcher_type'] == 'FLANN'
has_words = not need_words or data.words_exist(image)
has_flann = not need_flann or data.feature_index_exists(image)
has_features = data.features_exist(image)
if has_features and has_flann and has_words:
logger.info('Skip recomputing {} features for image {}'.format(
data.feature_type().upper(), image))
return
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
data.load_image(image), data.config)
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_flann:
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config['bow_matcher_type'])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def _compute_pair_reconstructability(
args: TPairArguments) -> Tuple[str, str, float]:
log.setup()
im1, im2, p1, p2, camera1, camera2, threshold = args
R, inliers = two_view_reconstruction_rotation_only(p1, p2, camera1,
camera2, threshold)
r = pairwise_reconstructability(len(p1), len(inliers))
return (im1, im2, r)
def undistort_image_and_masks(arguments):
shot, undistorted_shots, opensfm_config, udata, file_path, imageFilter, self_compute, self_path = arguments
log.setup()
logger.debug('Undistorting image {}'.format(shot.id))
# Undistort
image = None
if self_compute:
image = opensfm_interface.load_image(os.path.join(
self_path, 'images', shot.id),
unchanged=True,
anydepth=True)
else:
image = opensfm_interface.load_image(os.path.join(
file_path, 'images', shot.id),
unchanged=True,
anydepth=True)
if image is not None:
if imageFilter is not None:
image = imageFilter(shot.id, image)
max_size = opensfm_config['undistorted_image_max_size']
undistorted = undistort_image(shot, undistorted_shots, image,
cv2.INTER_AREA, max_size)
for k, v in undistorted.items():
print('k value in undistored images: ' + str(k))
udata.save_undistorted_image(k, v)
# Undistort mask
mask_files = {}
mask = opensfm_interface.load_mask(mask_files, shot.id)
if mask is not None:
undistorted = undistort_image(shot, undistorted_shots, mask,
cv2.INTER_NEAREST, 1e9)
for k, v in undistorted.items():
udata.save_undistorted_mask(k, v)
# Undistort segmentation
segmentation = opensfm_interface.load_segmentation(file_path, shot.id)
if segmentation is not None:
undistorted = undistort_image(shot, undistorted_shots, segmentation,
cv2.INTER_NEAREST, 1e9)
for k, v in undistorted.items():
udata.save_undistorted_segmentation(k, v)
# Undistort detections
detection = None
if self_compute:
detection = opensfm_interface.load_detection(self_path, shot.id)
else:
detection = opensfm_interface.load_detection(file_path, shot.id)
if detection is not None:
undistorted = undistort_image(shot, undistorted_shots, detection,
cv2.INTER_NEAREST, 1e9)
for k, v in undistorted.items():
udata.save_undistorted_detection(k, v)
def match(args):
"""Compute all matches for a single image"""
log.setup()
im1, candidates, i, n, ctx = args
logger.info('Matching {} - {} / {}'.format(im1, i + 1, n))
config = ctx.data.config
robust_matching_min_match = config['robust_matching_min_match']
lowes_ratio = config['lowes_ratio']
im1_all_robust_matches = {}
im1_valid_rmatches = {}
im1_T = {}
im1_F = {}
im1_valid_inliers = {}
im1_fmr = ctx.data.load_feature_matching_results(im1)
p1, f1, c1 = ctx.data.load_features(im1)
for im2 in candidates:
# robust matching
t_robust_matching = timer()
camera1 = ctx.cameras[ctx.exifs[im1]['camera']]
camera2 = ctx.cameras[ctx.exifs[im2]['camera']]
unthresholded_matches = im1_fmr[im2]
classified_matches = np.concatenate(( \
np.array(unthresholded_matches['indices1']).reshape((-1,1)),
np.array(unthresholded_matches['indices2']).reshape((-1,1)),
np.array(unthresholded_matches['distances1']).reshape((-1,1)),
np.array(unthresholded_matches['distances2']).reshape((-1,1)),
np.array(unthresholded_matches['scores']).reshape((-1,1))
), axis=1)
lowes_matches_indices = np.where(
(classified_matches[:, 2] <= lowes_ratio)
& (classified_matches[:, 3] <= lowes_ratio))[0]
relevant_feature_matching_indices = np.argsort(
classified_matches[:, 4])[::-1][:len(lowes_matches_indices)]
thresholded_matches = classified_matches[
relevant_feature_matching_indices, :]
p2, f2, c2 = ctx.data.load_features(im2)
rmatches, T, F, validity = classifier.robust_match_fundamental_weighted(
p1, p2, thresholded_matches, config)
im1_all_robust_matches[im2] = thresholded_matches
im1_valid_rmatches[im2] = 1
im1_T[im2] = T.tolist()
im1_F[im2] = F.tolist()
im1_valid_inliers[im2] = validity
logger.debug("Full matching {0} / {1}, time: {2}s".format(
len(rmatches), len(thresholded_matches),
timer() - t_robust_matching))
ctx.data.save_weighted_matches(im1, im1_all_robust_matches)
def match(args):
"""Compute all matches for a single image"""
log.setup()
im1, candidates, i, n, ctx = args
logger.info('Matching {} - {} / {}'.format(im1, i + 1, n))
config = ctx.data.config
matcher_type = config['matcher_type']
robust_matching_min_match = config['robust_matching_min_match']
im1_matches = {}
for im2 in candidates:
# symmetric matching
t = timer()
p1, f1, c1 = ctx.data.load_features(im1)
p2, f2, c2 = ctx.data.load_features(im2)
if matcher_type == 'WORDS':
w1 = ctx.data.load_words(im1)
w2 = ctx.data.load_words(im2)
matches = matching.match_words_symmetric(f1, w1, f2, w2, config)
elif matcher_type == 'FLANN':
i1 = ctx.data.load_feature_index(im1, f1)
i2 = ctx.data.load_feature_index(im2, f2)
matches = matching.match_flann_symmetric(f1, i1, f2, i2, config)
elif matcher_type == 'BRUTEFORCE':
matches = matching.match_brute_force_symmetric(f1, f2, config)
else:
raise ValueError("Invalid matcher_type: {}".format(matcher_type))
logger.debug('{} - {} has {} candidate matches'.format(
im1, im2, len(matches)))
if len(matches) < robust_matching_min_match:
im1_matches[im2] = []
continue
# robust matching
t_robust_matching = timer()
camera1 = ctx.cameras[ctx.exifs[im1]['camera']]
camera2 = ctx.cameras[ctx.exifs[im2]['camera']]
rmatches = matching.robust_match(p1, p2, camera1, camera2, matches,
config)
if len(rmatches) < robust_matching_min_match:
im1_matches[im2] = []
continue
im1_matches[im2] = rmatches
logger.debug('Robust matching time : {0}s'.format(timer() -
t_robust_matching))
logger.debug("Full matching {0} / {1}, time: {2}s".format(
len(rmatches), len(matches),
timer() - t))
ctx.data.save_matches(im1, im1_matches)
def detect(args):
image, data = args
log.setup()
need_words = (data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
logger.info("Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image))
return
logger.info("Extracting {} features for image {}".format(
data.feature_type().upper(), image))
start = timer()
image_array = data.load_image(image)
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image,
image_array))
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config["bow_matcher_type"])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def clean_depthmap(arguments):
"""Clean depthmap by checking consistency with neighbors."""
log.setup()
data, neighbors, shot, file_pathx, self_compute, self_path = arguments
if data.clean_depthmap_exists(shot.id):
logger.info("Using precomputed clean depthmap {}".format(shot.id))
return
logger.info("Cleaning depthmap for image {}".format(shot.id))
dc = csfm.DepthmapCleaner()
dc.set_same_depth_threshold(data.config['depthmap_same_depth_threshold'])
dc.set_min_consistent_views(data.config['depthmap_min_consistent_views'])
add_views_to_depth_cleaner(data, neighbors, dc, self_compute)
depth = dc.clean()
# Save and display results
get_raw = None
if self_compute:
udata = opensfm_interface.UndistortedDataSet(data.path, data.config,
'undistorted')
get_raw = udata.load_raw_depthmap(shot.id)
else:
get_raw = data.load_raw_depthmap(shot.id)
print('here in for data load get raw depth map')
raw_depth, raw_plane, raw_score, raw_nghbr, nghbrs = get_raw
#raw_depth, raw_plane, raw_score, raw_nghbr, nghbrs = data.load_raw_depthmap(shot.id)
data.save_clean_depthmap(shot.id, depth, raw_plane, raw_score)
if data.config['depthmap_save_debug_files']:
image = data.load_undistorted_image(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with io.open_wt(data._depthmap_file(shot.id, 'clean.npz.ply')) as fout:
fout.write(ply)
if data.config.get('interactive'):
import matplotlib.pyplot as plt
plt.figure()
plt.suptitle("Shot: " + shot.id)
plt.subplot(2, 2, 1)
plt.imshow(raw_depth)
plt.colorbar()
plt.subplot(2, 2, 2)
plt.imshow(depth)
plt.colorbar()
plt.show()
def formulate_graph(args):
log.setup()
data, images, scores, criteria, edge_threshold = args
start = timer()
G = nx.Graph()
for i, img1 in enumerate(sorted(images)):
for j, img2 in enumerate(sorted(images)):
if j <= i:
continue
if img1 in scores and img2 in scores[img1]:
if criteria == 'inlier-logp':
inlier_logp = -np.log(scores[img1][img2])
if inlier_logp < -np.log(edge_threshold):
G.add_edge(img1, img2, weight=inlier_logp)
# if img1 == '0000.jpg' and img2 == '0013.jpg':
# print ('{} - {} : {} / {} et: {}'.format(img1, img2, scores[img1][img2], inlier_logp, -np.log(edge_threshold)))
# else:
# logger.info('scores: {} / {} et: {}'.format(scores[img1][img2], inlier_logp, edge_threshold))
elif 'cost' in criteria:
if scores[img1][img2] < edge_threshold:
G.add_edge(img1, img2, weight=scores[img1][img2])
else:
if scores[img1][img2] >= edge_threshold:
G.add_edge(img1, img2, weight=scores[img1][img2])
try:
pagerank = nx.pagerank(G, alpha=0.9)
except:
pagerank = {}
for n in G.nodes():
pagerank[n] = 1.0
lcc = nx.clustering(G, nodes=G.nodes(), weight='weight')
for n in G.nodes():
G.node[n]['pagerank'] = pagerank[n]
G.node[n]['lcc'] = lcc[n]
end = timer()
report = {
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), 'similarity-graphs.json')
# G = nx.minimum_spanning_tree(G)
# i1 = sorted(images)[0]
# i2 = sorted(images)[-1]
# G.add_edge(i1, i2, weight=scores[i1][i2])
return G
def match_unwrap_args(args):
"""Wrapper for parallel processing of pair matching.
Compute all pair matchings of a given image and save them.
"""
log.setup()
im1 = args[0]
im2 = args[1]
cameras = args[2]
exifs = args[3]
data: DataSetBase = args[4]
config_override = args[5]
camera1 = cameras[exifs[im1]["camera"]]
camera2 = cameras[exifs[im2]["camera"]]
matches = match(im1, im2, camera1, camera2, data, config_override)
return im1, im2, matches
def remove(args):
log.setup()
idx, mkv_dirs = args
# TESTING - only correct 10 images
#if idx > 10:
#return
img_set = []
for mkv_dir in mkv_dirs:
img_set.append(os.path.join(mkv_dir, _int_to_shot_id(idx)))
banding_cnt = 0
for img_file in img_set:
freq = csfm.is_banding_present(img_file)
if freq != -1:
logger.info('{}: {}Hz banding'.format(idx, freq))
banding_cnt += 1
else:
logger.info('{}: no banding'.format(idx))
logger.info('{}: banding found in {}/4 images'.format(_int_to_shot_id(idx), banding_cnt))
if banding_cnt >= 1:
logger.info('{}: removing banding...'.format(idx))
for img_file in img_set:
img_original = cv.imread(img_file)
#csfm.run_notch_filter()
##img_corrected = gamma_correction(img_original)
#img_corrected = basic_linear_transform(img_original)
img_corrected = horizontal_banding_removal(img_original)
# high intensity pixels' color is distorted by the filtering process above.
# so we replace them by their original values
grayscaled = cv.cvtColor(img_original, cv.COLOR_BGR2GRAY)
retval, mask = cv.threshold(grayscaled, 220, 255, cv.THRESH_BINARY)
img_corrected[mask == 255] = img_original[mask == 255]
# there is a 72x64 pixel area in each raw image that appears to be 2D barcode
# used by the NC Tech Immersive Studio. don't know if the size of this area
# changes when image resolution changes (currently 3968x3008), so need to keep
# an eye on it if/when we switch camera resolution
img_corrected[0:64, 0:72] = img_original[0:64, 0:72]
cv.imwrite(img_file, img_corrected)
def detect(feature_path, image_path, image, opensfm_config):
log.setup()
need_words = opensfm_config['matcher_type'] == 'WORDS' or opensfm_config[
'matching_bow_neighbors'] > 0
#has_words = not need_words or data.words_exist(image)
#has_features = data.features_exist(image)
# if has_features and has_words:
# logger.info('Skip recomputing {} features for image {}'.format(
# data.feature_type().upper(), image))
# return
#logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image))
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
load_image(image_path), opensfm_config)
#p_unmasked is points
mask_files = defaultdict(lambda: None)
fmask = load_features_mask(feature_path, image, image_path, p_unmasked,
mask_files, opensfm_config)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
#logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
save_features(feature_path, opensfm_config, image, p_sorted, f_sorted,
c_sorted)
if need_words:
bows = bow.load_bows(opensfm_config)
n_closest = opensfm_config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
opensfm_config['bow_matcher_type'])
save_words(feature_path, image_path, closest_words)
def vt_rankings(args):
log.setup()
images, data = args
libvot = data.config['libvot']
start = timer()
subprocess.Popen("ls -d {}/images/* > {}/vt_image_list.txt".format(data.data_path, data.data_path), shell=True, stdout=subprocess.PIPE).stdout.read()
subprocess.Popen("{}/build/bin/libvot_feature -thread_num 10 -output_folder {}/sift/ {}/vt_image_list.txt".format(libvot, data.data_path, data.data_path), shell=True, stdout=subprocess.PIPE).stdout.read()
subprocess.Popen("ls -d {}/sift/*.sift > {}/vt_sift_list.txt".format(data.data_path, data.data_path), shell=True, stdout=subprocess.PIPE).stdout.read()
subprocess.Popen("{}/build/bin/image_search {}/vt_sift_list.txt {}/vocab_out".format(libvot, data.data_path, data.data_path), shell=True, stdout=subprocess.PIPE).stdout.read()
end = timer()
report = {
"wall_time": end - start,
}
data.save_report(io.json_dumps(report),
'rankings.json')
def compute_depthmap(arguments):
"""Compute depthmap for a single shot."""
log.setup()
data, neighbors, min_depth, max_depth, shot = arguments
method = data.config['depthmap_method']
if data.raw_depthmap_exists(shot.id):
logger.info("Using precomputed raw depthmap {}".format(shot.id))
return
logger.info("Computing depthmap for image {0} with {1}".format(
shot.id, method))
de = csfm.DepthmapEstimator()
de.set_depth_range(min_depth, max_depth, 100)
de.set_patchmatch_iterations(data.config['depthmap_patchmatch_iterations'])
de.set_patch_size(data.config['depthmap_patch_size'])
de.set_min_patch_sd(data.config['depthmap_min_patch_sd'])
add_views_to_depth_estimator(data, neighbors, de)
if (method == 'BRUTE_FORCE'):
depth, plane, score, nghbr = de.compute_brute_force()
elif (method == 'PATCH_MATCH'):
depth, plane, score, nghbr = de.compute_patch_match()
elif (method == 'PATCH_MATCH_SAMPLE'):
depth, plane, score, nghbr = de.compute_patch_match_sample()
else:
raise ValueError(
'Unknown depthmap method type '
'(must be BRUTE_FORCE, PATCH_MATCH or PATCH_MATCH_SAMPLE)')
good_score = score > data.config['depthmap_min_correlation_score']
depth = depth * (depth < max_depth) * good_score
# Save and display results
neighbor_ids = [i.id for i in neighbors[1:]]
data.save_raw_depthmap(shot.id, depth, plane, score, nghbr, neighbor_ids)
if data.config['depthmap_save_debug_files']:
image = data.load_undistorted_image(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with io.open_wt(data._depthmap_file(shot.id, 'raw.npz.ply')) as fout:
fout.write(ply)
def clean_depthmap(arguments):
"""Clean depthmap by checking consistency with neighbors."""
log.setup()
data: UndistortedDataSet = arguments[0]
neighbors = arguments[1]
shot = arguments[2]
if data.clean_depthmap_exists(shot.id):
logger.info("Using precomputed clean depthmap {}".format(shot.id))
return
logger.info("Cleaning depthmap for image {}".format(shot.id))
dc = pydense.DepthmapCleaner()
dc.set_same_depth_threshold(data.config["depthmap_same_depth_threshold"])
dc.set_min_consistent_views(data.config["depthmap_min_consistent_views"])
add_views_to_depth_cleaner(data, neighbors, dc)
depth = dc.clean()
# Save and display results
raw_depth, raw_plane, raw_score, raw_nghbr, nghbrs = data.load_raw_depthmap(
shot.id)
data.save_clean_depthmap(shot.id, depth, raw_plane, raw_score)
if data.config["depthmap_save_debug_files"]:
image = data.load_undistorted_image(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with io.open_wt(data.depthmap_file(shot.id, "clean.npz.ply")) as fout:
fout.write(ply)
if data.config.get("interactive"):
import matplotlib.pyplot as plt
plt.figure()
plt.suptitle("Shot: " + shot.id)
plt.subplot(2, 2, 1)
plt.imshow(raw_depth)
plt.colorbar()
plt.subplot(2, 2, 2)
plt.imshow(depth)
plt.colorbar()
plt.show()
def match_unwrap_args(args):
""" Wrapper for parralel processing of pair matching
Compute all pair matchings of a given image and save them.
"""
log.setup()
im1, candidates, ctx = args
need_words = 'WORDS' in ctx.data.config['matcher_type']
need_index = 'FLANN' in ctx.data.config['matcher_type']
im1_matches = {}
p1, f1, _ = feature_loader.load_points_features_colors(ctx.data, im1)
w1 = feature_loader.load_words(ctx.data, im1) if need_words else None
m1 = feature_loader.load_masks(ctx.data, im1)
camera1 = ctx.cameras[ctx.exifs[im1]['camera']]
f1_filtered = f1 if m1 is None else f1[m1]
i1 = feature_loader.load_features_index(
ctx.data, im1, f1_filtered) if need_index else None
for im2 in candidates:
p2, f2, _ = feature_loader.load_points_features_colors(ctx.data, im2)
w2 = feature_loader.load_words(ctx.data, im2) if need_words else None
m2 = feature_loader.load_masks(ctx.data, im2)
camera2 = ctx.cameras[ctx.exifs[im2]['camera']]
f2_filtered = f2 if m2 is None else f2[m2]
i2 = feature_loader.load_features_index(
ctx.data, im2, f2_filtered) if need_index else None
im1_matches[im2] = match(im1, im2, camera1, camera2, p1, p2, f1, f2,
w1, w2, i1, i2, m1, m2, ctx.data)
num_matches = sum(1 for m in im1_matches.values() if len(m) > 0)
logger.debug('Image {} matches: {} out of {}'.format(
im1, num_matches, len(candidates)))
all_im1_matches = {} if ctx.overwrite else ctx.data.load_matches(im1)
all_im1_matches.update(im1_matches)
ctx.data.save_matches(im1, all_im1_matches)
return im1, im1_matches
def match_unwrap_args(args):
"""Wrapper for parallel processing of pair matching.
Compute all pair matchings of a given image and save them.
"""
log.setup()
im1, candidates, ctx = args
im1_matches = {}
camera1 = ctx.cameras[ctx.exifs[im1]['camera']]
for im2 in candidates:
camera2 = ctx.cameras[ctx.exifs[im2]['camera']]
im1_matches[im2] = match(im1, im2, camera1, camera2, ctx.data)
num_matches = sum(1 for m in im1_matches.values() if len(m) > 0)
logger.debug('Image {} matches: {} out of {}'.format(
im1, num_matches, len(candidates)))
return im1, im1_matches
def detect(args):
log.setup()
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
start = timer()
mask = data.mask_as_array(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config['preemptive_max']
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config, mask)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config['matcher_type'] == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report),
'features/{}.json'.format(image))
def detect(args):
log.setup()
image, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
if not data.feature_index_exists(image):
start = timer()
mask = data.load_combined_mask(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config['preemptive_max']
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.load_image(image), data.config, mask)
if len(p_unsorted) == 0:
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config['matcher_type'] == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report),
'features/{}.json'.format(image))
def clean_depthmap(arguments):
"""Clean depthmap by checking consistency with neighbors."""
log.setup()
data, neighbors, shot = arguments
if data.clean_depthmap_exists(shot.id):
logger.info("Using precomputed clean depthmap {}".format(shot.id))
return
logger.info("Cleaning depthmap for image {}".format(shot.id))
dc = csfm.DepthmapCleaner()
dc.set_same_depth_threshold(data.config['depthmap_same_depth_threshold'])
dc.set_min_consistent_views(data.config['depthmap_min_consistent_views'])
add_views_to_depth_cleaner(data, neighbors, dc)
depth = dc.clean()
# Save and display results
raw_depth, raw_plane, raw_score, raw_nghbr, nghbrs = data.load_raw_depthmap(shot.id)
data.save_clean_depthmap(shot.id, depth, raw_plane, raw_score)
if data.config['depthmap_save_debug_files']:
image = data.undistorted_image_as_array(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with io.open_wt(data._depthmap_file(shot.id, 'clean.npz.ply')) as fout:
fout.write(ply)
if data.config.get('interactive'):
import matplotlib.pyplot as plt
plt.figure()
plt.suptitle("Shot: " + shot.id)
plt.subplot(2, 2, 1)
plt.imshow(raw_depth)
plt.colorbar()
plt.subplot(2, 2, 2)
plt.imshow(depth)
plt.colorbar()
plt.show()
def prune_depthmap(arguments):
"""Prune depthmap to remove redundant points."""
log.setup()
data, neighbors, shot = arguments
if data.pruned_depthmap_exists(shot.id):
logger.info("Using precomputed pruned depthmap {}".format(shot.id))
return
logger.info("Pruning depthmap for image {}".format(shot.id))
dp = csfm.DepthmapPruner()
dp.set_same_depth_threshold(data.config['depthmap_same_depth_threshold'])
add_views_to_depth_pruner(data, neighbors, dp)
points, normals, colors = dp.prune()
# Save and display results
data.save_pruned_depthmap(shot.id, points, normals, colors)
if data.config['depthmap_save_debug_files']:
ply = point_cloud_to_ply(points, normals, colors)
with io.open_wt(data._depthmap_file(shot.id, 'pruned.npz.ply')) as fout:
fout.write(ply)
def undistort_image_and_masks(arguments):
shot, undistorted_shots, data = arguments
log.setup()
logger.debug('Undistorting image {}'.format(shot.id))
# Undistort image
image = data.load_image(shot.id)
if image is not None:
max_size = data.config['undistorted_image_max_size']
undistorted = undistort_image(shot, undistorted_shots, image,
cv2.INTER_AREA, max_size)
for k, v in undistorted.items():
data.save_undistorted_image(k, v)
# Undistort mask
mask = data.load_mask(shot.id)
if mask is not None:
undistorted = undistort_image(shot, undistorted_shots, mask,
cv2.INTER_NEAREST, 1e9)
for k, v in undistorted.items():
data.save_undistorted_mask(k, v)
# Undistort segmentation
segmentation = data.load_segmentation(shot.id)
if segmentation is not None:
undistorted = undistort_image(shot, undistorted_shots, segmentation,
cv2.INTER_NEAREST, 1e9)
for k, v in undistorted.items():
data.save_undistorted_segmentation(k, v)
# Undistort detections
detection = data.load_detection(shot.id)
if detection is not None:
undistorted = undistort_image(shot, undistorted_shots, detection,
cv2.INTER_NEAREST, 1e9)
for k, v in undistorted.items():
data.save_undistorted_detection(k, v)
def compute_depthmap(arguments):
"""Compute depthmap for a single shot."""
log.setup()
data, neighbors, min_depth, max_depth, shot = arguments
method = data.config['depthmap_method']
if data.raw_depthmap_exists(shot.id):
logger.info("Using precomputed raw depthmap {}".format(shot.id))
return
logger.info("Computing depthmap for image {0} with {1}".format(shot.id, method))
de = csfm.DepthmapEstimator()
de.set_depth_range(min_depth, max_depth, 100)
de.set_patchmatch_iterations(data.config['depthmap_patchmatch_iterations'])
de.set_min_patch_sd(data.config['depthmap_min_patch_sd'])
add_views_to_depth_estimator(data, neighbors, de)
if (method == 'BRUTE_FORCE'):
depth, plane, score, nghbr = de.compute_brute_force()
elif (method == 'PATCH_MATCH'):
depth, plane, score, nghbr = de.compute_patch_match()
elif (method == 'PATCH_MATCH_SAMPLE'):
depth, plane, score, nghbr = de.compute_patch_match_sample()
else:
raise ValueError(
'Unknown depthmap method type '
'(must be BRUTE_FORCE, PATCH_MATCH or PATCH_MATCH_SAMPLE)')
good_score = score > data.config['depthmap_min_correlation_score']
depth = depth * (depth < max_depth) * good_score
# Save and display results
neighbor_ids = [i.id for i in neighbors[1:]]
data.save_raw_depthmap(shot.id, depth, plane, score, nghbr, neighbor_ids)
if data.config['depthmap_save_debug_files']:
image = data.undistorted_image_as_array(shot.id)
image = scale_down_image(image, depth.shape[1], depth.shape[0])
ply = depthmap_to_ply(shot, depth, image)
with io.open_wt(data._depthmap_file(shot.id, 'raw.npz.ply')) as fout:
fout.write(ply)
if data.config.get('interactive'):
import matplotlib.pyplot as plt
plt.figure()
plt.suptitle("Shot: " + shot.id + ", neighbors: " + ', '.join(neighbor_ids))
plt.subplot(2, 3, 1)
plt.imshow(image)
plt.subplot(2, 3, 2)
plt.imshow(color_plane_normals(plane))
plt.subplot(2, 3, 3)
plt.imshow(depth)
plt.colorbar()
plt.subplot(2, 3, 4)
plt.imshow(score)
plt.colorbar()
plt.subplot(2, 3, 5)
plt.imshow(nghbr)
plt.colorbar()
plt.show()
def match(args):
"""Compute all matches for a single image"""
log.setup()
im1, candidates, i, n, ctx = args
logger.info('Matching {} - {} / {}'.format(im1, i + 1, n))
config = ctx.data.config
robust_matching_min_match = config['robust_matching_min_match']
preemptive_threshold = config['preemptive_threshold']
lowes_ratio = config['lowes_ratio']
preemptive_lowes_ratio = config['preemptive_lowes_ratio']
im1_matches = {}
for im2 in candidates:
# preemptive matching
if preemptive_threshold > 0:
t = timer()
config['lowes_ratio'] = preemptive_lowes_ratio
matches_pre = matching.match_lowe_bf(
ctx.f_pre[im1], ctx.f_pre[im2], config)
config['lowes_ratio'] = lowes_ratio
logger.debug("Preemptive matching {0}, time: {1}s".format(
len(matches_pre), timer() - t))
if len(matches_pre) < preemptive_threshold:
logger.debug(
"Discarding based of preemptive matches {0} < {1}".format(
len(matches_pre), preemptive_threshold))
continue
# symmetric matching
t = timer()
p1, f1, c1 = ctx.data.load_features(im1)
p2, f2, c2 = ctx.data.load_features(im2)
if config['matcher_type'] == 'FLANN':
i1 = ctx.data.load_feature_index(im1, f1)
i2 = ctx.data.load_feature_index(im2, f2)
else:
i1 = None
i2 = None
matches = matching.match_symmetric(f1, i1, f2, i2, config)
logger.debug('{} - {} has {} candidate matches'.format(
im1, im2, len(matches)))
if len(matches) < robust_matching_min_match:
im1_matches[im2] = []
continue
# robust matching
t_robust_matching = timer()
camera1 = ctx.cameras[ctx.exifs[im1]['camera']]
camera2 = ctx.cameras[ctx.exifs[im2]['camera']]
rmatches = matching.robust_match(p1, p2, camera1, camera2, matches,
config)
if len(rmatches) < robust_matching_min_match:
im1_matches[im2] = []
continue
im1_matches[im2] = rmatches
logger.debug('Robust matching time : {0}s'.format(
timer() - t_robust_matching))
logger.debug("Full matching {0} / {1}, time: {2}s".format(
len(rmatches), len(matches), timer() - t))
ctx.data.save_matches(im1, im1_matches)
