def detect(args):
image, data = args
log.setup()
need_words = data.config[
'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
need_flann = data.config['matcher_type'] == 'FLANN'
has_words = not need_words or data.words_exist(image)
has_flann = not need_flann or data.feature_index_exists(image)
has_features = data.features_exist(image)
if has_features and has_flann and has_words:
logger.info('Skip recomputing {} features for image {}'.format(
data.feature_type().upper(), image))
return
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
data.load_image(image), data.config)
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_flann:
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config['bow_matcher_type'])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args):
image, data = args
log.setup()
need_words = (data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
logger.info("Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image))
return
logger.info("Extracting {} features for image {}".format(
data.feature_type().upper(), image))
start = timer()
image_array = data.load_image(image)
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image,
image_array))
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config["bow_matcher_type"])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def match_candidates_with_bow(data, images, max_neighbors):
"""Find candidate matching pairs using BoW-based distance."""
if max_neighbors <= 0:
return set()
words = {im: data.load_words(im) for im in images}
bows = bow.load_bows(data.config)
pairs = set()
for im in images:
order, other = bow_distances(im, images, words, None, bows)
for i in order[:max_neighbors]:
pairs.add(tuple(sorted((im, other[i]))))
return pairs
def detect(args):
image, data = args
print('detect')
print(image)
print(data)
#log.setup()
need_words = data.config[
'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
#logger.info('Skip recomputing {} features for image {}'.format(data.feature_type().upper(), image))
return
#logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image))
#start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
data.load_image(image), data.config)
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
#logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config['bow_matcher_type'])
data.save_words(image, closest_words)
def load_histograms(data, images):
""" Load BoW histograms of given images """
min_num_feature = 8
histograms = {}
bows = bow.load_bows(data.config)
for im in images:
filtered_words = feature_loader.instance.load_words(data,
im,
masked=True)
if len(filtered_words) <= min_num_feature:
logger.warning("Too few filtered features in image {}: {}".format(
im, len(filtered_words)))
continue
histograms[im] = bows.histogram(filtered_words[:, 0])
return histograms
def detect(feature_path, image_path, image, opensfm_config):
log.setup()
need_words = opensfm_config['matcher_type'] == 'WORDS' or opensfm_config[
'matching_bow_neighbors'] > 0
#has_words = not need_words or data.words_exist(image)
#has_features = data.features_exist(image)
# if has_features and has_words:
# logger.info('Skip recomputing {} features for image {}'.format(
# data.feature_type().upper(), image))
# return
#logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image))
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
load_image(image_path), opensfm_config)
#p_unmasked is points
mask_files = defaultdict(lambda: None)
fmask = load_features_mask(feature_path, image, image_path, p_unmasked,
mask_files, opensfm_config)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
#logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
save_features(feature_path, opensfm_config, image, p_sorted, f_sorted,
c_sorted)
if need_words:
bows = bow.load_bows(opensfm_config)
n_closest = opensfm_config['bow_words_to_match']
closest_words = bows.map_to_words(f_sorted, n_closest,
opensfm_config['bow_matcher_type'])
save_words(feature_path, image_path, closest_words)
def load_histograms(feature_path, opensfm_config, images):
""" Load BoW histograms of given images """
min_num_feature = 8
histograms = {}
bows = bow.load_bows(opensfm_config)
for im in images:
filtered_words = new_feature_loader.instance.load_words(feature_path, opensfm_config, im, masked=True)
if filtered_words is None:
logger.error("No words in image {}".format(im))
continue
if len(filtered_words) <= min_num_feature:
logger.warning("Too few filtered features in image {}: {}".format(
im, len(filtered_words)))
continue
histograms[im] = bows.histogram(filtered_words[:, 0])
return histograms
def load_histograms(data: DataSetBase, images: Iterable[str]) -> Dict[str, np.ndarray]:
"""Load BoW histograms of given images"""
min_num_feature = 8
histograms = {}
bows = bow.load_bows(data.config)
for im in images:
filtered_words = feature_loader.instance.load_words(data, im, masked=True)
if filtered_words is None:
logger.error("No words in image {}".format(im))
continue
if len(filtered_words) <= min_num_feature:
logger.warning(
"Too few filtered features in image {}: {}".format(
im, len(filtered_words)
)
)
continue
histograms[im] = bows.histogram(filtered_words[:, 0])
return histograms
def load_histograms(data, images):
""" Load BoW histograms of given images """
min_num_feature = 8
histograms = {}
bows = bow.load_bows(data.config)
for im in images:
words = data.load_words(im)
if words is None:
logger.error("Could not load words for image {}".format(im))
continue
mask = data.load_masks(data, im) if hasattr(data, 'load_masks') else None
filtered_words = words[mask] if mask else words
if len(filtered_words) <= min_num_feature:
logger.warning("Too few filtered features in image {}: {}".format(
im, len(filtered_words)))
continue
histograms[im] = bows.histogram(words[:, 0])
return histograms
def detect(
image: str,
image_array: np.ndarray,
segmentation_array: Optional[np.ndarray],
instances_array: Optional[np.ndarray],
data: DataSetBase,
force: bool = False,
) -> None:
log.setup()
need_words = (
data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0
)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if not force and has_features and has_words:
logger.info(
"Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image
)
)
return
logger.info(
"Extracting {} features for image {}".format(data.feature_type().upper(), image)
)
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image, image_array)
)
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
s_unsorted, i_unsorted = bake_segmentation(
image_array, p_unmasked, segmentation_array, instances_array, exif
)
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = masking.load_features_mask(data, image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
if s_unsorted is not None:
semantic_data = features.SemanticData(
s_unsorted[order],
i_unsorted[order] if i_unsorted is not None else None,
data.segmentation_labels(),
)
else:
semantic_data = None
features_data = features.FeaturesData(p_sorted, f_sorted, c_sorted, semantic_data)
data.save_features(image, features_data)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(
f_sorted, n_closest, data.config["bow_matcher_type"]
)
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args: Tuple[str, DataSetBase]):
image, data = args
log.setup()
need_words = (data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
logger.info("Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image))
return
logger.info("Extracting {} features for image {}".format(
data.feature_type().upper(), image))
start = timer()
image_array = data.load_image(image)
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image,
image_array))
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
panoptic_data = [None, None]
for i, p_data in enumerate(
[data.load_segmentation(image),
data.load_instances(image)]):
if p_data is None:
continue
new_height, new_width = p_data.shape
ps = upright.opensfm_to_upright(
p_unmasked[:, :2],
exif["width"],
exif["height"],
exif["orientation"],
new_width=new_width,
new_height=new_height,
).astype(int)
panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]]
s_unsorted, i_unsorted = panoptic_data
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
# pyre-fixme[16]: `None` has no attribute `__getitem__`.
s_sorted = s_unsorted[order] if s_unsorted is not None else None
i_sorted = i_unsorted[order] if i_unsorted is not None else None
data.save_features(image, p_sorted, f_sorted, c_sorted, s_sorted, i_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config["bow_matcher_type"])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args):
log.setup()
image, data = args
need_words = data.config['matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0
need_flann = data.config['matcher_type'] == 'FLANN'
has_words = not need_words or data.words_exist(image)
has_flann = not need_flann or data.feature_index_exists(image)
has_features = data.features_exist(image)
if has_features and has_flann and has_words:
logger.info('Skip recomputing {} features for image {}'.format(
data.feature_type().upper(), image))
return
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
start = timer()
exif = data.load_exif( image )
camera_models = data.load_camera_models()
image_camera_model = camera_models[ exif[ 'camera' ] ]
if image_camera_model.projection_type in ['equirectangular', 'spherical'] and data.config['matching_unfolded_cube']:
logger.info('Features unfolded cube.')
# For spherical cameras create an undistorted image for the purposes of
# feature finding (and later matching).
max_size = data.config.get('ai_process_size', -1)
if max_size == -1:
max_size = img.shape[1]
img = data.load_image( image )
undist_tile_size = max_size//4
undist_img = np.zeros( (max_size//2, max_size, 3 ), np.uint8 )
undist_mask = np.full( (max_size//2, max_size, 1 ), 255, np.uint8 )
undist_mask[ undist_tile_size:2*undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = 0
undist_mask[ undist_tile_size:2*undist_tile_size, undist_tile_size:2*undist_tile_size ] = 0
# The bottom mask to remove the influence of the camera person should be configurable. It depends on the forward
# direction of the camera and where the camera person positions themselves in relation to this direction. It'save_feature_index
# probably worth it to take care with this because the floor could help hold the reconstructions together.
#undist_mask[ 5*undist_tile_size//4:7*undist_tile_size//4, undist_tile_size//3:undist_tile_size ] = 0
#undist_mask[ 3*undist_tile_size//2:2*undist_tile_size, undist_tile_size//2:undist_tile_size ] = 0
spherical_shot = types.Shot()
spherical_shot.pose = types.Pose()
spherical_shot.id = image
spherical_shot.camera = image_camera_model
perspective_shots = undistort.perspective_views_of_a_panorama( spherical_shot, undist_tile_size )
for subshot in perspective_shots:
undistorted = undistort.render_perspective_view_of_a_panorama( img, spherical_shot, subshot )
subshot_id_prefix = '{}_perspective_view_'.format( spherical_shot.id )
subshot_name = subshot.id[ len(subshot_id_prefix): ] if subshot.id.startswith( subshot_id_prefix ) else subshot.id
( subshot_name, ext ) = os.path.splitext( subshot_name )
if subshot_name == 'front':
undist_img[ :undist_tile_size, :undist_tile_size ] = undistorted
#print( 'front')
elif subshot_name == 'left':
undist_img[ :undist_tile_size, undist_tile_size:2*undist_tile_size ] = undistorted
#print( 'left')
elif subshot_name == 'back':
undist_img[ :undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = undistorted
#print( 'back')
elif subshot_name == 'right':
undist_img[ :undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted
#print( 'right')
elif subshot_name == 'top':
undist_img[ undist_tile_size:2*undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted
#print( 'top')
elif subshot_name == 'bottom':
undist_img[ undist_tile_size:2*undist_tile_size, :undist_tile_size ] = undistorted
#print( 'bottom')
#data.save_undistorted_image(subshot.id, undistorted)
data.save_undistorted_image(image.split(".")[0], undist_img)
# We might consider combining a user supplied mask here as well
undist_img = resized_image(undist_img, data.config)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(undist_img, data.config, undist_mask)
# Visualize the features on the unfolded cube
# --------------------------------------------------------------
if False:
h_ud, w_ud, _ = undist_img.shape
denorm_ud = denormalized_image_coordinates( p_unsorted[:, :2], w_ud, h_ud )
print( p_unsorted.shape )
print( denorm_ud.shape )
rcolors = []
for point in denorm_ud:
color = np.random.randint(0,255,(3)).tolist()
cv2.circle( undist_img, (int(point[0]),int(point[1])), 1, color, -1 )
rcolors.append( color )
data.save_undistorted_image( image + '_unfolded_cube.jpg', undist_img)
# --------------------------------------------------------------
if len(p_unsorted) > 0:
# Mask pixels that are out of valid image bounds before converting to equirectangular image coordinates
bearings = image_camera_model.unfolded_pixel_bearings( p_unsorted[:, :2] )
p_mask = np.array([ point is not None for point in bearings ])
p_unsorted = p_unsorted[ p_mask ]
f_unsorted = f_unsorted[ p_mask ]
c_unsorted = c_unsorted[ p_mask ]
p_unsorted[:, :2] = unfolded_cube_to_equi_normalized_image_coordinates( p_unsorted[:, :2], image_camera_model )
# Visualize the same features converted back to equirectangular image coordinates
# -----------------------------------------------------------------------------------------
if False:
timg = resized_image( img, data.config )
h, w, _ = timg.shape
denorm = denormalized_image_coordinates( p_unsorted[:, :2], w, h )
for ind, point in enumerate( denorm ):
cv2.circle( timg, (int(point[0]),int(point[1])), 1, rcolors[ind], -1 )
data.save_undistorted_image('original.jpg', timg)
#------------------------------------------------------------------------------------------
else:
mask = data.load_combined_mask(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.load_image(image), data.config, mask)
if len(p_unsorted) == 0:
logger.warning('No features found in image {}'.format(image))
return
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
data.save_features(image, p_sorted, f_sorted, c_sorted)
if need_flann:
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config['bow_words_to_match']
closest_words = bows.map_to_words(
f_sorted, n_closest, data.config['bow_matcher_type'])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
