def load_features_mask(self, image, points):
"""Load a feature-wise mask.
This is a binary array true for features that lie inside the
combined mask.
The array is all true when there's no mask.
"""
if points is None or len(points) == 0:
return np.array([], dtype=bool)
mask_image = self.load_combined_mask(image)
if mask_image is None:
logger.debug('No segmentation for {}, no features masked.'.format(image))
return np.ones((points.shape[0],), dtype=bool)
exif = self.load_exif(image)
width = exif["width"]
height = exif["height"]
orientation = exif["orientation"]
new_height, new_width = mask_image.shape
ps = upright.opensfm_to_upright(
points[:, :2], width, height, orientation,
new_width=new_width, new_height=new_height).astype(int)
mask = mask_image[ps[:, 1], ps[:, 0]]
n_removed = np.sum(mask == 0)
logger.debug('Masking {} / {} ({:.2f}) features for {}'.format(
n_removed, len(mask), n_removed / len(mask), image))
return np.array(mask, dtype=bool)
def bake_segmentation(
image: np.ndarray,
points: np.ndarray,
segmentation: Optional[np.ndarray],
instances: Optional[np.ndarray],
exif: Dict[str, Any],
) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]:
exif_height, exif_width, exif_orientation = (
exif["height"],
exif["width"],
exif["orientation"],
)
height, width = image.shape[:2]
if exif_height != height or exif_width != width:
logger.error(
f"Image has inconsistent EXIF dimensions ({exif_width}, {exif_height}) and image dimensions ({width}, {height}). Orientation={exif_orientation}"
)
panoptic_data = [None, None]
for i, p_data in enumerate([segmentation, instances]):
if p_data is None:
continue
new_height, new_width = p_data.shape
ps = upright.opensfm_to_upright(
points[:, :2],
width,
height,
exif["orientation"],
new_width=new_width,
new_height=new_height,
).astype(int)
panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]]
return tuple(panoptic_data)
def load_features_mask(self, image, feat):
"""Load a feature-wise mask if it exists, otherwise return None.
This mask is used when performing features matching.
"""
if feat is None or len(feat) == 0:
return np.array([], dtype=bool)
mask_image = self.load_combined_mask(image)
if mask_image is None:
logger.debug(
'No segmentation for {}, no features masked.'.format(image))
return np.ones((feat.shape[0], ), dtype=bool)
exif = self.load_exif(image)
width = exif["width"]
height = exif["height"]
orientation = exif["orientation"]
new_height, new_width = mask_image.shape
ps = upright.opensfm_to_upright(feat,
width,
height,
orientation,
new_width=new_width,
new_height=new_height).astype(int)
mask = mask_image[ps[:, 1], ps[:, 0]]
n_removed = len(mask) - np.sum(mask)
logger.debug('Masking {} / {} ({:.2f}) features for {}'.format(
n_removed, len(mask), n_removed / len(mask), image))
return np.array(mask, dtype=bool)
def bake_segmentation(
points: np.ndarray,
segmentation: Optional[np.ndarray],
instances: Optional[np.ndarray],
exif: Dict[str, Any],
):
panoptic_data = [None, None]
for i, p_data in enumerate([segmentation, instances]):
if p_data is None:
continue
new_height, new_width = p_data.shape
ps = upright.opensfm_to_upright(
points[:, :2],
exif["width"],
exif["height"],
exif["orientation"],
new_width=new_width,
new_height=new_height,
).astype(int)
panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]]
return panoptic_data
def detect(args: Tuple[str, DataSetBase]):
image, data = args
log.setup()
need_words = (data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if has_features and has_words:
logger.info("Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image))
return
logger.info("Extracting {} features for image {}".format(
data.feature_type().upper(), image))
start = timer()
image_array = data.load_image(image)
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image,
image_array))
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
panoptic_data = [None, None]
for i, p_data in enumerate(
[data.load_segmentation(image),
data.load_instances(image)]):
if p_data is None:
continue
new_height, new_width = p_data.shape
ps = upright.opensfm_to_upright(
p_unmasked[:, :2],
exif["width"],
exif["height"],
exif["orientation"],
new_width=new_width,
new_height=new_height,
).astype(int)
panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]]
s_unsorted, i_unsorted = panoptic_data
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = data.load_features_mask(image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
# pyre-fixme[16]: `None` has no attribute `__getitem__`.
s_sorted = s_unsorted[order] if s_unsorted is not None else None
i_sorted = i_unsorted[order] if i_unsorted is not None else None
data.save_features(image, p_sorted, f_sorted, c_sorted, s_sorted, i_sorted)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(f_sorted, n_closest,
data.config["bow_matcher_type"])
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
