def load_features_mask(self, image, points): """Load a feature-wise mask. This is a binary array true for features that lie inside the combined mask. The array is all true when there's no mask. """ if points is None or len(points) == 0: return np.array([], dtype=bool) mask_image = self.load_combined_mask(image) if mask_image is None: logger.debug('No segmentation for {}, no features masked.'.format(image)) return np.ones((points.shape[0],), dtype=bool) exif = self.load_exif(image) width = exif["width"] height = exif["height"] orientation = exif["orientation"] new_height, new_width = mask_image.shape ps = upright.opensfm_to_upright( points[:, :2], width, height, orientation, new_width=new_width, new_height=new_height).astype(int) mask = mask_image[ps[:, 1], ps[:, 0]] n_removed = np.sum(mask == 0) logger.debug('Masking {} / {} ({:.2f}) features for {}'.format( n_removed, len(mask), n_removed / len(mask), image)) return np.array(mask, dtype=bool)
def bake_segmentation( image: np.ndarray, points: np.ndarray, segmentation: Optional[np.ndarray], instances: Optional[np.ndarray], exif: Dict[str, Any], ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]: exif_height, exif_width, exif_orientation = ( exif["height"], exif["width"], exif["orientation"], ) height, width = image.shape[:2] if exif_height != height or exif_width != width: logger.error( f"Image has inconsistent EXIF dimensions ({exif_width}, {exif_height}) and image dimensions ({width}, {height}). Orientation={exif_orientation}" ) panoptic_data = [None, None] for i, p_data in enumerate([segmentation, instances]): if p_data is None: continue new_height, new_width = p_data.shape ps = upright.opensfm_to_upright( points[:, :2], width, height, exif["orientation"], new_width=new_width, new_height=new_height, ).astype(int) panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]] return tuple(panoptic_data)
def load_features_mask(self, image, feat): """Load a feature-wise mask if it exists, otherwise return None. This mask is used when performing features matching. """ if feat is None or len(feat) == 0: return np.array([], dtype=bool) mask_image = self.load_combined_mask(image) if mask_image is None: logger.debug( 'No segmentation for {}, no features masked.'.format(image)) return np.ones((feat.shape[0], ), dtype=bool) exif = self.load_exif(image) width = exif["width"] height = exif["height"] orientation = exif["orientation"] new_height, new_width = mask_image.shape ps = upright.opensfm_to_upright(feat, width, height, orientation, new_width=new_width, new_height=new_height).astype(int) mask = mask_image[ps[:, 1], ps[:, 0]] n_removed = len(mask) - np.sum(mask) logger.debug('Masking {} / {} ({:.2f}) features for {}'.format( n_removed, len(mask), n_removed / len(mask), image)) return np.array(mask, dtype=bool)
def bake_segmentation( points: np.ndarray, segmentation: Optional[np.ndarray], instances: Optional[np.ndarray], exif: Dict[str, Any], ): panoptic_data = [None, None] for i, p_data in enumerate([segmentation, instances]): if p_data is None: continue new_height, new_width = p_data.shape ps = upright.opensfm_to_upright( points[:, :2], exif["width"], exif["height"], exif["orientation"], new_width=new_width, new_height=new_height, ).astype(int) panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]] return panoptic_data
def detect(args: Tuple[str, DataSetBase]): image, data = args log.setup() need_words = (data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: logger.info("Skip recomputing {} features for image {}".format( data.feature_type().upper(), image)) return logger.info("Extracting {} features for image {}".format( data.feature_type().upper(), image)) start = timer() image_array = data.load_image(image) p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array)) # Load segmentation and bake it in the data if data.config["features_bake_segmentation"]: exif = data.load_exif(image) panoptic_data = [None, None] for i, p_data in enumerate( [data.load_segmentation(image), data.load_instances(image)]): if p_data is None: continue new_height, new_width = p_data.shape ps = upright.opensfm_to_upright( p_unmasked[:, :2], exif["width"], exif["height"], exif["orientation"], new_width=new_width, new_height=new_height, ).astype(int) panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]] s_unsorted, i_unsorted = panoptic_data p_unsorted = p_unmasked f_unsorted = f_unmasked c_unsorted = c_unmasked # Load segmentation, make a mask from it mask and apply it else: s_unsorted, i_unsorted = None, None fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] # pyre-fixme[16]: `None` has no attribute `__getitem__`. s_sorted = s_unsorted[order] if s_unsorted is not None else None i_sorted = i_unsorted[order] if i_unsorted is not None else None data.save_features(image, p_sorted, f_sorted, c_sorted, s_sorted, i_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words(f_sorted, n_closest, data.config["bow_matcher_type"]) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))