def detect(args): image, data = args log.setup() need_words = data.config[ 'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0 need_flann = data.config['matcher_type'] == 'FLANN' has_words = not need_words or data.words_exist(image) has_flann = not need_flann or data.feature_index_exists(image) has_features = data.features_exist(image) if has_features and has_flann and has_words: logger.info('Skip recomputing {} features for image {}'.format( data.feature_type().upper(), image)) return logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) start = timer() p_unmasked, f_unmasked, c_unmasked = features.extract_features( data.load_image(image), data.config) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_flann: index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) if need_words: bows = bow.load_bows(data.config) n_closest = data.config['bow_words_to_match'] closest_words = bows.map_to_words(f_sorted, n_closest, data.config['bow_matcher_type']) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args): image, data = args log.setup() need_words = (data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: logger.info("Skip recomputing {} features for image {}".format( data.feature_type().upper(), image)) return logger.info("Extracting {} features for image {}".format( data.feature_type().upper(), image)) start = timer() image_array = data.load_image(image) p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array)) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words(f_sorted, n_closest, data.config["bow_matcher_type"]) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def match_candidates_with_bow(data, images, max_neighbors): """Find candidate matching pairs using BoW-based distance.""" if max_neighbors <= 0: return set() words = {im: data.load_words(im) for im in images} bows = bow.load_bows(data.config) pairs = set() for im in images: order, other = bow_distances(im, images, words, None, bows) for i in order[:max_neighbors]: pairs.add(tuple(sorted((im, other[i])))) return pairs
def detect(args): image, data = args print('detect') print(image) print(data) #log.setup() need_words = data.config[ 'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0 has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: #logger.info('Skip recomputing {} features for image {}'.format(data.feature_type().upper(), image)) return #logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image)) #start = timer() p_unmasked, f_unmasked, c_unmasked = features.extract_features( data.load_image(image), data.config) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: #logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config['bow_words_to_match'] closest_words = bows.map_to_words(f_sorted, n_closest, data.config['bow_matcher_type']) data.save_words(image, closest_words)
def load_histograms(data, images): """ Load BoW histograms of given images """ min_num_feature = 8 histograms = {} bows = bow.load_bows(data.config) for im in images: filtered_words = feature_loader.instance.load_words(data, im, masked=True) if len(filtered_words) <= min_num_feature: logger.warning("Too few filtered features in image {}: {}".format( im, len(filtered_words))) continue histograms[im] = bows.histogram(filtered_words[:, 0]) return histograms
def detect(feature_path, image_path, image, opensfm_config): log.setup() need_words = opensfm_config['matcher_type'] == 'WORDS' or opensfm_config[ 'matching_bow_neighbors'] > 0 #has_words = not need_words or data.words_exist(image) #has_features = data.features_exist(image) # if has_features and has_words: # logger.info('Skip recomputing {} features for image {}'.format( # data.feature_type().upper(), image)) # return #logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image)) p_unmasked, f_unmasked, c_unmasked = features.extract_features( load_image(image_path), opensfm_config) #p_unmasked is points mask_files = defaultdict(lambda: None) fmask = load_features_mask(feature_path, image, image_path, p_unmasked, mask_files, opensfm_config) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: #logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] save_features(feature_path, opensfm_config, image, p_sorted, f_sorted, c_sorted) if need_words: bows = bow.load_bows(opensfm_config) n_closest = opensfm_config['bow_words_to_match'] closest_words = bows.map_to_words(f_sorted, n_closest, opensfm_config['bow_matcher_type']) save_words(feature_path, image_path, closest_words)
def load_histograms(feature_path, opensfm_config, images): """ Load BoW histograms of given images """ min_num_feature = 8 histograms = {} bows = bow.load_bows(opensfm_config) for im in images: filtered_words = new_feature_loader.instance.load_words(feature_path, opensfm_config, im, masked=True) if filtered_words is None: logger.error("No words in image {}".format(im)) continue if len(filtered_words) <= min_num_feature: logger.warning("Too few filtered features in image {}: {}".format( im, len(filtered_words))) continue histograms[im] = bows.histogram(filtered_words[:, 0]) return histograms
def load_histograms(data: DataSetBase, images: Iterable[str]) -> Dict[str, np.ndarray]: """Load BoW histograms of given images""" min_num_feature = 8 histograms = {} bows = bow.load_bows(data.config) for im in images: filtered_words = feature_loader.instance.load_words(data, im, masked=True) if filtered_words is None: logger.error("No words in image {}".format(im)) continue if len(filtered_words) <= min_num_feature: logger.warning( "Too few filtered features in image {}: {}".format( im, len(filtered_words) ) ) continue histograms[im] = bows.histogram(filtered_words[:, 0]) return histograms
def load_histograms(data, images): """ Load BoW histograms of given images """ min_num_feature = 8 histograms = {} bows = bow.load_bows(data.config) for im in images: words = data.load_words(im) if words is None: logger.error("Could not load words for image {}".format(im)) continue mask = data.load_masks(data, im) if hasattr(data, 'load_masks') else None filtered_words = words[mask] if mask else words if len(filtered_words) <= min_num_feature: logger.warning("Too few filtered features in image {}: {}".format( im, len(filtered_words))) continue histograms[im] = bows.histogram(words[:, 0]) return histograms
def detect( image: str, image_array: np.ndarray, segmentation_array: Optional[np.ndarray], instances_array: Optional[np.ndarray], data: DataSetBase, force: bool = False, ) -> None: log.setup() need_words = ( data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0 ) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if not force and has_features and has_words: logger.info( "Skip recomputing {} features for image {}".format( data.feature_type().upper(), image ) ) return logger.info( "Extracting {} features for image {}".format(data.feature_type().upper(), image) ) start = timer() p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array) ) # Load segmentation and bake it in the data if data.config["features_bake_segmentation"]: exif = data.load_exif(image) s_unsorted, i_unsorted = bake_segmentation( image_array, p_unmasked, segmentation_array, instances_array, exif ) p_unsorted = p_unmasked f_unsorted = f_unmasked c_unsorted = c_unmasked # Load segmentation, make a mask from it mask and apply it else: s_unsorted, i_unsorted = None, None fmask = masking.load_features_mask(data, image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] if s_unsorted is not None: semantic_data = features.SemanticData( s_unsorted[order], i_unsorted[order] if i_unsorted is not None else None, data.segmentation_labels(), ) else: semantic_data = None features_data = features.FeaturesData(p_sorted, f_sorted, c_sorted, semantic_data) data.save_features(image, features_data) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words( f_sorted, n_closest, data.config["bow_matcher_type"] ) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args: Tuple[str, DataSetBase]): image, data = args log.setup() need_words = (data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: logger.info("Skip recomputing {} features for image {}".format( data.feature_type().upper(), image)) return logger.info("Extracting {} features for image {}".format( data.feature_type().upper(), image)) start = timer() image_array = data.load_image(image) p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array)) # Load segmentation and bake it in the data if data.config["features_bake_segmentation"]: exif = data.load_exif(image) panoptic_data = [None, None] for i, p_data in enumerate( [data.load_segmentation(image), data.load_instances(image)]): if p_data is None: continue new_height, new_width = p_data.shape ps = upright.opensfm_to_upright( p_unmasked[:, :2], exif["width"], exif["height"], exif["orientation"], new_width=new_width, new_height=new_height, ).astype(int) panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]] s_unsorted, i_unsorted = panoptic_data p_unsorted = p_unmasked f_unsorted = f_unmasked c_unsorted = c_unmasked # Load segmentation, make a mask from it mask and apply it else: s_unsorted, i_unsorted = None, None fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] # pyre-fixme[16]: `None` has no attribute `__getitem__`. s_sorted = s_unsorted[order] if s_unsorted is not None else None i_sorted = i_unsorted[order] if i_unsorted is not None else None data.save_features(image, p_sorted, f_sorted, c_sorted, s_sorted, i_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words(f_sorted, n_closest, data.config["bow_matcher_type"]) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args): log.setup() image, data = args need_words = data.config['matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0 need_flann = data.config['matcher_type'] == 'FLANN' has_words = not need_words or data.words_exist(image) has_flann = not need_flann or data.feature_index_exists(image) has_features = data.features_exist(image) if has_features and has_flann and has_words: logger.info('Skip recomputing {} features for image {}'.format( data.feature_type().upper(), image)) return logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) start = timer() exif = data.load_exif( image ) camera_models = data.load_camera_models() image_camera_model = camera_models[ exif[ 'camera' ] ] if image_camera_model.projection_type in ['equirectangular', 'spherical'] and data.config['matching_unfolded_cube']: logger.info('Features unfolded cube.') # For spherical cameras create an undistorted image for the purposes of # feature finding (and later matching). max_size = data.config.get('ai_process_size', -1) if max_size == -1: max_size = img.shape[1] img = data.load_image( image ) undist_tile_size = max_size//4 undist_img = np.zeros( (max_size//2, max_size, 3 ), np.uint8 ) undist_mask = np.full( (max_size//2, max_size, 1 ), 255, np.uint8 ) undist_mask[ undist_tile_size:2*undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = 0 undist_mask[ undist_tile_size:2*undist_tile_size, undist_tile_size:2*undist_tile_size ] = 0 # The bottom mask to remove the influence of the camera person should be configurable. It depends on the forward # direction of the camera and where the camera person positions themselves in relation to this direction. It'save_feature_index # probably worth it to take care with this because the floor could help hold the reconstructions together. #undist_mask[ 5*undist_tile_size//4:7*undist_tile_size//4, undist_tile_size//3:undist_tile_size ] = 0 #undist_mask[ 3*undist_tile_size//2:2*undist_tile_size, undist_tile_size//2:undist_tile_size ] = 0 spherical_shot = types.Shot() spherical_shot.pose = types.Pose() spherical_shot.id = image spherical_shot.camera = image_camera_model perspective_shots = undistort.perspective_views_of_a_panorama( spherical_shot, undist_tile_size ) for subshot in perspective_shots: undistorted = undistort.render_perspective_view_of_a_panorama( img, spherical_shot, subshot ) subshot_id_prefix = '{}_perspective_view_'.format( spherical_shot.id ) subshot_name = subshot.id[ len(subshot_id_prefix): ] if subshot.id.startswith( subshot_id_prefix ) else subshot.id ( subshot_name, ext ) = os.path.splitext( subshot_name ) if subshot_name == 'front': undist_img[ :undist_tile_size, :undist_tile_size ] = undistorted #print( 'front') elif subshot_name == 'left': undist_img[ :undist_tile_size, undist_tile_size:2*undist_tile_size ] = undistorted #print( 'left') elif subshot_name == 'back': undist_img[ :undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = undistorted #print( 'back') elif subshot_name == 'right': undist_img[ :undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted #print( 'right') elif subshot_name == 'top': undist_img[ undist_tile_size:2*undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted #print( 'top') elif subshot_name == 'bottom': undist_img[ undist_tile_size:2*undist_tile_size, :undist_tile_size ] = undistorted #print( 'bottom') #data.save_undistorted_image(subshot.id, undistorted) data.save_undistorted_image(image.split(".")[0], undist_img) # We might consider combining a user supplied mask here as well undist_img = resized_image(undist_img, data.config) p_unsorted, f_unsorted, c_unsorted = features.extract_features(undist_img, data.config, undist_mask) # Visualize the features on the unfolded cube # -------------------------------------------------------------- if False: h_ud, w_ud, _ = undist_img.shape denorm_ud = denormalized_image_coordinates( p_unsorted[:, :2], w_ud, h_ud ) print( p_unsorted.shape ) print( denorm_ud.shape ) rcolors = [] for point in denorm_ud: color = np.random.randint(0,255,(3)).tolist() cv2.circle( undist_img, (int(point[0]),int(point[1])), 1, color, -1 ) rcolors.append( color ) data.save_undistorted_image( image + '_unfolded_cube.jpg', undist_img) # -------------------------------------------------------------- if len(p_unsorted) > 0: # Mask pixels that are out of valid image bounds before converting to equirectangular image coordinates bearings = image_camera_model.unfolded_pixel_bearings( p_unsorted[:, :2] ) p_mask = np.array([ point is not None for point in bearings ]) p_unsorted = p_unsorted[ p_mask ] f_unsorted = f_unsorted[ p_mask ] c_unsorted = c_unsorted[ p_mask ] p_unsorted[:, :2] = unfolded_cube_to_equi_normalized_image_coordinates( p_unsorted[:, :2], image_camera_model ) # Visualize the same features converted back to equirectangular image coordinates # ----------------------------------------------------------------------------------------- if False: timg = resized_image( img, data.config ) h, w, _ = timg.shape denorm = denormalized_image_coordinates( p_unsorted[:, :2], w, h ) for ind, point in enumerate( denorm ): cv2.circle( timg, (int(point[0]),int(point[1])), 1, rcolors[ind], -1 ) data.save_undistorted_image('original.jpg', timg) #------------------------------------------------------------------------------------------ else: mask = data.load_combined_mask(image) if mask is not None: logger.info('Found mask to apply for image {}'.format(image)) p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.load_image(image), data.config, mask) if len(p_unsorted) == 0: logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_flann: index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) if need_words: bows = bow.load_bows(data.config) n_closest = data.config['bow_words_to_match'] closest_words = bows.map_to_words( f_sorted, n_closest, data.config['bow_matcher_type']) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))