def detect(args): image, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) if not data.feature_index_exists(image): mask = data.mask_as_array(image) if mask is not None: logger.info('Found mask to apply for image {}'.format(image)) preemptive_max = data.config.get('preemptive_max', 200) p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.image_as_array(image), data.config, mask) if len(p_unsorted) == 0: return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] data.save_features(image, p_sorted, f_sorted, c_sorted) data.save_preemptive_features(image, p_pre, f_pre) if data.config.get('matcher_type', 'FLANN') == 'FLANN': index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index)
def detect(args): image, data = args log.setup() need_words = data.config[ 'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0 need_flann = data.config['matcher_type'] == 'FLANN' has_words = not need_words or data.words_exist(image) has_flann = not need_flann or data.feature_index_exists(image) has_features = data.features_exist(image) if has_features and has_flann and has_words: logger.info('Skip recomputing {} features for image {}'.format( data.feature_type().upper(), image)) return logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) start = timer() p_unmasked, f_unmasked, c_unmasked = features.extract_features( data.load_image(image), data.config) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_flann: index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) if need_words: bows = bow.load_bows(data.config) n_closest = data.config['bow_words_to_match'] closest_words = bows.map_to_words(f_sorted, n_closest, data.config['bow_matcher_type']) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args): image, data = args log.setup() need_words = (data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: logger.info("Skip recomputing {} features for image {}".format( data.feature_type().upper(), image)) return logger.info("Extracting {} features for image {}".format( data.feature_type().upper(), image)) start = timer() image_array = data.load_image(image) p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array)) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words(f_sorted, n_closest, data.config["bow_matcher_type"]) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args): image, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) if not data.features_exist(image): mask = data.mask_as_array(image) if mask is not None: print("found mask for image:%s" % image) logger.info('Found mask to apply for image {}'.format(image)) else: print("Not found mask for the image") preemptive_max = data.config.get('preemptive_max', 200) the_image = data.image_as_array(image) save_no_mask = False all_content = features.extract_features(the_image, data.config, mask, save_no_mask) if save_no_mask: p_unsorted, f_unsorted, c_unsorted = all_content[0] p_nomask, f_nomask, c_nomask = all_content[1] else: p_unsorted, f_unsorted, c_unsorted = all_content if len(p_unsorted) == 0: return ''' size_nomask = p_nomask[:, 2] order_nomask = np.argsort(size_nomask) p_nomask = p_nomask[order_nomask, :] f_nomask = f_nomask[order_nomask, :] c_nomask = c_nomask[order_nomask, :] p_nomask_pre = p_nomask[-preemptive_max:] f_nomask_pre = f_nomask[-preemptive_max:] data.save_features(image+'_nomask', p_nomask, f_nomask, c_nomask) data.save_preemptive_features(image+'_nomask', p_nomask_pre, f_nomask_pre) index_nomask = features.build_flann_index(f_nomask, data.config) data.save_feature_index(image+'_nomask', index_nomask) ''' size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if data.config.get('preemptive_threshold', 0) > 0: p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] data.save_preemptive_features(image, p_pre, f_pre) if data.config.get('matcher_type', "BRUTEFORCE") == "FLANN": index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index)
def detect(args): image, data = args print('detect') print(image) print(data) #log.setup() need_words = data.config[ 'matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0 has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: #logger.info('Skip recomputing {} features for image {}'.format(data.feature_type().upper(), image)) return #logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image)) #start = timer() p_unmasked, f_unmasked, c_unmasked = features.extract_features( data.load_image(image), data.config) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: #logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config['bow_words_to_match'] closest_words = bows.map_to_words(f_sorted, n_closest, data.config['bow_matcher_type']) data.save_words(image, closest_words)
def detect(feature_path, image_path, image, opensfm_config): log.setup() need_words = opensfm_config['matcher_type'] == 'WORDS' or opensfm_config[ 'matching_bow_neighbors'] > 0 #has_words = not need_words or data.words_exist(image) #has_features = data.features_exist(image) # if has_features and has_words: # logger.info('Skip recomputing {} features for image {}'.format( # data.feature_type().upper(), image)) # return #logger.info('Extracting {} features for image {}'.format(data.feature_type().upper(), image)) p_unmasked, f_unmasked, c_unmasked = features.extract_features( load_image(image_path), opensfm_config) #p_unmasked is points mask_files = defaultdict(lambda: None) fmask = load_features_mask(feature_path, image, image_path, p_unmasked, mask_files, opensfm_config) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: #logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] save_features(feature_path, opensfm_config, image, p_sorted, f_sorted, c_sorted) if need_words: bows = bow.load_bows(opensfm_config) n_closest = opensfm_config['bow_words_to_match'] closest_words = bows.map_to_words(f_sorted, n_closest, opensfm_config['bow_matcher_type']) save_words(feature_path, image_path, closest_words)
def extract_features(problem, data): """ Extract features from all images, and save to DataSet. """ assert 'masks' not in data.config for image in data.images(): if data.feature_index_exists(image): print "{} - extracting features (cached)".format(image) else: print "{} - extracting features".format(image) data.config['masks'] = problem.image2masks[image] points, descriptors, colors = features.extract_features( data.image_as_array(image), data.config) del data.config['masks'] data.save_features(image, points, descriptors, colors) index = features.build_flann_index(descriptors, data.config) data.save_feature_index(image, index)
def detect(args): log.setup() image, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) if not data.feature_index_exists(image): start = timer() mask = data.load_combined_mask(image) if mask is not None: logger.info('Found mask to apply for image {}'.format(image)) preemptive_max = data.config['preemptive_max'] p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.load_image(image), data.config, mask) if len(p_unsorted) == 0: return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] data.save_features(image, p_sorted, f_sorted, c_sorted) data.save_preemptive_features(image, p_pre, f_pre) if data.config['matcher_type'] == 'FLANN': index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args): log.setup() image, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) if not data.feature_index_exists(image): start = timer() mask = data.mask_as_array(image) if mask is not None: logger.info('Found mask to apply for image {}'.format(image)) preemptive_max = data.config['preemptive_max'] p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.image_as_array(image), data.config, mask) if len(p_unsorted) == 0: return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] data.save_features(image, p_sorted, f_sorted, c_sorted) data.save_preemptive_features(image, p_pre, f_pre) if data.config['matcher_type'] == 'FLANN': index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args): image, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) if not data.feature_index_exists(image): preemptive_max = data.config.get('preemptive_max', 200) p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.image_as_array(image), data.config) if len(p_unsorted) == 0: return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] data.save_features(image, p_sorted, f_sorted, c_sorted) data.save_preemptive_features(image, p_pre, f_pre) index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index)
def detect( image: str, image_array: np.ndarray, segmentation_array: Optional[np.ndarray], instances_array: Optional[np.ndarray], data: DataSetBase, force: bool = False, ) -> None: log.setup() need_words = ( data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0 ) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if not force and has_features and has_words: logger.info( "Skip recomputing {} features for image {}".format( data.feature_type().upper(), image ) ) return logger.info( "Extracting {} features for image {}".format(data.feature_type().upper(), image) ) start = timer() p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array) ) # Load segmentation and bake it in the data if data.config["features_bake_segmentation"]: exif = data.load_exif(image) s_unsorted, i_unsorted = bake_segmentation( image_array, p_unmasked, segmentation_array, instances_array, exif ) p_unsorted = p_unmasked f_unsorted = f_unmasked c_unsorted = c_unmasked # Load segmentation, make a mask from it mask and apply it else: s_unsorted, i_unsorted = None, None fmask = masking.load_features_mask(data, image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] if s_unsorted is not None: semantic_data = features.SemanticData( s_unsorted[order], i_unsorted[order] if i_unsorted is not None else None, data.segmentation_labels(), ) else: semantic_data = None features_data = features.FeaturesData(p_sorted, f_sorted, c_sorted, semantic_data) data.save_features(image, features_data) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words( f_sorted, n_closest, data.config["bow_matcher_type"] ) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args: Tuple[str, DataSetBase]): image, data = args log.setup() need_words = (data.config["matcher_type"] == "WORDS" or data.config["matching_bow_neighbors"] > 0) has_words = not need_words or data.words_exist(image) has_features = data.features_exist(image) if has_features and has_words: logger.info("Skip recomputing {} features for image {}".format( data.feature_type().upper(), image)) return logger.info("Extracting {} features for image {}".format( data.feature_type().upper(), image)) start = timer() image_array = data.load_image(image) p_unmasked, f_unmasked, c_unmasked = features.extract_features( image_array, data.config, is_high_res_panorama(data, image, image_array)) # Load segmentation and bake it in the data if data.config["features_bake_segmentation"]: exif = data.load_exif(image) panoptic_data = [None, None] for i, p_data in enumerate( [data.load_segmentation(image), data.load_instances(image)]): if p_data is None: continue new_height, new_width = p_data.shape ps = upright.opensfm_to_upright( p_unmasked[:, :2], exif["width"], exif["height"], exif["orientation"], new_width=new_width, new_height=new_height, ).astype(int) panoptic_data[i] = p_data[ps[:, 1], ps[:, 0]] s_unsorted, i_unsorted = panoptic_data p_unsorted = p_unmasked f_unsorted = f_unmasked c_unsorted = c_unmasked # Load segmentation, make a mask from it mask and apply it else: s_unsorted, i_unsorted = None, None fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning("No features found in image {}".format(image)) size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] # pyre-fixme[16]: `None` has no attribute `__getitem__`. s_sorted = s_unsorted[order] if s_unsorted is not None else None i_sorted = i_unsorted[order] if i_unsorted is not None else None data.save_features(image, p_sorted, f_sorted, c_sorted, s_sorted, i_sorted) if need_words: bows = bow.load_bows(data.config) n_closest = data.config["bow_words_to_match"] closest_words = bows.map_to_words(f_sorted, n_closest, data.config["bow_matcher_type"]) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def detect(args): image, tags, data = args logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) DEBUG = 0 # check if features already exist if not data.feature_index_exists(image): mask = data.mask_as_array(image) if mask is not None: logger.info('Found mask to apply for image {}'.format(image)) preemptive_max = data.config.get('preemptive_max', 200) p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.image_as_array(image), data.config, mask) if len(p_unsorted) == 0: return #===== prune features in tags =====# if data.config.get('prune_features_on_tags', False): # setup img = cv2.imread(os.path.join(data.data_path, 'images', image)) [height, width, _] = img.shape p_denorm = features.denormalized_image_coordinates( p_unsorted, width, height) # expand tag contour with grid points beyond unit square expn = 2.0 gridpts = np.array( [[-expn, expn], [expn, expn], [expn, -expn], [-expn, -expn]], dtype='float32') # find features to prune rm_list = [] for tag in tags: # contour from tag region contours = np.array(tag.corners) if DEBUG > 0: for i in range(0, 3): cv2.line(img, (tag.corners[i, 0].astype('int'), tag.corners[i, 1].astype('int')), (tag.corners[i + 1, 0].astype('int'), tag.corners[i + 1, 1].astype('int')), [0, 255, 0], 12) cv2.line(img, (tag.corners[3, 0].astype('int'), tag.corners[3, 1].astype('int')), (tag.corners[0, 0].astype('int'), tag.corners[0, 1].astype('int')), [0, 255, 0], 12) # scale contour outward H = np.array(tag.homography, dtype='float32') contours_expanded = cv2.perspectiveTransform( np.array([gridpts]), H) # for each point for pidx in range(0, len(p_unsorted)): # point pt = p_denorm[pidx, 0:2] # point in contour inout = cv2.pointPolygonTest( contours_expanded.astype('int'), (pt[0], pt[1]), False) # check result if inout >= 0: rm_list.append(pidx) # prune features p_unsorted = np.delete(p_unsorted, np.array(rm_list), axis=0) f_unsorted = np.delete(f_unsorted, np.array(rm_list), axis=0) c_unsorted = np.delete(c_unsorted, np.array(rm_list), axis=0) # debug if DEBUG > 0: p_denorm = np.delete(p_denorm, np.array(rm_list), axis=0) for pidx in range(0, len(p_denorm)): pt = p_denorm[pidx, 0:2] cv2.circle(img, (pt[0].astype('int'), pt[1].astype('int')), 5, [0, 0, 255], -1) cv2.namedWindow('ShowImage', cv2.WINDOW_NORMAL) height, width, channels = img.shape showw = max(752, width / 4) showh = max(480, height / 4) cv2.resizeWindow('ShowImage', showw, showh) cv2.imshow('ShowImage', img) cv2.waitKey(0) #===== prune features in tags =====# # sort for preemptive size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] # save data.save_features(image, p_sorted, f_sorted, c_sorted) data.save_preemptive_features(image, p_pre, f_pre) if data.config.get('matcher_type', 'FLANN') == 'FLANN': index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) #===== tag features =====# if data.config.get('use_apriltags', False) or data.config.get( 'use_arucotags', False) or data.config.get('use_chromatags', False): # setup try: tags_all = data.load_tag_detection() except: return tags = tags_all[image] pt = [] ft = [] ct = [] it = [] imexif = data.load_exif(image) # for each tag in image for tag in tags: # normalize corners img = cv2.imread(os.path.join(data.data_path, 'images', image)) [height, width, _] = img.shape #print 'width = ',str(imexif['width']) #print 'height= ',str(imexif['height']) #print 'width2= ',str(width) #print 'heigh2= ',str(height) norm_tag_corners = features.normalized_image_coordinates( tag.corners, width, height) #imexif['width'], imexif['height']) # for each corner of tag for r in range(0, 4): # tag corners pt.append(norm_tag_corners[r, :]) # tag id ft.append(tag.id) # colors ct.append(tag.colors[r, :]) # corner id (0,1,2,3) it.append(r) # if tag features found if pt: pt = np.array(pt) ft = np.array(ft) ct = np.array(ct) it = np.array(it) data.save_tag_features(image, pt, ft, it, ct)
def detect(self, args): image, data = args log.setup() print() print(data) self.feature_pdc = {} need_words = data.config['matcher_type'] == 'WORDS' or data.config[ 'matching_bow_neighbors'] > 0 logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) start = timer() #print(image)## 1.jpg p_unmasked, f_unmasked, c_unmasked = features.extract_features( data.load_image(image), data.config) #print(p_unmasked, f_unmasked, c_unmasked) fmask = data.load_features_mask(image, p_unmasked) p_unsorted = p_unmasked[fmask] f_unsorted = f_unmasked[fmask] c_unsorted = c_unmasked[fmask] if len(p_unsorted) == 0: logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] # points == numpy.ndarray f_sorted = f_unsorted[order, :] # descriptors c_sorted = c_unsorted[order, :] # colors #data.save_features(image, p_sorted, f_sorted, c_sorted) points = p_sorted.astype(np.float32) descriptors = f_sorted.astype(np.uint8) colors = c_sorted OPENSFM_FEATURES_VERSION = 1 self.feature_pdc.update({'points': points}) self.feature_pdc.update({'descriptors': descriptors}) self.feature_pdc.update({'colors': colors}) self.feature_pdc.update( {'OPENSFM_FEATURES_VERSION': OPENSFM_FEATURES_VERSION}) # self.feature_pdc.update({'points':p_sorted}) # self.feature_pdc.update({'descriptors':f_sorted}) # self.feature_pdc.update({'colors':c_sorted}) self.feature_of_images.update({image: self.feature_pdc}) print("self.feature_of_images==", hex(id(self.feature_of_images))) print(image) # if need_words: # bows = bow.load_bows(data.config) # n_closest = data.config['bow_words_to_match'] # closest_words = bows.map_to_words( # f_sorted, n_closest, data.config['bow_matcher_type']) # data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report_of_features(image, report) data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))
def detect(args): id = current_process()._identity prefix = "process %s: " % str(id[0]) if len(id) > 0 else "" image, data = args if data.features_exist(image): return print prefix + "detecting features for image %s" % image logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) # Detect features # Retrieve image mask mask = data.mask_as_array(image) if mask is not None: print prefix + "found mask for image: %s" % image logger.info('Found mask to apply for image {}'.format(image)) # Obtain segmentation path path_seg = data.data_path + "/images/output/results/frontend_vgg/" + os.path.splitext( image)[0] + '.png' else: print prefix + "not found mask for image %s" % image path_seg = None preemptive_max = data.config.get('preemptive_max', 200) the_image = data.image_as_array(image) # Extract features print prefix + "extracting features from image %s" % image save_no_mask = False all_content = features.extract_features(the_image, data.config, mask, save_no_mask, path_seg) if save_no_mask: p_unsorted, f_unsorted, c_unsorted = all_content[0] p_nomask, f_nomask, c_nomask = all_content[1] else: p_unsorted, f_unsorted, c_unsorted = all_content if len(p_unsorted) == 0: print prefix + "exit" return # Save features to file print prefix + "saving features" size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if data.config.get('preemptive_threshold', 0) > 0: p_pre = p_sorted[-preemptive_max:] f_pre = f_sorted[-preemptive_max:] data.save_preemptive_features(image, p_pre, f_pre) # Prepare FLANN matching if necessary if data.config.get('matcher_type', "BRUTEFORCE") == "FLANN": index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) # Done print prefix + "exit"
def detect(args): log.setup() image, data = args need_words = data.config['matcher_type'] == 'WORDS' or data.config['matching_bow_neighbors'] > 0 need_flann = data.config['matcher_type'] == 'FLANN' has_words = not need_words or data.words_exist(image) has_flann = not need_flann or data.feature_index_exists(image) has_features = data.features_exist(image) if has_features and has_flann and has_words: logger.info('Skip recomputing {} features for image {}'.format( data.feature_type().upper(), image)) return logger.info('Extracting {} features for image {}'.format( data.feature_type().upper(), image)) start = timer() exif = data.load_exif( image ) camera_models = data.load_camera_models() image_camera_model = camera_models[ exif[ 'camera' ] ] if image_camera_model.projection_type in ['equirectangular', 'spherical'] and data.config['matching_unfolded_cube']: logger.info('Features unfolded cube.') # For spherical cameras create an undistorted image for the purposes of # feature finding (and later matching). max_size = data.config.get('ai_process_size', -1) if max_size == -1: max_size = img.shape[1] img = data.load_image( image ) undist_tile_size = max_size//4 undist_img = np.zeros( (max_size//2, max_size, 3 ), np.uint8 ) undist_mask = np.full( (max_size//2, max_size, 1 ), 255, np.uint8 ) undist_mask[ undist_tile_size:2*undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = 0 undist_mask[ undist_tile_size:2*undist_tile_size, undist_tile_size:2*undist_tile_size ] = 0 # The bottom mask to remove the influence of the camera person should be configurable. It depends on the forward # direction of the camera and where the camera person positions themselves in relation to this direction. It'save_feature_index # probably worth it to take care with this because the floor could help hold the reconstructions together. #undist_mask[ 5*undist_tile_size//4:7*undist_tile_size//4, undist_tile_size//3:undist_tile_size ] = 0 #undist_mask[ 3*undist_tile_size//2:2*undist_tile_size, undist_tile_size//2:undist_tile_size ] = 0 spherical_shot = types.Shot() spherical_shot.pose = types.Pose() spherical_shot.id = image spherical_shot.camera = image_camera_model perspective_shots = undistort.perspective_views_of_a_panorama( spherical_shot, undist_tile_size ) for subshot in perspective_shots: undistorted = undistort.render_perspective_view_of_a_panorama( img, spherical_shot, subshot ) subshot_id_prefix = '{}_perspective_view_'.format( spherical_shot.id ) subshot_name = subshot.id[ len(subshot_id_prefix): ] if subshot.id.startswith( subshot_id_prefix ) else subshot.id ( subshot_name, ext ) = os.path.splitext( subshot_name ) if subshot_name == 'front': undist_img[ :undist_tile_size, :undist_tile_size ] = undistorted #print( 'front') elif subshot_name == 'left': undist_img[ :undist_tile_size, undist_tile_size:2*undist_tile_size ] = undistorted #print( 'left') elif subshot_name == 'back': undist_img[ :undist_tile_size, 2*undist_tile_size:3*undist_tile_size ] = undistorted #print( 'back') elif subshot_name == 'right': undist_img[ :undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted #print( 'right') elif subshot_name == 'top': undist_img[ undist_tile_size:2*undist_tile_size, 3*undist_tile_size:4*undist_tile_size ] = undistorted #print( 'top') elif subshot_name == 'bottom': undist_img[ undist_tile_size:2*undist_tile_size, :undist_tile_size ] = undistorted #print( 'bottom') #data.save_undistorted_image(subshot.id, undistorted) data.save_undistorted_image(image.split(".")[0], undist_img) # We might consider combining a user supplied mask here as well undist_img = resized_image(undist_img, data.config) p_unsorted, f_unsorted, c_unsorted = features.extract_features(undist_img, data.config, undist_mask) # Visualize the features on the unfolded cube # -------------------------------------------------------------- if False: h_ud, w_ud, _ = undist_img.shape denorm_ud = denormalized_image_coordinates( p_unsorted[:, :2], w_ud, h_ud ) print( p_unsorted.shape ) print( denorm_ud.shape ) rcolors = [] for point in denorm_ud: color = np.random.randint(0,255,(3)).tolist() cv2.circle( undist_img, (int(point[0]),int(point[1])), 1, color, -1 ) rcolors.append( color ) data.save_undistorted_image( image + '_unfolded_cube.jpg', undist_img) # -------------------------------------------------------------- if len(p_unsorted) > 0: # Mask pixels that are out of valid image bounds before converting to equirectangular image coordinates bearings = image_camera_model.unfolded_pixel_bearings( p_unsorted[:, :2] ) p_mask = np.array([ point is not None for point in bearings ]) p_unsorted = p_unsorted[ p_mask ] f_unsorted = f_unsorted[ p_mask ] c_unsorted = c_unsorted[ p_mask ] p_unsorted[:, :2] = unfolded_cube_to_equi_normalized_image_coordinates( p_unsorted[:, :2], image_camera_model ) # Visualize the same features converted back to equirectangular image coordinates # ----------------------------------------------------------------------------------------- if False: timg = resized_image( img, data.config ) h, w, _ = timg.shape denorm = denormalized_image_coordinates( p_unsorted[:, :2], w, h ) for ind, point in enumerate( denorm ): cv2.circle( timg, (int(point[0]),int(point[1])), 1, rcolors[ind], -1 ) data.save_undistorted_image('original.jpg', timg) #------------------------------------------------------------------------------------------ else: mask = data.load_combined_mask(image) if mask is not None: logger.info('Found mask to apply for image {}'.format(image)) p_unsorted, f_unsorted, c_unsorted = features.extract_features( data.load_image(image), data.config, mask) if len(p_unsorted) == 0: logger.warning('No features found in image {}'.format(image)) return size = p_unsorted[:, 2] order = np.argsort(size) p_sorted = p_unsorted[order, :] f_sorted = f_unsorted[order, :] c_sorted = c_unsorted[order, :] data.save_features(image, p_sorted, f_sorted, c_sorted) if need_flann: index = features.build_flann_index(f_sorted, data.config) data.save_feature_index(image, index) if need_words: bows = bow.load_bows(data.config) n_closest = data.config['bow_words_to_match'] closest_words = bows.map_to_words( f_sorted, n_closest, data.config['bow_matcher_type']) data.save_words(image, closest_words) end = timer() report = { "image": image, "num_features": len(p_sorted), "wall_time": end - start, } data.save_report(io.json_dumps(report), 'features/{}.json'.format(image))