def compute_num_inliers(res_dict, cfg): '''Compile match numbers at different stages into the dictionary.''' # if cfg.method_dict['config_{}_{}'.format(cfg.dataset, # cfg.task)]['use_custom_matches']: # raise NotImplementedError( # 'Probably read only once? What to do with runs?') # Load pre-computed pairs with the new visibility criteria data_dir = get_data_path(cfg) pairs_per_th = get_pairs_per_threshold(data_dir) stereo_thresholds = list(pairs_per_th.keys()) epipolar_err_dict = {} # Load epipolar error file for th in [None] + stereo_thresholds: epipolar_err_dict['matcher'] = load_h5( get_stereo_epipolar_pre_match_file(cfg, th)) epipolar_err_dict['filter'] = load_h5( get_stereo_epipolar_refined_match_file(cfg, th)) epipolar_err_dict['geom'] = load_h5( get_stereo_epipolar_final_match_file(cfg, th)) for key_stage, values1 in epipolar_err_dict.items(): # Simply return average of all pairs num_matches = [] for key, values2 in values1.items(): num_matches.append(len(values2)) # Save the number of inliers vis_label = '' if th is None else '_th_{}'.format(th) res_dict['num_matches_{}{}'.format(key_stage, vis_label)] = float( np.mean(num_matches) if len(num_matches) > 0 else 0)
def compute_matching_scores_depth_projection(res_dict, cfg): '''Compute matching scores (with depth) and add them to the dictionary.''' px_th_list = cfg.matching_score_and_repeatability_px_threshold # Load pre-computed pairs with the new visibility criteria data_dir = get_data_path(cfg) pairs_per_th = get_pairs_per_threshold(data_dir) stereo_thresholds = list(pairs_per_th.keys()) reprojection_err_dict = {} # Load epipolar error file for th in [None] + stereo_thresholds: reprojection_err_dict['pre_match'] = load_h5( get_stereo_depth_projection_pre_match_file(cfg, th)) reprojection_err_dict['refined_match'] = load_h5( get_stereo_depth_projection_refined_match_file(cfg, th)) reprojection_err_dict['final_match'] = load_h5( get_stereo_depth_projection_final_match_file(cfg, th)) for key_stage, values1 in reprojection_err_dict.items(): acc = [] for px_th in px_th_list: ms = [] # Simply return average of all pairs for key, values2 in values1.items(): if len(values2) > 0: ms += [np.mean(values2 < px_th)] else: ms += [0] acc += [float(np.mean(ms) if len(ms) > 0 else 0)] # Now compute average number of keypoints vis_label = '' if th is None else '_th_{}'.format(th) res_dict['matching_scores_depth_projection_{}{}'.format( key_stage, vis_label)] = acc
def main(cfg): '''Main function to compute matches. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' if os.path.exists(get_match_file(cfg)): print(' -- already exists, skipping match computation') return # Get data directory data_dir = get_data_path(cfg) # Load pre-computed pairs with the new visibility criteria print('Reading list of all possible pairs') pairs = get_pairs_per_threshold(data_dir)['0.0'] print('{} pre-computed pairs'.format(len(pairs))) # Load descriptors descriptors_dict = load_h5(get_desc_file(cfg)) keypoints_dict = load_h5(get_kp_file(cfg)) # Feature Matching print('Computing matches') num_cores = cfg.num_opencv_threads if cfg.num_opencv_threads > 0 else int( len(os.sched_getaffinity(0)) * 0.9) if WITH_FAISS: num_cores = min(4, num_cores) result = Parallel(n_jobs=num_cores)( delayed(compute_matches)(np.asarray(descriptors_dict[pair.split( '-')[0]]), np.asarray(descriptors_dict[pair.split( '-')[1]]), cfg, np.asarray(keypoints_dict[pair.split( '-')[0]]), np.asarray(keypoints_dict[pair.split('-')[1]])) for pair in tqdm(pairs)) # Make match dictionary matches_dict = {} timings_list = [] for i, pair in enumerate(pairs): matches_dict[pair] = result[i][0] timings_list.append(result[i][1]) # Check match directory if not os.path.exists(get_match_path(cfg)): os.makedirs(get_match_path(cfg)) # Finally save packed matches save_h5(matches_dict, get_match_file(cfg)) # Save computational cost save_h5({'cost': np.mean(timings_list)}, get_match_cost_file(cfg)) print('Matching cost (averaged over image pairs): {:0.2f} sec'.format( np.mean(timings_list)))
def compute_repeatability(res_dict, cfg): '''Compute repeatability and add it to the dictionary.''' px_th_list = cfg.matching_score_and_repeatability_px_threshold # Load pre-computed pairs with the new visibility criteria data_dir = get_data_path(cfg) pairs_per_th = get_pairs_per_threshold(data_dir) stereo_thresholds = list(pairs_per_th.keys()) # Load epipolar error file for th in [None] + stereo_thresholds: ms_list_list = [[] for i in range(len(px_th_list))] repeatability_dict = load_h5(get_repeatability_score_file(cfg, th)) for key, values in repeatability_dict.items(): # Simply return average of all pairs for idx in range(len(px_th_list)): ms_list_list[idx] += [values[idx]] # Now compute average number of keypoints acc = [] for px_th, ms_list in zip(px_th_list, ms_list_list): acc += [float(np.mean(ms_list) if len(ms_list) > 0 else 0)] vis_label = '' if th is None else '_th_{}'.format(th) res_dict['repeatability{}'.format(vis_label)] = acc
def compute_timings(res_dict, cfg): '''Compile timings (if any) into the dictionary.''' # Load cost files try: cost_match = load_h5(get_match_cost_file(cfg)) res_dict['match_cost'] = cost_match['cost'] except Exception: res_dict['match_cost'] = 0 try: cost_filter = load_h5(get_filter_cost_file(cfg)) res_dict['filter_cost'] = cost_filter['cost'] except Exception: res_dict['filter_cost'] = 0 try: cost_geom = load_h5(get_geom_cost_file(cfg)) res_dict['geom_cost'] = cost_geom['cost'] except Exception: res_dict['geom_cost'] = 0
def save_match_inlier(sfm_cfg, key_list, mask_dict): match_dict = load_h5(get_match_file(sfm_cfg)) if len(match_dict) != len(mask_dict): raise RuntimeError('Number of pairs from CNe output is different ' 'from original data!') for key, match_mask in mask_dict.items(): mask_index = np.where(match_mask) match_idx_pairs_inlier = match_dict[key_list[key]][:, mask_index] match_dict[key_list[key]] = np.squeeze(match_idx_pairs_inlier) save_h5(match_dict, get_filter_match_file(sfm_cfg))
def compute_avg_num_keypoints(res_dict, cfg): '''Compute the average number of keypoints and add it to the dictionary.''' # Load keypoints file keypoints_dict = load_h5(get_kp_file(cfg)) # Simply return average of all keypoints per image num_kp_list = [] for key, values in keypoints_dict.items(): num_kp_list += [len(values)] # Now compute average number of keypoints res_dict['avg_num_keypoints'] = float(np.mean(num_kp_list))
def compute_matching_scores_epipolar(res_dict, cfg): '''Compute Matching Scores (with calib) and add them to the dictionary.''' # Load pre-computed pairs with the new visibility criteria data_dir = get_data_path(cfg) pairs_per_th = get_pairs_per_threshold(data_dir) stereo_thresholds = list(pairs_per_th.keys()) epipolar_err_dict = {} # Load epipolar error file values = {} for th in [None] + stereo_thresholds: # Init empty list epipolar_err_dict['pre_match'] = load_h5( get_stereo_epipolar_pre_match_file(cfg, th)) epipolar_err_dict['refined_match'] = load_h5( get_stereo_epipolar_refined_match_file(cfg, th)) epipolar_err_dict['final_match'] = load_h5( get_stereo_epipolar_final_match_file(cfg, th)) for key_stage, values1 in epipolar_err_dict.items(): if key_stage not in values: values[key_stage] = [] # Simply return average of all pairs ms_list = [] for key, values2 in values1.items(): if len(values2) > 0: ms_list += [ np.mean( values2 < cfg.matching_score_epipolar_threshold) ] else: ms_list += [0] # Now compute average number of keypoints vis_label = '' if th is None else '_th_{}'.format(th) values[key_stage].append( float(np.mean(ms_list) if len(ms_list) > 0 else 0))
def load_calib(calib_fullpath_list, subset_index=None): '''Load all calibration files and create a dictionary.''' calib = {} if subset_index is None: for _calib_file in calib_fullpath_list: img_name = os.path.splitext( os.path.basename(_calib_file))[0].replace('calibration_', '') # _calib_file.split( # '/')[-1].replace('calibration_', '')[:-3] # # Don't know why, but rstrip .h5 also strips # # more then necssary sometimes! # # # # img_name = _calib_file.split( # # '/')[-1].replace('calibration_', '').rstrip('.h5') calib[img_name] = load_h5(_calib_file) else: for idx in subset_index: _calib_file = calib_fullpath_list[idx] img_name = os.path.splitext( os.path.basename(_calib_file))[0].replace('calibration_', '') calib[img_name] = load_h5(_calib_file) return calib
def compute_qt_auc_colmap(res_dict, cfg): '''Compute pose accuracy (multiview) and add it to the dictionary.''' qt_acc_list = [] # For all the bags cfg_bag = deepcopy(cfg) qt_hist_list = np.empty([0, 10]) for bag_id in range(get_num_in_bag(cfg_bag)): cfg_bag.bag_id = bag_id # Load pose error for colmap pose_err_dict = load_h5(get_colmap_pose_file(cfg_bag)) # Gather err_q, err_t err_qt = [] for key, value in pose_err_dict.items(): err_qt += [value] err_qt = np.asarray(err_qt) # Take the maximum among q and t errors err_qt = np.max(err_qt, axis=1) # Convert to degree err_qt = err_qt * 180.0 / np.pi # Make infs to a large value so that np.histogram can be used. err_qt[err_qt == np.inf] = 1e6 # Create histogram bars = np.arange(11) qt_hist, _ = np.histogram(err_qt, bars) # Normalize histogram with all possible pairs (note that we already # have error results filled in with infs if necessary, thus we don't # have to worry about them) num_pair = float(len(err_qt)) qt_hist = qt_hist.astype(float) / num_pair # Make cumulative and store to list qt_acc_list += [np.cumsum(qt_hist)] qt_hist_list = np.concatenate( (qt_hist_list, np.expand_dims(qt_hist, axis=0)), axis=0) # Aggregate all results qt_acc = np.mean(qt_acc_list, axis=0) qt_hist = np.squeeze(np.mean(qt_hist_list, axis=0)) # Save to dictionary res_dict['qt_colmap_01_10'] = qt_hist.tolist() res_dict['qt_auc_colmap_05'] = np.mean(qt_acc[:5]) res_dict['qt_auc_colmap_10'] = np.mean(qt_acc)
def compute_qt_auc(res_dict, cfg): '''Compute pose accuracy (stereo) and add it to the dictionary.''' # Load pre-computed pairs with the new visibility criteria data_dir = get_data_path(cfg) pairs_per_th = get_pairs_per_threshold(data_dir) stereo_thresholds = list(pairs_per_th.keys()) # Load pose error for stereo for th in [None] + stereo_thresholds: pose_err_dict = load_h5(get_stereo_pose_file(cfg, th)) # Gather err_q, err_t err_qt = [] for key, value in pose_err_dict.items(): err_qt += [value] if len(err_qt) > 0: err_qt = np.asarray(err_qt) # Take the maximum among q and t errors err_qt = np.max(err_qt, axis=1) # Convert to degree err_qt = err_qt * 180.0 / np.pi # Make infs to a large value so that np.histogram can be used. err_qt[err_qt == np.inf] = 1e6 # Create histogram bars = np.arange(11) qt_hist, _ = np.histogram(err_qt, bars) # Normalize histogram with all possible pairs num_pair = float(len(err_qt)) qt_hist = qt_hist.astype(float) / num_pair # Make cumulative qt_acc = np.cumsum(qt_hist) else: qt_acc = [0] * 10 # Save to dictionary label = '' if th is None else '_th_{}'.format(th) res_dict['qt_01_10{}'.format(label)] = qt_hist.tolist() res_dict['qt_auc_05{}'.format(label)] = np.mean(qt_acc[:5]) res_dict['qt_auc_10{}'.format(label)] = np.mean(qt_acc)
def compute_num_input_matches(res_dict, cfg): '''Save the number of input matches given to Colmap.''' # TODO fix this after re-implementing custom matches # if cfg.method_dict['config_{}_{}'.format(cfg.dataset, # cfg.task)]['use_custom_matches']: # raise NotImplementedError( # 'TODO Load the right dict with custom matches') # Read match dict matches_dict = load_h5(get_filter_match_file(cfg)) # For every bag, compute the number of matches going into colmap bag_size_json = load_json( getattr(cfg, 'splits_{}_{}'.format(cfg.dataset, cfg.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] # Average it per bag size first, then across all bag sizes num_input_matches = [] for bag_size, cur_bag_size_num in zip(bag_size_list, bag_size_num): num_input_matches_bagsize = [] for bag_id in range(cur_bag_size_num): cfg_bag = deepcopy(cfg) cfg_bag.bag_size = bag_size cfg_bag.bag_id = bag_id images = get_colmap_image_path_list(cfg_bag) keys = [os.path.splitext(os.path.basename(im))[0] for im in images] pairs = [] for i in range(len(keys)): for j in range(i + 1, len(keys)): pairs.append('-'.join( sorted([keys[i], keys[j]], reverse=True))) for pair in pairs: num_input_matches_bagsize.append(matches_dict[pair].shape[-1]) num_input_matches.append(np.mean(num_input_matches_bagsize)) res_dict['num_input_matches'] = np.mean(num_input_matches)
def load_depth(depth_path): return load_h5(depth_path)['depth']
def load_h5_valid_image(path, deprecated_images): return remove_keys(load_h5(path), deprecated_images)
def main(cfg): '''Main function. Takes config as input. ''' # Back up config cfg_orig = deepcopy(cfg) method = cfg_orig.method_dict # Add config options to the dict master_dict = OrderedDict() master_dict['config'] = method # Add date master_dict['properties'] = OrderedDict() master_dict['properties'][ 'processing_date'] = pack_helper.get_current_date() print('Adding processing date: {}'.format( master_dict['properties']['processing_date'])) # Add submission flag master_dict['properties']['is_submission'] = cfg.is_submission print('Flagging as user submission: {}'.format(cfg.is_submission)) # Add descriptor properties cfg_desc = deepcopy(cfg_orig) cfg_desc.dataset = 'phototourism' cfg_desc.scene = 'british_museum' try: descriptors_dict = load_h5(get_desc_file(cfg_desc)) desc_type, desc_size, desc_nbytes = pack_helper.get_descriptor_properties( cfg_desc, descriptors_dict) except Exception: desc_type = 'none' desc_size = 0 desc_nbytes = 0 master_dict['properties']['descriptor_type'] = desc_type master_dict['properties']['descriptor_size'] = desc_size master_dict['properties']['descriptor_nbytes'] = desc_nbytes print('Adding descriptor properties: {} {} ({} bytes)'.format( master_dict['properties']['descriptor_size'], master_dict['properties']['descriptor_type'], master_dict['properties']['descriptor_nbytes'])) deprecated_images_all = load_json(cfg.json_deprecated_images) if cfg.dataset in deprecated_images_all and cfg.scene in deprecated_images_all[ cfg.dataset]: deprecated_images = deprecated_images_all[cfg.dataset][cfg.scene] else: deprecated_images = [] # Read data and splits DATASET_LIST = ['phototourism', 'pragueparks', 'googleurban'] for dataset in DATASET_LIST: # Skip if not in config if 'config_{}_stereo'.format( dataset) not in method and 'config_{}_multiview'.format( dataset) not in method: continue # Create empty dictionary master_dict[dataset] = OrderedDict() res_dict = OrderedDict() master_dict[dataset]['results'] = res_dict # Save number of runs master_dict[dataset]['num_runs_stereo'] = getattr( cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) master_dict[dataset]['num_runs_multiview'] = getattr( cfg_orig, 'num_runs_{}_multiview'.format(cfg_orig.subset)) # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] bag_size_str = ['{}bag'.format(b) for b in bag_size_list] # Create empty dicts for scene in ['allseq'] + scene_list: res_dict[scene] = OrderedDict() for task in ['stereo', 'multiview']: res_dict[scene][task] = OrderedDict() res_dict[scene][task]['run_avg'] = OrderedDict() if task == 'multiview': for bag in bag_size_str + ['bag_avg']: res_dict[scene]['multiview']['run_avg'][ bag] = OrderedDict() # Stereo -- multiple runs t = time() cur_key = 'config_{}_stereo'.format(dataset) if cfg_orig.eval_stereo and cur_key in method and method[cur_key]: num_runs = getattr(cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'stereo' for scene in scene_list: cfg.scene = scene res_dict[scene]['stereo']['run_avg'] = OrderedDict() for run in range(num_runs): res_dict[scene]['stereo']['run_{}'.format( run)] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] # metric_list += ['matching_scores_epipolar'] metric_list += ['num_inliers'] if dataset != 'googleurban': metric_list += ['matching_scores_depth_projection'] metric_list += ['repeatability'] metric_list += ['qt_auc'] metric_list += ['timings'] for run in range(num_runs): # Compute and pack results cfg.run = run cur_dict = res_dict[scene]['stereo']['run_{}'.format(run)] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)(cur_dict, deprecated_images, cfg) print( ' -- Packing "{}"/"{}"/stereo, run: {}/{}, metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, metric, time() - t_cur)) # Compute average across runs, for stereo t_cur = time() pack_helper.average_stereo_over_runs(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for stereo t_cur = time() pack_helper.average_stereo_over_scenes(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing stereo in {:.01f} sec.'.format(time() - t)) else: print('Skipping "{}/stereo"'.format(dataset)) # Multiview -- multiple runs t = time() cur_key = 'config_{}_multiview'.format(dataset) if cfg_orig.eval_multiview and cur_key in method and method[cur_key]: num_runs = getattr(cfg, 'num_runs_{}_multiview'.format(cfg.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'multiview' for scene in scene_list: cfg.scene = scene for run in ['run_avg' ] + ['run_{}'.format(f) for f in range(num_runs)]: res_dict[scene]['multiview'][run] = OrderedDict() for bags_label in ['bag_avg'] + bag_size_str: res_dict[scene]['multiview'][run][ bags_label] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] metric_list += ['num_input_matches'] metric_list += ['qt_auc_colmap'] metric_list += ['ATE'] metric_list += ['colmap_stats'] for run in range(num_runs): for bag_size in bag_size_list: # Compute and pack results cfg.run = run cfg.bag_size = bag_size cur_dict = res_dict[scene]['multiview'] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)( cur_dict['run_{}'.format(run)]['{}bag'.format( bag_size)], deprecated_images, cfg) print( ' -- Packing "{}"/"{}"/multiview, run {}/{}, "{}", metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, '{}bag'.format(bag_size), metric, time() - t_cur)) # Compute average across bags any_key = random.choice([ key for key in cur_dict['run_{}'.format(run)] if ('bag' in key and key != 'bag_avg') ]) for metric in cur_dict['run_{}'.format(run)][any_key]: pack_helper.average_multiview_over_bags( cfg, cur_dict['run_{}'.format(run)], bag_size_list) # Compute average across runs, for multiview t_cur = time() pack_helper.average_multiview_over_runs(cfg, res_dict, num_runs, bag_size_str + ['bag_avg']) print( ' -- Packing "{}"/multiview: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for multiview t_cur = time() pack_helper.average_multiview_over_scenes( cfg, res_dict, num_runs, ['bag_avg'] + bag_size_str) print( ' -- Packing "{}"/multiview: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing multiview in {:.01f} sec.'.format( time() - t)) else: print('Skipping "{}/multiview"'.format(dataset)) # Add a unique identifier (equivalent to "submission id" in previous versions. if cfg.is_challenge: master_dict['uuid'] = get_uuid(cfg) # Dump packed result if not os.path.exists(cfg.path_pack): os.makedirs(cfg.path_pack) json_dump_file = os.path.join( cfg.path_pack, '{}.json'.format(cfg.method_dict['config_common']['json_label'])) print(' -- Saving to: "{}"'.format(json_dump_file)) with open(json_dump_file, 'w') as outfile: json.dump(master_dict, outfile, indent=2) # Add a short results summary. print() print('-- SUMMARY --') print('Subset: "{}"'.format(cfg.subset)) for dataset in DATASET_LIST: print() print('Dataset "{}"'.format(dataset)) if dataset in master_dict: # Stereo if 'stereo' in master_dict[dataset]['results'][ 'allseq'] and cfg.eval_stereo: print('-- Stereo mAA(10 deg): {:.05f}'.format( master_dict[dataset]['results']['allseq']['stereo'] ['run_avg']['qt_auc_10_th_0.1']['mean'])) for scene in master_dict[dataset]['results']: if scene != 'allseq': print('---- Scene "{}" -> Stereo mAA(10 deg): {:.05f}'. format( scene, master_dict[dataset]['results'][scene] ['stereo']['run_avg']['qt_auc_10_th_0.1'] ['mean'])) if 'multiview' in master_dict[dataset]['results'][ 'allseq'] and cfg.eval_multiview: print('-- Multiview mAA(10 deg): {:.05f}'.format( master_dict[dataset]['results']['allseq']['multiview'] ['run_avg']['bag_avg']['qt_auc_colmap_10']['mean'])) for scene in master_dict[dataset]['results']: if scene != 'allseq': print( '---- Scene "{}" -> Multiview mAA(10 deg): {:.05f}' .format( scene, master_dict[dataset]['results'][scene] ['multiview']['run_avg']['bag_avg'] ['qt_auc_colmap_10']['mean']))
for method in method_list: label = method['config_common']['json_label'] export_root = Path('../submission', label) export_root.mkdir(parents=True) cfg.method_dict = deepcopy(method) for seq in scene_list: print('Working on {}: {}/{}'.format(label, cfg.dataset, seq)) (export_root / seq).mkdir() for n in ['keypoints.h5', 'descriptors.h5', 'scores.h5']: copyfile(cfg.import_path / seq / n, export_root / seq / n) mpath = cfg.import_path / seq / 'matches.h5' copyfile(mpath, export_root / seq / 'matches_multiview.h5') keypoints_dict = load_h5(cfg.import_path / seq / 'keypoints.h5') matches_dict = load_h5(mpath) pairs = list(matches_dict.keys()) cfg.task = 'stereo' for run in range(3): print('Run {}'.format(run)) random.shuffle(pairs) calib = {'K': np.eye(3)} names = [p.split('-') for p in pairs] result = Parallel(n_jobs=num_cores)( delayed(compute_model)(cfg, np.asarray(matches_dict[pair]), np.asarray(keypoints_dict[n0]), np.asarray(keypoints_dict[n1]), calib, calib, None, None)
def main(cfg): '''Visualization of colmap points. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' bag_size_json = load_json( getattr(cfg, 'splits_{}_{}'.format(cfg.dataset, cfg.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] # # Do not re-run if files already exist -- off for now # skip = True # for _bag_size in bag_size_list: # cfg_bag = deepcopy(cfg) # cfg_bag.bag_size = _bag_size # viz_folder_hq, viz_folder_lq = get_colmap_viz_folder(cfg_bag) # for _bag_id in range( # getattr(cfg_bag, # 'num_viz_colmap_subsets_bagsize{}'.format(_bag_size))): # if any([ # not os.path.exists( # os.path.join( # viz_folder_lq, # 'colmap-bagsize{:d}-bag{:02d}-image{:02d}.jpg'. # format(_bag_size, _bag_id, i))) # for i in range(_bag_size) # ]): # skip = False # break # if not os.path.exists( # os.path.join( # viz_folder_lq, # 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( # _bag_size, _bag_id))): # skip = False # break # if skip: # print(' -- already exists, skipping colmap visualization') # return print(' -- Visualizations, multiview: "{}/{}"'.format( cfg.dataset, cfg.scene)) t_start = time() # Create results folder if it does not exist for _bag_size in bag_size_list: cfg_bag = deepcopy(cfg) cfg_bag.bag_size = _bag_size viz_folder_hq, viz_folder_lq = get_colmap_viz_folder(cfg_bag) if not os.path.exists(viz_folder_hq): os.makedirs(viz_folder_hq) if not os.path.exists(viz_folder_lq): os.makedirs(viz_folder_lq) # Load keypoints keypoints_dict = load_h5(get_kp_file(cfg)) # Loop over bag sizes for _bag_size in bag_size_list: cfg_bag = deepcopy(cfg) cfg_bag.bag_size = _bag_size num_bags = getattr( cfg_bag, 'num_viz_colmap_subsets_bagsize{}'.format(_bag_size)) for _bag_id in range(num_bags): print( ' -- Visualizations, multiview: "{}/{}", bag_size={}, bag {}/{}' .format(cfg.dataset, cfg.scene, _bag_size, _bag_id + 1, num_bags)) # Retrieve list of images cfg_bag.bag_id = _bag_id images_in_bag = get_colmap_image_path_list(cfg_bag) # Retrieve reconstruction colmap_output_path = get_colmap_output_path(cfg_bag) # is_colmap_valid = os.path.exists( # os.path.join(colmap_output_path, '0')) best_index = get_best_colmap_index(cfg_bag) if best_index != -1: colmap_images = read_images_binary( os.path.join(colmap_output_path, str(best_index), 'images.bin')) for i, image_path in enumerate(images_in_bag): # Limit to 10 or so, even for bag size 25 if i >= cfg.max_num_images_viz_multiview: break # Load image and keypoints im, _ = load_image(image_path, use_color_image=True, crop_center=False, force_rgb=True) used = None key = os.path.splitext(os.path.basename(image_path))[0] if best_index != -1: for j in colmap_images: if key in colmap_images[j].name: # plot all keypoints used = colmap_images[j].point3D_ids != -1 break if used is None: used = [False] * keypoints_dict[key].shape[0] used = np.array(used) fig = plt.figure(figsize=(20, 20)) plt.imshow(im) plt.plot(keypoints_dict[key][~used, 0], keypoints_dict[key][~used, 1], 'r.', markersize=12) plt.plot(keypoints_dict[key][used, 0], keypoints_dict[key][used, 1], 'b.', markersize=12) plt.tight_layout() plt.axis('off') # TODO Ideally we would save to pdf # but it does not work on 16.04, so we do png instead # https://bugs.launchpad.net/ubuntu/+source/imagemagick/+bug/1796563 viz_folder_hq, viz_folder_lq = get_colmap_viz_folder(cfg_bag) viz_file_hq = os.path.join( viz_folder_hq, 'bagsize{:d}-bag{:02d}-image{:02d}.png'.format( _bag_size, _bag_id, i)) viz_file_lq = os.path.join( viz_folder_lq, 'bagsize{:d}-bag{:02d}-image{:02d}.jpg'.format( _bag_size, _bag_id, i)) plt.savefig(viz_file_hq, bbox_inches='tight') # Convert with imagemagick os.system('convert -quality 75 -resize \"400>\" {} {}'.format( viz_file_hq, viz_file_lq)) plt.close() if best_index != -1: colmap_points = read_points3d_binary( os.path.join(colmap_output_path, str(best_index), 'points3D.bin')) points3d = [] for k in colmap_points: points3d.append([ colmap_points[k].xyz[0], colmap_points[k].xyz[1], colmap_points[k].xyz[2] ]) points3d = np.array(points3d) points3d -= np.median(points3d, axis=0)[None, ...] points3d /= np.abs(points3d).max() + 1e-6 pcd = os.path.join( get_colmap_viz_folder(cfg_bag)[0], 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( _bag_size, _bag_id)) with open(pcd, 'w') as f: f.write('# .PCD v.7 - Point Cloud Data file format\n') f.write('VERSION .7\n') f.write('FIELDS x y z\n') f.write('SIZE 4 4 4\n') f.write('TYPE F F F\n') f.write('COUNT 1 1 1\n') f.write('WIDTH {}\n'.format(len(colmap_points))) f.write('HEIGHT 1\n') f.write('VIEWPOINT 0 0 0 1 0 0 0\n') f.write('POINTS {}\n'.format(len(colmap_points))) f.write('DATA ascii\n') for p in points3d: f.write('{:.05f} {:.05f} {:.05f}\n'.format( p[0], p[1], p[2])) copyfile( os.path.join( get_colmap_viz_folder(cfg_bag)[0], 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( _bag_size, _bag_id)), os.path.join( get_colmap_viz_folder(cfg_bag)[1], 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( _bag_size, _bag_id))) print('done [{:.02f} s.]'.format(time() - t_start))
def main(cfg): '''Main function. Takes config as input. ''' # Back up config cfg_orig = deepcopy(cfg) method = cfg_orig.method_dict # Add config options to the dict master_dict = OrderedDict() master_dict['config'] = method # Add date master_dict['properties'] = OrderedDict() master_dict['properties'][ 'processing_date'] = pack_helper.get_current_date() print('Adding processing date: {}'.format( master_dict['properties']['processing_date'])) # Add descriptor properties cfg_desc = deepcopy(cfg_orig) cfg_desc.dataset = 'phototourism' cfg_desc.scene = 'british_museum' try: descriptors_dict = load_h5(get_desc_file(cfg_desc)) desc_type, desc_size, desc_nbytes = pack_helper.get_descriptor_properties( cfg_desc, descriptors_dict) except Exception: desc_type = 'none' desc_size = 0 desc_nbytes = 0 master_dict['properties']['descriptor_type'] = desc_type master_dict['properties']['descriptor_size'] = desc_size master_dict['properties']['descriptor_nbytes'] = desc_nbytes print('Adding descriptor properties: {} {} ({} bytes)'.format( master_dict['properties']['descriptor_size'], master_dict['properties']['descriptor_type'], master_dict['properties']['descriptor_nbytes'])) # Read data and splits for dataset in ['phototourism']: setattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset), './json/data/{}_{}.json'.format(dataset, cfg_orig.subset)) setattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset), './json/bag_size/{}_{}.json'.format(dataset, cfg_orig.subset)) # Create empty dictionary master_dict[dataset] = OrderedDict() res_dict = OrderedDict() master_dict[dataset]['results'] = res_dict # Save number of runs master_dict[dataset]['num_runs_stereo'] = getattr( cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) master_dict[dataset]['num_runs_multiview'] = getattr( cfg_orig, 'num_runs_{}_multiview'.format(cfg_orig.subset)) # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] bag_size_str = ['{}bag'.format(b) for b in bag_size_list] # Create empty dicts for scene in ['allseq'] + scene_list: res_dict[scene] = OrderedDict() for task in ['stereo', 'multiview', 'relocalization']: res_dict[scene][task] = OrderedDict() res_dict[scene][task]['run_avg'] = OrderedDict() if task == 'multiview': for bag in bag_size_str + ['bag_avg']: res_dict[scene]['multiview']['run_avg'][ bag] = OrderedDict() # Stereo -- multiple runs t = time() cur_key = 'config_{}_stereo'.format(dataset) if cfg_orig.eval_stereo and cur_key in method and method[cur_key]: num_runs = getattr(cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'stereo' for scene in scene_list: cfg.scene = scene res_dict[scene]['stereo']['run_avg'] = OrderedDict() for run in range(num_runs): res_dict[scene]['stereo']['run_{}'.format( run)] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] # metric_list += ['matching_scores_epipolar'] metric_list += ['num_inliers'] metric_list += ['matching_scores_depth_projection'] metric_list += ['repeatability'] metric_list += ['qt_auc'] metric_list += ['timings'] for run in range(num_runs): # Compute and pack results cfg.run = run cur_dict = res_dict[scene]['stereo']['run_{}'.format(run)] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)(cur_dict, cfg) print( ' -- Packing "{}"/"{}"/stereo, run: {}/{}, metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, metric, time() - t_cur)) # Compute average across runs, for stereo t_cur = time() pack_helper.average_stereo_over_runs(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for stereo t_cur = time() pack_helper.average_stereo_over_scenes(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing stereo in {:.01f} sec.'.format(time() - t)) else: print('Skipping "{}/stereo"'.format(dataset)) # Multiview -- multiple runs t = time() cur_key = 'config_{}_multiview'.format(dataset) if cfg_orig.eval_multiview and cur_key in method and method[cur_key]: num_runs = getattr(cfg, 'num_runs_{}_multiview'.format(cfg.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'multiview' for scene in scene_list: cfg.scene = scene for run in ['run_avg' ] + ['run_{}'.format(f) for f in range(num_runs)]: res_dict[scene]['multiview'][run] = OrderedDict() for bags_label in ['bag_avg'] + bag_size_str: res_dict[scene]['multiview'][run][ bags_label] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] metric_list += ['num_input_matches'] metric_list += ['qt_auc_colmap'] metric_list += ['ATE'] metric_list += ['colmap_stats'] for run in range(num_runs): for bag_size in bag_size_list: # Compute and pack results cfg.run = run cfg.bag_size = bag_size cur_dict = res_dict[scene]['multiview'] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)( cur_dict['run_{}'.format(run)]['{}bag'.format( bag_size)], cfg) print( ' -- Packing "{}"/"{}"/multiview, run {}/{}, "{}", metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, '{}bag'.format(bag_size), metric, time() - t_cur)) # Compute average across bags for metric in cur_dict['run_{}'.format(run)]['25bag']: pack_helper.average_multiview_over_bags( cfg, cur_dict['run_{}'.format(run)], bag_size_list) # Compute average across runs, for multiview t_cur = time() pack_helper.average_multiview_over_runs(cfg, res_dict, num_runs, bag_size_str + ['bag_avg']) print( ' -- Packing "{}"/multiview: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for multiview t_cur = time() pack_helper.average_multiview_over_scenes( cfg, res_dict, num_runs, ['bag_avg'] + bag_size_str) print( ' -- Packing "{}"/multiview: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing multiview in {:.01f} sec.'.format( time() - t)) # Relocalization -- multiple runs # TODO else: print('Skipping "{}/multiview"'.format(dataset)) # Dump packed result print(' -- Saving to: "{}"'.format( cfg.method_dict['config_common']['json_label'])) if not os.path.exists(cfg.path_pack): os.makedirs(cfg.path_pack) json_dump_file = os.path.join( cfg.path_pack, '{}.json'.format(cfg.method_dict['config_common']['json_label'])) with open(json_dump_file, 'w') as outfile: json.dump(master_dict, outfile, indent=2)
def make_xy(sfm_cfg): """ Messy conveniency function to re-format our data into that expected by the Context Networks release. """ xs = [] ys = [] Rs = [] ts = [] cx1s = [] cy1s = [] f1s = [] cx2s = [] cy2s = [] f2s = [] key_list = [] data_dir = get_data_path(sfm_cfg) keypoints_dict = load_h5(get_kp_file(sfm_cfg)) match_dict = load_h5(get_match_file(sfm_cfg)) calib_list = get_fullpath_list(data_dir, 'calibration') calib_dict = load_calib(calib_list) print('Converting data to a CNe friendly format...') for image_pair, match_idx_pairs in tqdm(match_dict.items()): key_list.append(image_pair) # Get image name and read image image_1, image_2 = image_pair.split('-') image1 = cv2.imread(os.path.join(data_dir, 'images', image_1 + '.jpg')) image2 = cv2.imread(os.path.join(data_dir, 'images', image_2 + '.jpg')) # Get dR R_1 = calib_dict[image_1]['R'] R_2 = calib_dict[image_2]['R'] dR = np.dot(R_2, R_1.T) # Get dt t_1 = calib_dict[image_1]['T'].reshape((3, 1)) t_2 = calib_dict[image_2]['T'].reshape((3, 1)) dt = t_2 - np.dot(dR, t_1) # Save R, t for evaluation Rs += [np.array(dR).reshape(3, 3)] # normalize t before saving dtnorm = np.sqrt(np.sum(dt**2)) assert (dtnorm > 1e-5) dt /= dtnorm ts += [np.array(dt).flatten()] # Save img1, center offset, f # img1s += [image1.transpose(2, 0, 1)] cx1 = (image1.shape[1] - 1.0) * 0.5 cy1 = (image1.shape[0] - 1.0) * 0.5 f1 = max(image1.shape[1] - 1.0, image1.shape[0] - 1.0) cx1s += [cx1] cy1s += [cy1] f1s += [f1] # Save img2, center offset, f # img2s += [image2.transpose(2, 0, 1)] cx2 = (image2.shape[1] - 1.0) * 0.5 cy2 = (image2.shape[0] - 1.0) * 0.5 f2 = max(image2.shape[1] - 1.0, image2.shape[0] - 1.0) cx2s += [cx2] cy2s += [cy2] f2s += [f2] # Get key points kp1 = np.asarray(keypoints_dict[image_1]) kp1 = kp1[:, :2] kp2 = np.asarray(keypoints_dict[image_2]) kp2 = kp2[:, :2] # Normalize Key points kp1 = (kp1 - np.asarray([cx1, cy1]).T) / np.asarray([f1, f1]).T kp2 = (kp2 - np.asarray([cx2, cy2]).T) / np.asarray([f2, f2]).T # Shuffle key points based on match index x1_index = match_idx_pairs[0, :] x2_index = match_idx_pairs[1, :] # Get shuffled key points for image 1 x1 = kp1[x1_index, :] # Assume depth = 1 z = np.ones((x1.shape[0], 1)) # Construct 3D points y1 = np.concatenate([x1 * z, z], axis=1) # Project 3D points to image 2 y1p = np.matmul(dR[None], y1[..., None]) + dt[None] # move back to the canonical plane x1p = y1p[:, :2, 0] / y1p[:, 2, 0][..., None] # Get shuffled key points for image 2 x2 = kp2[x2_index, :] # make xs in NHWC xs += [ np.concatenate([x1, x2], axis=1).T.reshape(4, 1, -1).transpose( (1, 2, 0)) ] # Get the geodesic distance using with x1, x2, dR, dt geod_d = get_sampsons(x1, x2, dR, dt) # Get *rough* reprojection errors. Note that the depth may be noisy. We # ended up not using this... reproj_d = np.sum((x2 - x1p)**2, axis=1) # add to label list ys += [np.stack([geod_d, reproj_d], axis=1)] res_dict = {} res_dict['xs'] = xs res_dict['ys'] = ys res_dict['Rs'] = Rs res_dict['ts'] = ts res_dict['cx1s'] = cx1s res_dict['cy1s'] = cy1s res_dict['f1s'] = f1s res_dict['cx2s'] = cx2s res_dict['cy2s'] = cy2s res_dict['f2s'] = f2s return res_dict, key_list
def validate_submission_files(sub_path, benchmark_repo_path, datasets, raw_data_path, logger): for dataset in datasets: raw_dataset_path = os.path.join(raw_data_path, dataset) # check if dataset folder exists sub_dataset_path = os.path.join(sub_path, dataset) if not os.path.isdir(sub_dataset_path): logger.add_new_log( 'Submission does not contain {} dataset.'.format(dataset)) continue # read seqs from json seqs = load_json( os.path.join(benchmark_repo_path, 'json/data/{}_test.json'.format(dataset))) for seq in seqs: # get number of image raw_seq_path = os.path.join(raw_dataset_path, seq) im_list = [ os.path.splitext(f)[0] for f in os.listdir(raw_seq_path) if (os.path.isfile(os.path.join(raw_seq_path, f)) and f.endswith(('png', 'jpg'))) ] num_im = len(im_list) # get all key pairs key_pairs = [ pair[0] + '-' + pair[1] for pair in list(product(im_list, im_list)) if pair[0] > pair[1] ] # check if seq folder exists sub_seq_path = os.path.join(sub_dataset_path, seq) if not os.path.isdir(sub_seq_path): logger.add_new_log( 'Submission does not contain {} sequence in {} dataset.'. format(seq, dataset)) continue # validate keypoints file kp_path = os.path.join(sub_seq_path, 'keypoints.h5') if not os.path.isfile(kp_path): logger.add_new_log( 'Submission does not contain keypoints file for {} sequence in {} dataset.' .format(seq, dataset)) else: keypoints = load_h5(kp_path) if sorted(list(keypoints.keys())) != sorted(im_list): logger.add_new_log( '{}-{}: Keypoints file does not contain all the image keys.' .format(dataset, seq)) if len(list(keypoints.values())[0].shape) != 2: logger.add_new_log( '{}-{}: Keypoints file is in wrong format.'.format( dataset, seq)) if list(keypoints.values())[0].shape[1] != 2: logger.add_new_log( '{}-{}: Keypoints file is in wrong format.'.format( dataset, seq)) # check number of keypoints if list(keypoints.values())[0].shape[0] > 8000: logger.add_new_log( '{}-{}: Keypoints file contains more than 8000 points.' .format(dataset, seq)) # check if match file exists first match_files = [ file for file in os.listdir(sub_seq_path) if os.path.isfile(os.path.join(sub_seq_path, file)) and file.startswith('match') ] # validate descriptor file desc_path = os.path.join(sub_seq_path, 'descriptors.h5') # much provide either descriptor file or match file if not os.path.isfile(desc_path) and len(match_files) == 0: logger.add_new_log( 'Submission does not contain descriptors file for {} sequence in {} dataset.' .format(seq, dataset)) elif not os.path.isfile(desc_path): pass else: descriptors = load_h5(desc_path) if sorted(list(descriptors.keys())) != sorted(im_list): logger.add_new_log( '{}-{}: Descriptors file does not contain all the image keys.' .format(dataset, seq)) if len(list(descriptors.values())[0].shape) != 2: logger.add_new_log( '{}-{}: Descriptors file is in wrong format'.format( dataset, seq)) if list(descriptors.values())[0].shape[1] < 64 or list( descriptors.values())[0].shape[1] > 2048: logger.add_new_log( '{}-{}: Descriptors file is in wrong format'.format( dataset, seq)) # check descriptor size desc_type, desc_size, desc_nbytes = get_descriptor_properties( {}, descriptors) if desc_nbytes > 512 and len(match_files) == 0: logger.add_new_log( '{}-{}: Descriptors size is larger than 512 bytes, you need to provide custom match file' .format(dataset, seq)) # validate match file # check match file name if 'matches.h5' in match_files: if len(match_files) != 1: logger.add_new_log( '{}-{}: matches.h5 exists. Do not need to provide any other match files.' .format(dataset, seq)) elif 'matches_multiview.h5' in match_files or 'matches_stereo_0.h5' in match_files or 'matches_stereo.h5' in match_files: if 'matches_multiview.h5' not in match_files: logger.add_new_log( '{}-{}: missing matches_multiview.h5'.format( dataset, seq)) if 'matches_stereo_0.h5' not in match_files and 'matches_stereo.h5' not in match_files: logger.add_new_log( '{}-{}: missing matches_stereo.h5'.format( dataset, seq)) if 'matches_stereo_1.h5' in match_files or 'matches_stereo_2.h5' in match_files: logger.add_new_log( '{}-{}: for 2021 challenge, we only run stereo once, no need to provide matches_stereo_1 and matches_stereo_2' .format(dataset, seq)) for match_file in match_files: matches = load_h5(os.path.join(sub_seq_path, match_file)) if len(matches.keys()) != len(key_pairs): logger.add_new_log( '{}-{}: Matches file contains wrong number of keys, should have {} keys, have {}.' .format(dataset, seq, len(key_pairs), len(matches.keys()))) elif sorted(list(matches.keys())) != sorted(key_pairs): logger.add_new_log( '{}-{}: Matches file contains worng keys, maybe the image names is in reverse order. Plase refer to submission instruction for proper custom match key naming convention' .format(dataset, seq)) if len(list(matches.values())[0].shape) != 2: logger.add_new_log( '{}-{}: Matches file is in wrong format.'.format( dataset, seq)) if list(matches.values())[0].shape[0] != 2: logger.add_new_log( '{}-{}: Matches file is in wrong format.'.format( dataset, seq))
def run_colmap_for_bag(cfg): '''Runs colmap to retrieve poses for each bag''' # Colmap pose file already exists, skip the session if os.path.exists(get_colmap_pose_file(cfg)): print(' -- already exists, skipping COLMAP eval') return # Load keypoints and matches keypoints_dict = load_h5(get_kp_file(cfg)) matches_dict = load_h5(get_filter_match_file(cfg)) print('Running COLMAP on "{}", bagsize {} -- bag {}'.format( cfg.scene, cfg.bag_size, cfg.bag_id)) # Additional sanity check to account for crash -- in this case colmap temp # directory can exist. This in an indication that you need to remove # results and rerun colmap. colmap_temp_path = get_colmap_temp_path(cfg) colmap_output_path = get_colmap_output_path(cfg) if os.path.exists(colmap_temp_path): print(' -- temp path exists - cleaning up from crash') rmtree(colmap_temp_path) if os.path.exists(colmap_output_path): rmtree(colmap_output_path) if os.path.exists(get_colmap_pose_file(cfg)): os.remove(get_colmap_pose_file(cfg)) # Check existance of colmap result and terminate if already exists. colmap_output_path = get_colmap_output_path(cfg) if os.path.exists(colmap_output_path): print(' -- already exists, skipping COLMAP session') return # Create output directory os.makedirs(colmap_output_path) # Create colmap temporary directory and copy files over. Remove anything # that might have existed. colmap_temp_path = get_colmap_temp_path(cfg) if os.path.exists(colmap_temp_path): rmtree(colmap_temp_path) # Make sure old data is gone and create a new temp folder assert (not os.path.exists(colmap_temp_path)) os.makedirs(colmap_temp_path) # Create colmap-friendy structures os.makedirs(os.path.join(colmap_temp_path, 'images')) os.makedirs(os.path.join(colmap_temp_path, 'features')) # Get list of all images in this bag image_subset_list = get_colmap_image_path_list(cfg) subset_index = get_colmap_image_subset_index(cfg, image_subset_list) # Copy images for _src in image_subset_list: _dst = os.path.join(colmap_temp_path, 'images', os.path.basename(_src)) copyfile(_src, _dst) # Write features to colmap friendly format for image_path in image_subset_list: # Retrieve image name, with and without extension image_name = os.path.basename(image_path) image_name_no_ext = os.path.splitext(image_name)[0] # Read keypoint keypoints = keypoints_dict[image_name_no_ext] # Keypoint file to write to kp_file = os.path.join(colmap_temp_path, 'features', image_name + '.txt') # Open a file to write with open(kp_file, 'w') as f: # Retieve the number of keypoints len_keypoints = len(keypoints) f.write(str(len_keypoints) + ' ' + str(128) + '\n') for i in range(len_keypoints): kp = ' '.join(str(k) for k in keypoints[i][:4]) desc = ' '.join(str(0) for d in range(128)) f.write(kp + ' ' + desc + '\n') # Write matches to colmap friendly format # Read visibilties data_dir = get_data_path(cfg) vis_list = get_fullpath_list(data_dir, 'visibility') # Load matches and store them to a text file # TODO: This seems to be done multiple times. Do we need to do this? print('Generate list of all possible pairs') pairs = compute_image_pairs(vis_list, len(image_subset_list), cfg.vis_th, subset_index) print('{} pairs generated'.format(len(pairs))) # Write to match file match_file = os.path.join(colmap_temp_path, 'matches.txt') with open(match_file, 'w') as f: for pair in pairs: image_1_name = os.path.basename(image_subset_list[pair[0]]) image_2_name = os.path.basename(image_subset_list[pair[1]]) image_1_name_no_ext = os.path.splitext(image_1_name)[0] image_2_name_no_ext = os.path.splitext(image_2_name)[0] # Load matches key = '-'.join([image_1_name_no_ext, image_2_name_no_ext]) matches = np.squeeze(matches_dict[key]) # only write when matches are given if matches.ndim == 2: f.write(image_1_name + ' ' + image_2_name + '\n') for _i in range(matches.shape[1]): f.write( str(matches[0, _i]) + ' ' + str(matches[1, _i]) + '\n') f.write('\n') f.close() # COLMAP runs -- wrapped in try except to throw errors if subprocess fails # and then clean up the colmap temp directory try: print('COLMAP Feature Import') cmd = ['colmap', 'feature_importer'] cmd += [ '--database_path', os.path.join(colmap_output_path, 'databases.db') ] cmd += ['--image_path', os.path.join(colmap_temp_path, 'images')] cmd += ['--import_path', os.path.join(colmap_temp_path, 'features')] colmap_res = subprocess.run(cmd) if colmap_res.returncode != 0: raise RuntimeError(' -- COLMAP failed to import features!') print('COLMAP Match Import') cmd = ['colmap', 'matches_importer'] cmd += [ '--database_path', os.path.join(colmap_output_path, 'databases.db') ] cmd += [ '--match_list_path', os.path.join(colmap_temp_path, 'matches.txt') ] cmd += ['--match_type', 'raw'] cmd += ['--SiftMatching.use_gpu', '0'] colmap_res = subprocess.run(cmd) if colmap_res.returncode != 0: raise RuntimeError(' -- COLMAP failed to import matches!') print('COLMAP Mapper') cmd = ['colmap', 'mapper'] cmd += ['--image_path', os.path.join(colmap_temp_path, 'images')] cmd += [ '--database_path', os.path.join(colmap_output_path, 'databases.db') ] cmd += ['--output_path', colmap_output_path] cmd += ['--Mapper.min_model_size', str(cfg.colmap_min_model_size)] colmap_res = subprocess.run(cmd) if colmap_res.returncode != 0: raise RuntimeError(' -- COLMAP failed to run mapper!') # Delete temp directory after working rmtree(colmap_temp_path) except Exception as err: # Remove colmap output path and temp path rmtree(colmap_temp_path) rmtree(colmap_output_path) # Re-throw error print(err) raise RuntimeError('Parts of colmap runs returns failed state!') print('Checking validity of the colmap run just in case') # Check validity of colmap reconstruction for all of them is_any_colmap_valid = False idx_list = [ os.path.join(colmap_output_path, _d) for _d in os.listdir(colmap_output_path) if os.path.isdir(os.path.join(colmap_output_path, _d)) ] for idx in idx_list: colmap_img_file = os.path.join(idx, 'images.bin') if is_colmap_img_valid(colmap_img_file): is_any_colmap_valid = True break if not is_any_colmap_valid: print('Error in reading colmap output -- ' 'removing colmap output directory') rmtree(colmap_output_path)
def main(cfg): '''Visualization of stereo keypoints and matches. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' # Files should not be named to prevent (easy) abuse # Instead we use 0, ..., cfg.num_viz_stereo_pairs viz_folder_hq, viz_folder_lq = get_stereo_viz_folder(cfg) # # Do not re-run if files already exist -- off for now # if os.path.exists(viz_folder_lq): # if all([ # os.path.exists( # os.path.join(viz_folder_lq, 'stereo-{}.jpg'.format(i))) # for i in range(cfg.num_viz_stereo_pairs) # ]): # print(' -- already exists, skipping stereo visualization') # return print(' -- Visualizations, stereo: "{}/{}"'.format(cfg.dataset, cfg.scene)) t_start = time() # Load keypoints, matches and errors keypoints_dict = load_h5(get_kp_file(cfg)) matches_dict = load_h5(get_match_file(cfg)) # Hacky: We need to recompute the errors, loading only for the keys errors_dict = load_h5(get_stereo_epipolar_final_match_file(cfg, th='0.1')) # Get data directory data_dir = get_data_path(cfg) # Create results folder if it does not exist if not os.path.exists(viz_folder_hq): os.makedirs(viz_folder_hq) if not os.path.exists(viz_folder_lq): os.makedirs(viz_folder_lq) # Sort alphabetically and pick different images sorted_keys = sorted(errors_dict) picked = [] pairs = [] for pair in sorted_keys: fn1, fn2 = pair.split('-') if fn1 not in picked and fn2 not in picked: picked += [fn1, fn2] pairs += [pair] if len(pairs) == cfg.num_viz_stereo_pairs: break # Load all depth maps depth = {} for pair in pairs: files = pair.split('-') for f in files: if f not in depth: depth[f] = load_depth( os.path.join(data_dir, 'depth_maps', '{}.h5'.format(f))) # Generate and save the images for i, pair in enumerate(pairs): # load metadata fn1, fn2 = pair.split('-') calib_dict = load_calib([ os.path.join(data_dir, 'calibration', 'calibration_{}.h5'.format(fn1)), os.path.join(data_dir, 'calibration', 'calibration_{}.h5'.format(fn2)) ]) calc1 = calib_dict[fn1] calc2 = calib_dict[fn2] matches = matches_dict[pair] ransac_inl_dict = load_h5(get_geom_inl_file(cfg)) inl = ransac_inl_dict[pair] # Get depth for keypoints kp1 = keypoints_dict[fn1] kp2 = keypoints_dict[fn2] kp1_int = np.round(kp1).astype(int) kp2_int = np.round(kp2).astype(int) kp1_int[:, 1] = np.clip(kp1_int[:, 1], 0, depth[fn1].shape[0] - 1) kp1_int[:, 0] = np.clip(kp1_int[:, 0], 0, depth[fn1].shape[1] - 1) d1 = np.expand_dims(depth[fn1][kp1_int[:, 1], kp1_int[:, 0]], axis=-1) d2 = np.expand_dims(depth[fn2][kp2_int[:, 1], kp2_int[:, 0]], axis=-1) # Get {R, t} from calibration information R_1, t_1 = calc1['R'], calc1['T'].reshape((3, 1)) R_2, t_2 = calc2['R'], calc2['T'].reshape((3, 1)) # Compute dR, dt dR = np.dot(R_2, R_1.T) dT = t_2 - np.dot(dR, t_1) # Normalize keypoints kp1n = normalize_keypoints(kp1, calc1['K']) kp2n = normalize_keypoints(kp2, calc2['K']) # Project with depth kp1n_p, kp2n_p = get_projected_kp(kp1n, kp2n, d1, d2, dR, dT) kp1_p = unnormalize_keypoints(kp1n_p, calc2['K']) kp2_p = unnormalize_keypoints(kp2n_p, calc1['K']) # Re-index keypoints from matches kp1_inl = kp1[inl[0]] kp2_inl = kp2[inl[1]] kp1_p_inl = kp1_p[inl[0]] kp2_p_inl = kp2_p[inl[1]] kp1n_inl = kp1n[inl[0]] kp2n_inl = kp2n[inl[1]] kp1n_p_inl = kp1n_p[inl[0]] kp2n_p_inl = kp2n_p[inl[1]] d1_inl = d1[inl[0]] d2_inl = d2[inl[1]] # Filter out keypoints with invalid depth nonzero_index = np.nonzero(np.squeeze(d1_inl * d2_inl)) zero_index = np.where(np.squeeze(d1_inl * d2_inl) == 0)[0] kp1_inl_nonzero = kp1_inl[nonzero_index] kp2_inl_nonzero = kp2_inl[nonzero_index] kp1_p_inl_nonzero = kp1_p_inl[nonzero_index] kp2_p_inl_nonzero = kp2_p_inl[nonzero_index] kp1n_inl_nonzero = kp1n_inl[nonzero_index] kp2n_inl_nonzero = kp2n_inl[nonzero_index] kp1n_p_inl_nonzero = kp1n_p_inl[nonzero_index] kp2n_p_inl_nonzero = kp2n_p_inl[nonzero_index] # Compute symmetric distance using the depth image true_d = get_truesym(kp1_inl_nonzero, kp2_inl_nonzero, kp1_p_inl_nonzero, kp2_p_inl_nonzero) # canvas im, v_offset, h_offset = build_composite_image( os.path.join(data_dir, 'images', fn1 + '.jpg'), os.path.join(data_dir, 'images', fn2 + '.jpg'), margin=5, axis=0 if cfg.viz_composite_vert else 1) plt.figure(figsize=(10, 10)) plt.imshow(im) linewidth = 2 # Plot matches on points without depth for idx in range(len(zero_index)): plt.plot((kp1[idx, 0] + h_offset[0], kp2[idx, 0] + h_offset[1]), (kp1[idx, 1] + v_offset[0], kp2[idx, 1] + v_offset[1]), color='b', linewidth=linewidth) # Plot matches on points with depth max_dist = 5 cmap = matplotlib.cm.get_cmap('summer') order = list(range(len(true_d))) random.shuffle(order) for idx in order: if true_d[idx] <= max_dist: min_val = 0 max_val = 255 - min_val col = cmap( int(max_val * (1 - (max_dist - true_d[idx]) / max_dist) + min_val)) # col = cmap(255 * (max_dist - true_d[idx]) / max_dist) else: col = 'r' plt.plot((kp1_inl_nonzero[idx, 0] + h_offset[0], kp2_inl_nonzero[idx, 0] + h_offset[1]), (kp1_inl_nonzero[idx, 1] + v_offset[0], kp2_inl_nonzero[idx, 1] + v_offset[1]), color=col, linewidth=linewidth) plt.tight_layout() plt.axis('off') viz_file_hq = os.path.join(viz_folder_hq, '{:05d}.png'.format(i)) viz_file_lq = os.path.join(viz_folder_lq, '{:05d}.jpg'.format(i)) plt.savefig(viz_file_hq, bbox_inches='tight') # Convert with imagemagick os.system('convert -quality 75 -resize \"500>\" {} {}'.format( viz_file_hq, viz_file_lq)) plt.close() print('done [{:.02f} s.]'.format(time() - t_start))
def main(cfg): '''Main function to compute model. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' if os.path.exists(get_geom_file(cfg)): print(' -- already exists, skipping model computation') return # Get data directory keypoints_dict = load_h5(get_kp_file(cfg)) # Load keypoints and matches matches_dict = load_h5(get_filter_match_file_for_computing_model(cfg)) # Feature Matching print('Computing model') num_cores = cfg.num_opencv_threads if cfg.num_opencv_threads > 0 else int( len(os.sched_getaffinity(0)) * 0.9) # Load camera information data_dir = get_data_path(cfg) images_list = get_fullpath_list(data_dir, 'images') image_names = get_item_name_list(images_list) calib_list = get_fullpath_list(data_dir, 'calibration') calib_dict = load_calib(calib_list) pairs_per_th = get_pairs_per_threshold(data_dir) # Get data directory try: desc_dict = defaultdict(list) desc_dict = load_h5(get_desc_file(cfg)) for k, v in desc_dict.items(): desc_dict[k] = v except Exception: desc_dict = defaultdict(list) try: aff_dict = defaultdict(list) aff_dict1 = load_h5(get_affine_file(cfg)) for k, v in aff_dict1.items(): aff_dict[k] = v except Exception: aff_dict = defaultdict(list) try: ori_dict = defaultdict(list) ori_dict1 = load_h5(get_angle_file(cfg)) for k, v in ori_dict1.items(): ori_dict[k] = v except Exception: ori_dict = defaultdict(list) try: scale_dict = defaultdict(list) scale_dict1 = load_h5(get_scale_file(cfg)) for k, v in scale_dict1.items(): scale_dict[k] = v except Exception: scale_dict = defaultdict(list) random.shuffle(pairs_per_th['0.0']) result = Parallel(n_jobs=num_cores)(delayed(compute_model)( cfg, np.asarray(matches_dict[pair]), np.asarray(keypoints_dict[pair.split('-')[0]]), np.asarray(keypoints_dict[pair.split('-')[1]]), calib_dict[pair.split( '-')[0]], calib_dict[pair.split('-')[1]], images_list[ image_names.index(pair.split('-')[0])], images_list[ image_names.index(pair.split('-')[1])], np.asarray(scale_dict[pair.split('-')[0]]), np.asarray(scale_dict[pair.split('-')[1]]), np.asarray(ori_dict[pair.split('-')[0]]), np.asarray(ori_dict[pair.split('-')[1]]), np.asarray(aff_dict[pair.split('-')[0]]), np.asarray(aff_dict[pair.split('-')[1]]), np.asarray(desc_dict[pair.split('-')[0]]), np.asarray(desc_dict[pair.split('-')[1]])) for pair in tqdm(pairs_per_th['0.0'])) # Make model dictionary model_dict = {} inl_dict = {} timings_list = [] for i, pair in enumerate(pairs_per_th['0.0']): model_dict[pair] = result[i][0] inl_dict[pair] = result[i][1] timings_list.append(result[i][2]) # Check model directory if not os.path.exists(get_geom_path(cfg)): os.makedirs(get_geom_path(cfg)) # Finally save packed models save_h5(model_dict, get_geom_file(cfg)) save_h5(inl_dict, get_geom_inl_file(cfg)) # Save computational cost save_h5({'cost': np.mean(timings_list)}, get_geom_cost_file(cfg)) print('Geometry cost (averaged over image pairs): {:0.2f} sec'.format( np.mean(timings_list)))
def main(cfg): '''Main function to compute matches. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' # Get data directory data_dir = get_data_path(cfg) # Load pre-computed pairs with the new visibility criteria pairs_per_th = get_pairs_per_threshold(data_dir) # Check if all files exist if is_stereo_complete(cfg): print(' -- already exists, skipping stereo eval') return # Load keypoints and matches keypoints_dict = load_h5(get_kp_file(cfg)) matches_dict = load_h5(get_match_file(cfg)) geom_dict = load_h5(get_geom_file(cfg)) geom_inl_dict = load_h5(get_geom_inl_file(cfg)) filter_matches_dict = load_h5(get_filter_match_file(cfg)) # Load visiblity and images images_list = get_fullpath_list(data_dir, 'images') vis_list = get_fullpath_list(data_dir, 'visibility') if cfg.dataset != 'googleurban': depth_maps_list = get_fullpath_list(data_dir, 'depth_maps') image_names = get_item_name_list(images_list) # Load camera information calib_list = get_fullpath_list(data_dir, 'calibration') calib_dict = load_calib(calib_list) # Generate all possible pairs print('Generating list of all possible pairs') pairs = compute_image_pairs(vis_list, len(image_names), cfg.vis_th) print('Old pairs with the point-based visibility threshold: {} ' '(for compatibility)'.format(len(pairs))) for k, v in pairs_per_th.items(): print('New pairs at visibility threshold {}: {}'.format(k, len(v))) # Evaluate each stereo pair in parallel # Compute it for all pairs (i.e. visibility threshold 0) print('Compute stereo metrics for all pairs') #num_cores = int(multiprocessing.cpu_count() * 0.9) num_cores = int(len(os.sched_getaffinity(0)) * 0.9) result = Parallel(n_jobs=num_cores)(delayed(compute_stereo_metrics_from_E)( images_list[image_names.index(pair.split('-')[0])], images_list[ image_names.index(pair.split('-')[1])], depth_maps_list[image_names.index(pair.split('-')[0])] if cfg. dataset != 'googleurban' else None, depth_maps_list[image_names.index( pair.split('-')[1])] if cfg.dataset != 'googleurban' else None, np.asarray(keypoints_dict[pair.split('-')[0]]), np.asarray(keypoints_dict[pair.split('-')[1]]), calib_dict[pair.split( '-')[0]], calib_dict[pair.split('-') [1]], geom_dict[pair], matches_dict[pair], filter_matches_dict[pair], geom_inl_dict[pair], cfg) for pair in tqdm(pairs_per_th['0.0'])) # Convert previous visibility list to strings old_keys = [] for pair in pairs: old_keys.append('{}-{}'.format(image_names[pair[0]], image_names[pair[1]])) # Extract scores, err_q, err_t from results all_keys = pairs_per_th['0.0'] err_dict, rep_s_dict = {}, {} geo_s_dict_pre_match, geo_s_dict_refined_match, \ geo_s_dict_final_match = {}, {}, {} true_s_dict_pre_match, true_s_dict_refined_match, \ true_s_dict_final_match = {}, {}, {} for i in range(len(result)): if all_keys[i] in old_keys: if result[i][5]: geo_s_dict_pre_match[ all_keys[i]] = result[i][0][0] if result[i][0] else None geo_s_dict_refined_match[ all_keys[i]] = result[i][0][1] if result[i][0] else None geo_s_dict_final_match[ all_keys[i]] = result[i][0][2] if result[i][0] else None true_s_dict_pre_match[ all_keys[i]] = result[i][1][0] if result[i][1] else None true_s_dict_refined_match[ all_keys[i]] = result[i][1][1] if result[i][1] else None true_s_dict_final_match[ all_keys[i]] = result[i][1][2] if result[i][1] else None err_q = result[i][2] err_t = result[i][3] rep_s_dict[all_keys[i]] = result[i][4] err_dict[all_keys[i]] = [err_q, err_t] print('Aggregating results for the old visibility constraint: ' '{}/{}'.format(len(geo_s_dict_pre_match), len(result))) # Repeat with the new visibility threshold err_dict_th, rep_s_dict_th = {}, {} geo_s_dict_pre_match_th, geo_s_dict_refined_match_th, \ geo_s_dict_final_match_th = {}, {}, {} true_s_dict_pre_match_th, true_s_dict_refined_match_th, \ true_s_dict_final_match_th = {}, {}, {} for th, cur_pairs in pairs_per_th.items(): _err_dict, _rep_s_dict = {}, {} _geo_s_dict_pre_match, _geo_s_dict_refined_match, \ _geo_s_dict_final_match = {}, {}, {} _true_s_dict_pre_match, _true_s_dict_refined_match, \ _true_s_dict_final_match = {}, {}, {} for i in range(len(all_keys)): if len(cur_pairs) > 0 and all_keys[i] in cur_pairs: if result[i][5]: _geo_s_dict_pre_match[all_keys[ i]] = result[i][0][0] if result[i][0] else None _geo_s_dict_refined_match[all_keys[ i]] = result[i][0][1] if result[i][0] else None _geo_s_dict_final_match[all_keys[ i]] = result[i][0][2] if result[i][0] else None _true_s_dict_pre_match[all_keys[ i]] = result[i][1][0] if result[i][1] else None _true_s_dict_refined_match[all_keys[ i]] = result[i][1][1] if result[i][1] else None _true_s_dict_final_match[all_keys[ i]] = result[i][1][2] if result[i][1] else None err_q = result[i][2] err_t = result[i][3] _rep_s_dict[ all_keys[i]] = result[i][4] if result[i][4] else None _err_dict[all_keys[i]] = [err_q, err_t] geo_s_dict_pre_match_th[th] = _geo_s_dict_pre_match geo_s_dict_refined_match_th[th] = _geo_s_dict_refined_match geo_s_dict_final_match_th[th] = _geo_s_dict_final_match true_s_dict_pre_match_th[th] = _true_s_dict_pre_match true_s_dict_refined_match_th[th] = _true_s_dict_refined_match true_s_dict_final_match_th[th] = _true_s_dict_final_match err_dict_th[th] = _err_dict rep_s_dict_th[th] = _rep_s_dict print('Aggregating results for threshold "{}": {}/{}'.format( th, len(geo_s_dict_pre_match_th[th]), len(result))) # Create results folder if it does not exist if not os.path.exists(get_stereo_path(cfg)): os.makedirs(get_stereo_path(cfg)) # Finally, save packed scores and errors if cfg.dataset != 'googleurban': save_h5(geo_s_dict_pre_match, get_stereo_epipolar_pre_match_file(cfg)) save_h5(geo_s_dict_refined_match, get_stereo_epipolar_refined_match_file(cfg)) save_h5(geo_s_dict_final_match, get_stereo_epipolar_final_match_file(cfg)) save_h5(true_s_dict_pre_match, get_stereo_depth_projection_pre_match_file(cfg)) save_h5(true_s_dict_refined_match, get_stereo_depth_projection_refined_match_file(cfg)) save_h5(true_s_dict_final_match, get_stereo_depth_projection_final_match_file(cfg)) save_h5(rep_s_dict, get_repeatability_score_file(cfg)) save_h5(err_dict, get_stereo_pose_file(cfg)) for th in pairs_per_th: if cfg.dataset != 'googleurban': save_h5(geo_s_dict_pre_match_th[th], get_stereo_epipolar_pre_match_file(cfg, th)) save_h5(geo_s_dict_refined_match_th[th], get_stereo_epipolar_refined_match_file(cfg, th)) save_h5(geo_s_dict_final_match_th[th], get_stereo_epipolar_final_match_file(cfg, th)) save_h5(true_s_dict_pre_match_th[th], get_stereo_depth_projection_pre_match_file(cfg, th)) save_h5(true_s_dict_refined_match_th[th], get_stereo_depth_projection_refined_match_file(cfg, th)) save_h5(true_s_dict_final_match_th[th], get_stereo_depth_projection_final_match_file(cfg, th)) save_h5(rep_s_dict_th[th], get_repeatability_score_file(cfg, th)) save_h5(err_dict_th[th], get_stereo_pose_file(cfg, th))