def get_num_in_bag(cfg): '''Retrieve number of bags per subset from the json file.''' scene_list = load_json( getattr(cfg, 'scenes_{}_{}'.format(cfg.dataset, cfg.subset))) bag_size_json = load_json( getattr(cfg, 'splits_{}_{}'.format(cfg.dataset, cfg.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] return bag_size_num[bag_size_list.index(cfg.bag_size)]
def validate_json(json_path, datasets, logger): # check if json file exist if not os.path.isfile(json_path): logger.add_new_log('Submission does not contain json file') return # load json try: method_list = load_json(json_path) except: logger.add_new_log( 'Following error occurs when loading json : \n {}'.format( sys.exc_info())) return # validate json if not type(method_list) is list: logger.add_new_log( 'Json should contain a list of method, please refer to the example json file.' ) return for i, method in enumerate(method_list): print('Validating method {}/{}: "{}"'.format( i + 1, len(method_list), method['config_common']['json_label'])) try: validate_method(method, is_challenge=True, datasets=datasets) except: logger.add_new_log( 'Following error occurs when validating json : \n {}'.format( sys.exc_info()))
def compute_num_input_matches(res_dict, deprecated_images, cfg): '''Save the number of input matches given to Colmap.''' # TODO fix this after re-implementing custom matches # if cfg.method_dict['config_{}_{}'.format(cfg.dataset, # cfg.task)]['use_custom_matches']: # raise NotImplementedError( # 'TODO Load the right dict with custom matches') # Read match dict matches_dict = load_h5_valid_image( get_filter_match_file(cfg) ,deprecated_images) # For every bag, compute the number of matches going into colmap bag_size_json = load_json( getattr(cfg, 'splits_{}_{}'.format(cfg.dataset, cfg.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] # Average it per bag size first, then across all bag sizes num_input_matches = [] for bag_size, cur_bag_size_num in zip(bag_size_list, bag_size_num): num_input_matches_bagsize = [] for bag_id in range(cur_bag_size_num): cfg_bag = deepcopy(cfg) cfg_bag.bag_size = bag_size cfg_bag.bag_id = bag_id # Skip if bag contain deprecated images if not valid_bag(cfg_bag, deprecated_images): continue images = get_colmap_image_path_list(cfg_bag) keys = [os.path.splitext(os.path.basename(im))[0] for im in images] pairs = [] for i in range(len(keys)): for j in range(i + 1, len(keys)): pairs.append('-'.join( sorted([keys[i], keys[j]], reverse=True))) for pair in pairs: num_input_matches_bagsize.append(matches_dict[pair].shape[-1]) num_input_matches.append(np.mean(num_input_matches_bagsize)) res_dict['num_input_matches'] = np.mean(num_input_matches)
def main(cfg): '''Visualization of stereo keypoints and matches. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' # Files should not be named to prevent (easy) abuse # Instead we use 0, ..., cfg.num_viz_stereo_pairs viz_folder_hq, viz_folder_lq = get_stereo_viz_folder(cfg) print(' -- Visualizations, stereo: "{}/{}"'.format(cfg.dataset, cfg.scene)) t_start = time() # Load deprecated images list deprecated_images_all = load_json(cfg.json_deprecated_images) if cfg.dataset in deprecated_images_all and cfg.scene in deprecated_images_all[ cfg.dataset]: deprecated_images = deprecated_images_all[cfg.dataset][cfg.scene] else: deprecated_images = [] # Load keypoints, matches and errors keypoints_dict = load_h5_valid_image(get_kp_file(cfg), deprecated_images) matches_dict = load_h5_valid_image(get_match_file(cfg), deprecated_images) ransac_inl_dict = load_h5_valid_image(get_geom_inl_file(cfg), deprecated_images) # Hacky: We need to recompute the errors, loading only for the keys data_dir = get_data_path(cfg) pairs_all = get_pairs_per_threshold(data_dir)['0.1'] pairs = [] for pair in pairs_all: if all([key not in deprecated_images for key in pair.split('-')]): pairs += [pair] # Create results folder if it does not exist if not os.path.exists(viz_folder_hq): os.makedirs(viz_folder_hq) if not os.path.exists(viz_folder_lq): os.makedirs(viz_folder_lq) # Sort alphabetically and pick different images sorted_keys = sorted(pairs) picked = [] pairs = [] for pair in sorted_keys: fn1, fn2 = pair.split('-') if fn1 not in picked and fn2 not in picked: picked += [fn1, fn2] pairs += [pair] if len(pairs) == cfg.num_viz_stereo_pairs: break # Load depth maps depth = {} if cfg.dataset != 'googleurban': for pair in pairs: files = pair.split('-') for f in files: if f not in depth: depth[f] = load_depth( os.path.join(data_dir, 'depth_maps', '{}.h5'.format(f))) # Generate and save the images for i, pair in enumerate(pairs): # load metadata fn1, fn2 = pair.split('-') calib_dict = load_calib([ os.path.join(data_dir, 'calibration', 'calibration_{}.h5'.format(fn1)), os.path.join(data_dir, 'calibration', 'calibration_{}.h5'.format(fn2)) ]) calc1 = calib_dict[fn1] calc2 = calib_dict[fn2] inl = ransac_inl_dict[pair] # Get depth for keypoints kp1 = keypoints_dict[fn1] kp2 = keypoints_dict[fn2] # Normalize keypoints kp1n = normalize_keypoints(kp1, calc1['K']) kp2n = normalize_keypoints(kp2, calc2['K']) # Get {R, t} from calibration information R_1, t_1 = calc1['R'], calc1['T'].reshape((3, 1)) R_2, t_2 = calc2['R'], calc2['T'].reshape((3, 1)) # Compute dR, dt dR = np.dot(R_2, R_1.T) dT = t_2 - np.dot(dR, t_1) if cfg.dataset == 'phototourism': kp1_int = np.round(kp1).astype(int) kp2_int = np.round(kp2).astype(int) kp1_int[:, 1] = np.clip(kp1_int[:, 1], 0, depth[fn1].shape[0] - 1) kp1_int[:, 0] = np.clip(kp1_int[:, 0], 0, depth[fn1].shape[1] - 1) kp2_int[:, 1] = np.clip(kp2_int[:, 1], 0, depth[fn2].shape[0] - 1) kp2_int[:, 0] = np.clip(kp2_int[:, 0], 0, depth[fn2].shape[1] - 1) d1 = np.expand_dims(depth[fn1][kp1_int[:, 1], kp1_int[:, 0]], axis=-1) d2 = np.expand_dims(depth[fn2][kp2_int[:, 1], kp2_int[:, 0]], axis=-1) # Project with depth kp1n_p, kp2n_p = get_projected_kp(kp1n, kp2n, d1, d2, dR, dT) kp1_p = unnormalize_keypoints(kp1n_p, calc2['K']) kp2_p = unnormalize_keypoints(kp2n_p, calc1['K']) # Re-index keypoints from matches kp1_inl = kp1[inl[0]] kp2_inl = kp2[inl[1]] kp1_p_inl = kp1_p[inl[0]] kp2_p_inl = kp2_p[inl[1]] kp1n_inl = kp1n[inl[0]] kp2n_inl = kp2n[inl[1]] kp1n_p_inl = kp1n_p[inl[0]] kp2n_p_inl = kp2n_p[inl[1]] d1_inl = d1[inl[0]] d2_inl = d2[inl[1]] # Filter out keypoints with invalid depth nonzero_index = np.nonzero(np.squeeze(d1_inl * d2_inl)) zero_index = np.where(np.squeeze(d1_inl * d2_inl) == 0)[0] kp1_inl_nonzero = kp1_inl[nonzero_index] kp2_inl_nonzero = kp2_inl[nonzero_index] kp1_p_inl_nonzero = kp1_p_inl[nonzero_index] kp2_p_inl_nonzero = kp2_p_inl[nonzero_index] kp1n_inl_nonzero = kp1n_inl[nonzero_index] kp2n_inl_nonzero = kp2n_inl[nonzero_index] kp1n_p_inl_nonzero = kp1n_p_inl[nonzero_index] kp2n_p_inl_nonzero = kp2n_p_inl[nonzero_index] # Compute symmetric distance using the depth image d = get_truesym(kp1_inl_nonzero, kp2_inl_nonzero, kp1_p_inl_nonzero, kp2_p_inl_nonzero) else: # All points are valid for computing the epipolar distance. zero_index = [] # Compute symmetric epipolar distance for every match. kp1_inl_nonzero = kp1[inl[0]] kp2_inl_nonzero = kp2[inl[1]] kp1n_inl_nonzero = kp1n[inl[0]] kp2n_inl_nonzero = kp2n[inl[1]] # d = np.zeros(inl.shape[1]) d = get_episym(kp1n_inl_nonzero, kp2n_inl_nonzero, dR, dT) # canvas im, v_offset, h_offset = build_composite_image( os.path.join( data_dir, 'images', fn1 + ('.png' if cfg.dataset == 'googleurban' else '.jpg')), os.path.join( data_dir, 'images', fn2 + ('.png' if cfg.dataset == 'googleurban' else '.jpg')), margin=5, axis=1 if (not cfg.viz_composite_vert or cfg.dataset == 'googleurban' or cfg.dataset == 'pragueparks') else 0) plt.figure(figsize=(10, 10)) plt.imshow(im) linewidth = 2 # Plot matches on points without depth for idx in range(len(zero_index)): plt.plot( (kp1_inl[idx, 0] + h_offset[0], kp2_inl[idx, 0] + h_offset[1]), (kp1_inl[idx, 1] + v_offset[0], kp2_inl[idx, 1] + v_offset[1]), color='b', linewidth=linewidth) # Plot matches # Points are normalized by the focals, which are on average ~670. max_dist = 5 if cfg.dataset == 'googleurban': max_dist = 2e-4 if cfg.dataset == 'pragueparks': max_dist = 2e-4 cmap = matplotlib.cm.get_cmap('summer') order = list(range(len(d))) random.shuffle(order) for idx in order: if d[idx] <= max_dist: min_val = 0 max_val = 255 - min_val col = cmap( int(max_val * (1 - (max_dist - d[idx]) / max_dist) + min_val)) # col = cmap(255 * (max_dist - d[idx]) / max_dist) else: col = 'r' plt.plot((kp1_inl_nonzero[idx, 0] + h_offset[0], kp2_inl_nonzero[idx, 0] + h_offset[1]), (kp1_inl_nonzero[idx, 1] + v_offset[0], kp2_inl_nonzero[idx, 1] + v_offset[1]), color=col, linewidth=linewidth) plt.tight_layout() plt.axis('off') viz_file_hq = os.path.join(viz_folder_hq, '{:05d}.png'.format(i)) viz_file_lq = os.path.join(viz_folder_lq, '{:05d}.jpg'.format(i)) plt.savefig(viz_file_hq, bbox_inches='tight') # Convert with imagemagick os.system('convert -quality 75 -resize \"500>\" {} {}'.format( viz_file_hq, viz_file_lq)) plt.close() print('Done [{:.02f} s.]'.format(time() - t_start))
def main(cfg): '''Main function. Takes config as input. ''' # Back up config cfg_orig = deepcopy(cfg) method = cfg_orig.method_dict # Add config options to the dict master_dict = OrderedDict() master_dict['config'] = method # Add date master_dict['properties'] = OrderedDict() master_dict['properties'][ 'processing_date'] = pack_helper.get_current_date() print('Adding processing date: {}'.format( master_dict['properties']['processing_date'])) # Add submission flag master_dict['properties']['is_submission'] = cfg.is_submission print('Flagging as user submission: {}'.format(cfg.is_submission)) # Add descriptor properties cfg_desc = deepcopy(cfg_orig) cfg_desc.dataset = 'phototourism' cfg_desc.scene = 'british_museum' try: descriptors_dict = load_h5(get_desc_file(cfg_desc)) desc_type, desc_size, desc_nbytes = pack_helper.get_descriptor_properties( cfg_desc, descriptors_dict) except Exception: desc_type = 'none' desc_size = 0 desc_nbytes = 0 master_dict['properties']['descriptor_type'] = desc_type master_dict['properties']['descriptor_size'] = desc_size master_dict['properties']['descriptor_nbytes'] = desc_nbytes print('Adding descriptor properties: {} {} ({} bytes)'.format( master_dict['properties']['descriptor_size'], master_dict['properties']['descriptor_type'], master_dict['properties']['descriptor_nbytes'])) deprecated_images_all = load_json(cfg.json_deprecated_images) if cfg.dataset in deprecated_images_all and cfg.scene in deprecated_images_all[ cfg.dataset]: deprecated_images = deprecated_images_all[cfg.dataset][cfg.scene] else: deprecated_images = [] # Read data and splits DATASET_LIST = ['phototourism', 'pragueparks', 'googleurban'] for dataset in DATASET_LIST: # Skip if not in config if 'config_{}_stereo'.format( dataset) not in method and 'config_{}_multiview'.format( dataset) not in method: continue # Create empty dictionary master_dict[dataset] = OrderedDict() res_dict = OrderedDict() master_dict[dataset]['results'] = res_dict # Save number of runs master_dict[dataset]['num_runs_stereo'] = getattr( cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) master_dict[dataset]['num_runs_multiview'] = getattr( cfg_orig, 'num_runs_{}_multiview'.format(cfg_orig.subset)) # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] bag_size_str = ['{}bag'.format(b) for b in bag_size_list] # Create empty dicts for scene in ['allseq'] + scene_list: res_dict[scene] = OrderedDict() for task in ['stereo', 'multiview']: res_dict[scene][task] = OrderedDict() res_dict[scene][task]['run_avg'] = OrderedDict() if task == 'multiview': for bag in bag_size_str + ['bag_avg']: res_dict[scene]['multiview']['run_avg'][ bag] = OrderedDict() # Stereo -- multiple runs t = time() cur_key = 'config_{}_stereo'.format(dataset) if cfg_orig.eval_stereo and cur_key in method and method[cur_key]: num_runs = getattr(cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'stereo' for scene in scene_list: cfg.scene = scene res_dict[scene]['stereo']['run_avg'] = OrderedDict() for run in range(num_runs): res_dict[scene]['stereo']['run_{}'.format( run)] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] # metric_list += ['matching_scores_epipolar'] metric_list += ['num_inliers'] if dataset != 'googleurban': metric_list += ['matching_scores_depth_projection'] metric_list += ['repeatability'] metric_list += ['qt_auc'] metric_list += ['timings'] for run in range(num_runs): # Compute and pack results cfg.run = run cur_dict = res_dict[scene]['stereo']['run_{}'.format(run)] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)(cur_dict, deprecated_images, cfg) print( ' -- Packing "{}"/"{}"/stereo, run: {}/{}, metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, metric, time() - t_cur)) # Compute average across runs, for stereo t_cur = time() pack_helper.average_stereo_over_runs(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for stereo t_cur = time() pack_helper.average_stereo_over_scenes(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing stereo in {:.01f} sec.'.format(time() - t)) else: print('Skipping "{}/stereo"'.format(dataset)) # Multiview -- multiple runs t = time() cur_key = 'config_{}_multiview'.format(dataset) if cfg_orig.eval_multiview and cur_key in method and method[cur_key]: num_runs = getattr(cfg, 'num_runs_{}_multiview'.format(cfg.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'multiview' for scene in scene_list: cfg.scene = scene for run in ['run_avg' ] + ['run_{}'.format(f) for f in range(num_runs)]: res_dict[scene]['multiview'][run] = OrderedDict() for bags_label in ['bag_avg'] + bag_size_str: res_dict[scene]['multiview'][run][ bags_label] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] metric_list += ['num_input_matches'] metric_list += ['qt_auc_colmap'] metric_list += ['ATE'] metric_list += ['colmap_stats'] for run in range(num_runs): for bag_size in bag_size_list: # Compute and pack results cfg.run = run cfg.bag_size = bag_size cur_dict = res_dict[scene]['multiview'] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)( cur_dict['run_{}'.format(run)]['{}bag'.format( bag_size)], deprecated_images, cfg) print( ' -- Packing "{}"/"{}"/multiview, run {}/{}, "{}", metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, '{}bag'.format(bag_size), metric, time() - t_cur)) # Compute average across bags any_key = random.choice([ key for key in cur_dict['run_{}'.format(run)] if ('bag' in key and key != 'bag_avg') ]) for metric in cur_dict['run_{}'.format(run)][any_key]: pack_helper.average_multiview_over_bags( cfg, cur_dict['run_{}'.format(run)], bag_size_list) # Compute average across runs, for multiview t_cur = time() pack_helper.average_multiview_over_runs(cfg, res_dict, num_runs, bag_size_str + ['bag_avg']) print( ' -- Packing "{}"/multiview: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for multiview t_cur = time() pack_helper.average_multiview_over_scenes( cfg, res_dict, num_runs, ['bag_avg'] + bag_size_str) print( ' -- Packing "{}"/multiview: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing multiview in {:.01f} sec.'.format( time() - t)) else: print('Skipping "{}/multiview"'.format(dataset)) # Add a unique identifier (equivalent to "submission id" in previous versions. if cfg.is_challenge: master_dict['uuid'] = get_uuid(cfg) # Dump packed result if not os.path.exists(cfg.path_pack): os.makedirs(cfg.path_pack) json_dump_file = os.path.join( cfg.path_pack, '{}.json'.format(cfg.method_dict['config_common']['json_label'])) print(' -- Saving to: "{}"'.format(json_dump_file)) with open(json_dump_file, 'w') as outfile: json.dump(master_dict, outfile, indent=2) # Add a short results summary. print() print('-- SUMMARY --') print('Subset: "{}"'.format(cfg.subset)) for dataset in DATASET_LIST: print() print('Dataset "{}"'.format(dataset)) if dataset in master_dict: # Stereo if 'stereo' in master_dict[dataset]['results'][ 'allseq'] and cfg.eval_stereo: print('-- Stereo mAA(10 deg): {:.05f}'.format( master_dict[dataset]['results']['allseq']['stereo'] ['run_avg']['qt_auc_10_th_0.1']['mean'])) for scene in master_dict[dataset]['results']: if scene != 'allseq': print('---- Scene "{}" -> Stereo mAA(10 deg): {:.05f}'. format( scene, master_dict[dataset]['results'][scene] ['stereo']['run_avg']['qt_auc_10_th_0.1'] ['mean'])) if 'multiview' in master_dict[dataset]['results'][ 'allseq'] and cfg.eval_multiview: print('-- Multiview mAA(10 deg): {:.05f}'.format( master_dict[dataset]['results']['allseq']['multiview'] ['run_avg']['bag_avg']['qt_auc_colmap_10']['mean'])) for scene in master_dict[dataset]['results']: if scene != 'allseq': print( '---- Scene "{}" -> Multiview mAA(10 deg): {:.05f}' .format( scene, master_dict[dataset]['results'][scene] ['multiview']['run_avg']['bag_avg'] ['qt_auc_colmap_10']['mean']))
def main(cfg): ''' Main routine for the benchmark ''' # Read data and splits for dataset in ['phototourism']: for subset in ['val', 'test']: setattr(cfg, 'scenes_{}_{}'.format(dataset, subset), './json/data/{}_{}.json'.format(dataset, subset)) setattr(cfg, 'splits_{}_{}'.format(dataset, subset), './json/bag_size/{}_{}.json'.format(dataset, subset)) # Read the list of methods and datasets method_list = load_json(cfg.json_method) for i, method in enumerate(method_list): print('Validating method {}/{}: "{}"'.format( i + 1, len(method_list), method['config_common']['json_label'])) validate_method(method, is_challenge=cfg.is_challenge) # Back up original config cfg_orig = deepcopy(cfg) job_dict = {} # Loop over methods, datasets/scenes, and tasks for method in method_list: # accumulate packing dependencies over datasets and runs all_stereo_jobs = [] all_multiview_jobs = [] all_relocalization_jobs = [] for dataset in ['phototourism']: # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] for scene in scene_list: print('Working on {}: {}/{}'.format( method['config_common']['json_label'], dataset, scene)) # For each task for task in ['stereo', 'multiview', 'relocalization']: # Skip if the key does not exist or it is empty cur_key = 'config_{}_{}'.format(dataset, task) if cur_key not in method or not method[cur_key]: print( 'Empty config for "{}", skipping!'.format(cur_key)) continue # Append method to config cfg = deepcopy(cfg_orig) cfg.method_dict = deepcopy(method) cfg.dataset = dataset cfg.task = task cfg.scene = scene # Features feature_jobs = create_eval_jobs([], 'feature', cfg, job_dict) # Matches match_jobs = create_eval_jobs(feature_jobs, 'match', cfg, job_dict) # Filter match_inlier_jobs = create_eval_jobs( match_jobs, 'filter', cfg, job_dict) # Empty dependencies stereo_jobs = [] multiview_jobs = [] relocalization_jobs = [] num_runs = getattr( cfg, 'num_runs_{}_{}'.format(cfg.subset, task)) for run in range(num_runs): cfg.run = run # Pose estimation and stereo evaluation if task == 'stereo' and cfg.eval_stereo: geom_model_jobs = create_eval_jobs( match_inlier_jobs, 'model', cfg, job_dict) stereo_jobs += create_eval_jobs( geom_model_jobs, 'stereo', cfg, job_dict) all_stereo_jobs += stereo_jobs # Visualization for stereo if task == 'stereo' and cfg.run_viz: eval_viz_stereo(stereo_jobs, cfg) # Multiview if task == 'multiview' and cfg.eval_multiview: multiview_jobs += eval_multiview( match_inlier_jobs, cfg, bag_size_list, bag_size_num) all_multiview_jobs += multiview_jobs # Visualization for colmap if task == 'multiview' and cfg.run_viz: eval_viz_colmap(multiview_jobs, cfg) if task == 'relocalization' and cfg.eval_relocalization: raise NotImplementedError( 'TODO relocalization task') # Packing -- can be skipped with --skip_packing=True # For instance, when only generating visualizations if not cfg.skip_packing: cfg = deepcopy(cfg_orig) cfg.method_dict = deepcopy(method) eval_packing( all_stereo_jobs + all_multiview_jobs + all_relocalization_jobs, cfg)
def main(cfg): '''Visualization of colmap points. Parameters ---------- cfg: Namespace Configurations for running this part of the code. ''' bag_size_json = load_json( getattr(cfg, 'splits_{}_{}'.format(cfg.dataset, cfg.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] # # Do not re-run if files already exist -- off for now # skip = True # for _bag_size in bag_size_list: # cfg_bag = deepcopy(cfg) # cfg_bag.bag_size = _bag_size # viz_folder_hq, viz_folder_lq = get_colmap_viz_folder(cfg_bag) # for _bag_id in range( # getattr(cfg_bag, # 'num_viz_colmap_subsets_bagsize{}'.format(_bag_size))): # if any([ # not os.path.exists( # os.path.join( # viz_folder_lq, # 'colmap-bagsize{:d}-bag{:02d}-image{:02d}.jpg'. # format(_bag_size, _bag_id, i))) # for i in range(_bag_size) # ]): # skip = False # break # if not os.path.exists( # os.path.join( # viz_folder_lq, # 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( # _bag_size, _bag_id))): # skip = False # break # if skip: # print(' -- already exists, skipping colmap visualization') # return print(' -- Visualizations, multiview: "{}/{}"'.format( cfg.dataset, cfg.scene)) t_start = time() # Create results folder if it does not exist for _bag_size in bag_size_list: cfg_bag = deepcopy(cfg) cfg_bag.bag_size = _bag_size viz_folder_hq, viz_folder_lq = get_colmap_viz_folder(cfg_bag) if not os.path.exists(viz_folder_hq): os.makedirs(viz_folder_hq) if not os.path.exists(viz_folder_lq): os.makedirs(viz_folder_lq) # Load keypoints keypoints_dict = load_h5(get_kp_file(cfg)) # Loop over bag sizes for _bag_size in bag_size_list: cfg_bag = deepcopy(cfg) cfg_bag.bag_size = _bag_size num_bags = getattr( cfg_bag, 'num_viz_colmap_subsets_bagsize{}'.format(_bag_size)) for _bag_id in range(num_bags): print( ' -- Visualizations, multiview: "{}/{}", bag_size={}, bag {}/{}' .format(cfg.dataset, cfg.scene, _bag_size, _bag_id + 1, num_bags)) # Retrieve list of images cfg_bag.bag_id = _bag_id images_in_bag = get_colmap_image_path_list(cfg_bag) # Retrieve reconstruction colmap_output_path = get_colmap_output_path(cfg_bag) # is_colmap_valid = os.path.exists( # os.path.join(colmap_output_path, '0')) best_index = get_best_colmap_index(cfg_bag) if best_index != -1: colmap_images = read_images_binary( os.path.join(colmap_output_path, str(best_index), 'images.bin')) for i, image_path in enumerate(images_in_bag): # Limit to 10 or so, even for bag size 25 if i >= cfg.max_num_images_viz_multiview: break # Load image and keypoints im, _ = load_image(image_path, use_color_image=True, crop_center=False, force_rgb=True) used = None key = os.path.splitext(os.path.basename(image_path))[0] if best_index != -1: for j in colmap_images: if key in colmap_images[j].name: # plot all keypoints used = colmap_images[j].point3D_ids != -1 break if used is None: used = [False] * keypoints_dict[key].shape[0] used = np.array(used) fig = plt.figure(figsize=(20, 20)) plt.imshow(im) plt.plot(keypoints_dict[key][~used, 0], keypoints_dict[key][~used, 1], 'r.', markersize=12) plt.plot(keypoints_dict[key][used, 0], keypoints_dict[key][used, 1], 'b.', markersize=12) plt.tight_layout() plt.axis('off') # TODO Ideally we would save to pdf # but it does not work on 16.04, so we do png instead # https://bugs.launchpad.net/ubuntu/+source/imagemagick/+bug/1796563 viz_folder_hq, viz_folder_lq = get_colmap_viz_folder(cfg_bag) viz_file_hq = os.path.join( viz_folder_hq, 'bagsize{:d}-bag{:02d}-image{:02d}.png'.format( _bag_size, _bag_id, i)) viz_file_lq = os.path.join( viz_folder_lq, 'bagsize{:d}-bag{:02d}-image{:02d}.jpg'.format( _bag_size, _bag_id, i)) plt.savefig(viz_file_hq, bbox_inches='tight') # Convert with imagemagick os.system('convert -quality 75 -resize \"400>\" {} {}'.format( viz_file_hq, viz_file_lq)) plt.close() if best_index != -1: colmap_points = read_points3d_binary( os.path.join(colmap_output_path, str(best_index), 'points3D.bin')) points3d = [] for k in colmap_points: points3d.append([ colmap_points[k].xyz[0], colmap_points[k].xyz[1], colmap_points[k].xyz[2] ]) points3d = np.array(points3d) points3d -= np.median(points3d, axis=0)[None, ...] points3d /= np.abs(points3d).max() + 1e-6 pcd = os.path.join( get_colmap_viz_folder(cfg_bag)[0], 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( _bag_size, _bag_id)) with open(pcd, 'w') as f: f.write('# .PCD v.7 - Point Cloud Data file format\n') f.write('VERSION .7\n') f.write('FIELDS x y z\n') f.write('SIZE 4 4 4\n') f.write('TYPE F F F\n') f.write('COUNT 1 1 1\n') f.write('WIDTH {}\n'.format(len(colmap_points))) f.write('HEIGHT 1\n') f.write('VIEWPOINT 0 0 0 1 0 0 0\n') f.write('POINTS {}\n'.format(len(colmap_points))) f.write('DATA ascii\n') for p in points3d: f.write('{:.05f} {:.05f} {:.05f}\n'.format( p[0], p[1], p[2])) copyfile( os.path.join( get_colmap_viz_folder(cfg_bag)[0], 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( _bag_size, _bag_id)), os.path.join( get_colmap_viz_folder(cfg_bag)[1], 'colmap-bagsize{:d}-bag{:02d}.pcd'.format( _bag_size, _bag_id))) print('done [{:.02f} s.]'.format(time() - t_start))
def validate_submission_files(sub_path, benchmark_repo_path, datasets, raw_data_path, logger): for dataset in datasets: raw_dataset_path = os.path.join(raw_data_path, dataset) # check if dataset folder exists sub_dataset_path = os.path.join(sub_path, dataset) if not os.path.isdir(sub_dataset_path): logger.add_new_log( 'Submission does not contain {} dataset.'.format(dataset)) continue # read seqs from json seqs = load_json( os.path.join(benchmark_repo_path, 'json/data/{}_test.json'.format(dataset))) for seq in seqs: # get number of image raw_seq_path = os.path.join(raw_dataset_path, seq) im_list = [ os.path.splitext(f)[0] for f in os.listdir(raw_seq_path) if (os.path.isfile(os.path.join(raw_seq_path, f)) and f.endswith(('png', 'jpg'))) ] num_im = len(im_list) # get all key pairs key_pairs = [ pair[0] + '-' + pair[1] for pair in list(product(im_list, im_list)) if pair[0] > pair[1] ] # check if seq folder exists sub_seq_path = os.path.join(sub_dataset_path, seq) if not os.path.isdir(sub_seq_path): logger.add_new_log( 'Submission does not contain {} sequence in {} dataset.'. format(seq, dataset)) continue # validate keypoints file kp_path = os.path.join(sub_seq_path, 'keypoints.h5') if not os.path.isfile(kp_path): logger.add_new_log( 'Submission does not contain keypoints file for {} sequence in {} dataset.' .format(seq, dataset)) else: keypoints = load_h5(kp_path) if sorted(list(keypoints.keys())) != sorted(im_list): logger.add_new_log( '{}-{}: Keypoints file does not contain all the image keys.' .format(dataset, seq)) if len(list(keypoints.values())[0].shape) != 2: logger.add_new_log( '{}-{}: Keypoints file is in wrong format.'.format( dataset, seq)) if list(keypoints.values())[0].shape[1] != 2: logger.add_new_log( '{}-{}: Keypoints file is in wrong format.'.format( dataset, seq)) # check number of keypoints if list(keypoints.values())[0].shape[0] > 8000: logger.add_new_log( '{}-{}: Keypoints file contains more than 8000 points.' .format(dataset, seq)) # check if match file exists first match_files = [ file for file in os.listdir(sub_seq_path) if os.path.isfile(os.path.join(sub_seq_path, file)) and file.startswith('match') ] # validate descriptor file desc_path = os.path.join(sub_seq_path, 'descriptors.h5') # much provide either descriptor file or match file if not os.path.isfile(desc_path) and len(match_files) == 0: logger.add_new_log( 'Submission does not contain descriptors file for {} sequence in {} dataset.' .format(seq, dataset)) elif not os.path.isfile(desc_path): pass else: descriptors = load_h5(desc_path) if sorted(list(descriptors.keys())) != sorted(im_list): logger.add_new_log( '{}-{}: Descriptors file does not contain all the image keys.' .format(dataset, seq)) if len(list(descriptors.values())[0].shape) != 2: logger.add_new_log( '{}-{}: Descriptors file is in wrong format'.format( dataset, seq)) if list(descriptors.values())[0].shape[1] < 64 or list( descriptors.values())[0].shape[1] > 2048: logger.add_new_log( '{}-{}: Descriptors file is in wrong format'.format( dataset, seq)) # check descriptor size desc_type, desc_size, desc_nbytes = get_descriptor_properties( {}, descriptors) if desc_nbytes > 512 and len(match_files) == 0: logger.add_new_log( '{}-{}: Descriptors size is larger than 512 bytes, you need to provide custom match file' .format(dataset, seq)) # validate match file # check match file name if 'matches.h5' in match_files: if len(match_files) != 1: logger.add_new_log( '{}-{}: matches.h5 exists. Do not need to provide any other match files.' .format(dataset, seq)) elif 'matches_multiview.h5' in match_files or 'matches_stereo_0.h5' in match_files or 'matches_stereo.h5' in match_files: if 'matches_multiview.h5' not in match_files: logger.add_new_log( '{}-{}: missing matches_multiview.h5'.format( dataset, seq)) if 'matches_stereo_0.h5' not in match_files and 'matches_stereo.h5' not in match_files: logger.add_new_log( '{}-{}: missing matches_stereo.h5'.format( dataset, seq)) if 'matches_stereo_1.h5' in match_files or 'matches_stereo_2.h5' in match_files: logger.add_new_log( '{}-{}: for 2021 challenge, we only run stereo once, no need to provide matches_stereo_1 and matches_stereo_2' .format(dataset, seq)) for match_file in match_files: matches = load_h5(os.path.join(sub_seq_path, match_file)) if len(matches.keys()) != len(key_pairs): logger.add_new_log( '{}-{}: Matches file contains wrong number of keys, should have {} keys, have {}.' .format(dataset, seq, len(key_pairs), len(matches.keys()))) elif sorted(list(matches.keys())) != sorted(key_pairs): logger.add_new_log( '{}-{}: Matches file contains worng keys, maybe the image names is in reverse order. Plase refer to submission instruction for proper custom match key naming convention' .format(dataset, seq)) if len(list(matches.values())[0].shape) != 2: logger.add_new_log( '{}-{}: Matches file is in wrong format.'.format( dataset, seq)) if list(matches.values())[0].shape[0] != 2: logger.add_new_log( '{}-{}: Matches file is in wrong format.'.format( dataset, seq))
def main(cfg): '''Main function. Takes config as input. ''' # Back up config cfg_orig = deepcopy(cfg) method = cfg_orig.method_dict # Add config options to the dict master_dict = OrderedDict() master_dict['config'] = method # Add date master_dict['properties'] = OrderedDict() master_dict['properties'][ 'processing_date'] = pack_helper.get_current_date() print('Adding processing date: {}'.format( master_dict['properties']['processing_date'])) # Add descriptor properties cfg_desc = deepcopy(cfg_orig) cfg_desc.dataset = 'phototourism' cfg_desc.scene = 'british_museum' try: descriptors_dict = load_h5(get_desc_file(cfg_desc)) desc_type, desc_size, desc_nbytes = pack_helper.get_descriptor_properties( cfg_desc, descriptors_dict) except Exception: desc_type = 'none' desc_size = 0 desc_nbytes = 0 master_dict['properties']['descriptor_type'] = desc_type master_dict['properties']['descriptor_size'] = desc_size master_dict['properties']['descriptor_nbytes'] = desc_nbytes print('Adding descriptor properties: {} {} ({} bytes)'.format( master_dict['properties']['descriptor_size'], master_dict['properties']['descriptor_type'], master_dict['properties']['descriptor_nbytes'])) # Read data and splits for dataset in ['phototourism']: setattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset), './json/data/{}_{}.json'.format(dataset, cfg_orig.subset)) setattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset), './json/bag_size/{}_{}.json'.format(dataset, cfg_orig.subset)) # Create empty dictionary master_dict[dataset] = OrderedDict() res_dict = OrderedDict() master_dict[dataset]['results'] = res_dict # Save number of runs master_dict[dataset]['num_runs_stereo'] = getattr( cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) master_dict[dataset]['num_runs_multiview'] = getattr( cfg_orig, 'num_runs_{}_multiview'.format(cfg_orig.subset)) # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] bag_size_str = ['{}bag'.format(b) for b in bag_size_list] # Create empty dicts for scene in ['allseq'] + scene_list: res_dict[scene] = OrderedDict() for task in ['stereo', 'multiview', 'relocalization']: res_dict[scene][task] = OrderedDict() res_dict[scene][task]['run_avg'] = OrderedDict() if task == 'multiview': for bag in bag_size_str + ['bag_avg']: res_dict[scene]['multiview']['run_avg'][ bag] = OrderedDict() # Stereo -- multiple runs t = time() cur_key = 'config_{}_stereo'.format(dataset) if cfg_orig.eval_stereo and cur_key in method and method[cur_key]: num_runs = getattr(cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'stereo' for scene in scene_list: cfg.scene = scene res_dict[scene]['stereo']['run_avg'] = OrderedDict() for run in range(num_runs): res_dict[scene]['stereo']['run_{}'.format( run)] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] # metric_list += ['matching_scores_epipolar'] metric_list += ['num_inliers'] metric_list += ['matching_scores_depth_projection'] metric_list += ['repeatability'] metric_list += ['qt_auc'] metric_list += ['timings'] for run in range(num_runs): # Compute and pack results cfg.run = run cur_dict = res_dict[scene]['stereo']['run_{}'.format(run)] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)(cur_dict, cfg) print( ' -- Packing "{}"/"{}"/stereo, run: {}/{}, metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, metric, time() - t_cur)) # Compute average across runs, for stereo t_cur = time() pack_helper.average_stereo_over_runs(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for stereo t_cur = time() pack_helper.average_stereo_over_scenes(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing stereo in {:.01f} sec.'.format(time() - t)) else: print('Skipping "{}/stereo"'.format(dataset)) # Multiview -- multiple runs t = time() cur_key = 'config_{}_multiview'.format(dataset) if cfg_orig.eval_multiview and cur_key in method and method[cur_key]: num_runs = getattr(cfg, 'num_runs_{}_multiview'.format(cfg.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'multiview' for scene in scene_list: cfg.scene = scene for run in ['run_avg' ] + ['run_{}'.format(f) for f in range(num_runs)]: res_dict[scene]['multiview'][run] = OrderedDict() for bags_label in ['bag_avg'] + bag_size_str: res_dict[scene]['multiview'][run][ bags_label] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] metric_list += ['num_input_matches'] metric_list += ['qt_auc_colmap'] metric_list += ['ATE'] metric_list += ['colmap_stats'] for run in range(num_runs): for bag_size in bag_size_list: # Compute and pack results cfg.run = run cfg.bag_size = bag_size cur_dict = res_dict[scene]['multiview'] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)( cur_dict['run_{}'.format(run)]['{}bag'.format( bag_size)], cfg) print( ' -- Packing "{}"/"{}"/multiview, run {}/{}, "{}", metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, '{}bag'.format(bag_size), metric, time() - t_cur)) # Compute average across bags for metric in cur_dict['run_{}'.format(run)]['25bag']: pack_helper.average_multiview_over_bags( cfg, cur_dict['run_{}'.format(run)], bag_size_list) # Compute average across runs, for multiview t_cur = time() pack_helper.average_multiview_over_runs(cfg, res_dict, num_runs, bag_size_str + ['bag_avg']) print( ' -- Packing "{}"/multiview: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for multiview t_cur = time() pack_helper.average_multiview_over_scenes( cfg, res_dict, num_runs, ['bag_avg'] + bag_size_str) print( ' -- Packing "{}"/multiview: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing multiview in {:.01f} sec.'.format( time() - t)) # Relocalization -- multiple runs # TODO else: print('Skipping "{}/multiview"'.format(dataset)) # Dump packed result print(' -- Saving to: "{}"'.format( cfg.method_dict['config_common']['json_label'])) if not os.path.exists(cfg.path_pack): os.makedirs(cfg.path_pack) json_dump_file = os.path.join( cfg.path_pack, '{}.json'.format(cfg.method_dict['config_common']['json_label'])) with open(json_dump_file, 'w') as outfile: json.dump(master_dict, outfile, indent=2)
def reformat_json(path_json): config = load_json(path_json) with open(path_json, 'w') as f: json.dump(config, f, indent=2)
if cfg.path_json == '': if cfg.is_challenge: raise RuntimeError('Must provide json file for challenge submission') if not cfg.kp_name: raise RuntimeError('Must define kp_name') if not cfg.desc_name: raise RuntimeError('Must define desc_name') if cfg.match_name and cfg.num_keypoints != -1: raise RuntimeError('Can not crop keypoints list with a custom matcher') cfg.kp_name = validate_label(cfg.kp_name) cfg.desc_name = validate_label(cfg.desc_name) cfg.match_name = validate_label(cfg.match_name) else: # read keypoints, descriptor, and match name from json method_list = load_json(cfg.path_json) if len(method_list)!=1: raise RuntimeError('Multiple method found in json file. Only support json fils with single method') cfg.method_dict = method_list[0] cfg.kp_name = cfg.method_dict['config_common']['keypoint'] cfg.desc_name = cfg.method_dict['config_common']['descriptor'] if cfg.method_dict['config_phototourism_stereo']['use_custom_matches']: cfg.match_name = {} for data_task in itertools.product(cfg.datasets, cfg.tasks): key = data_task[0]+'_'+data_task[1] cfg.match_name[key] = cfg.method_dict['config_'+key]['custom_matches_name'] if cfg.is_challenge: # compute hash for h5 files
import numpy as np from copy import deepcopy from utils.io_helper import load_json, load_h5, save_h5 from compute_model import compute_model if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--json_method', type=str, required=True) parser.add_argument('--import_path', type=Path, required=True) cfg = parser.parse_args() cfg.subset = 'test' cfg.dataset = 'phototourism' cfg.num_opencv_threads = 0 method_list = load_json(cfg.json_method) scene_list = load_json('json/data/phototourism_{}.json'.format(cfg.subset)) num_cores = cfg.num_opencv_threads if cfg.num_opencv_threads > 0 else int( len(os.sched_getaffinity(0)) * 0.9) for method in method_list: label = method['config_common']['json_label'] export_root = Path('../submission', label) export_root.mkdir(parents=True) cfg.method_dict = deepcopy(method) for seq in scene_list: print('Working on {}: {}/{}'.format(label, cfg.dataset, seq)) (export_root / seq).mkdir() for n in ['keypoints.h5', 'descriptors.h5', 'scores.h5']: