def main(cfg): '''Main function. Takes config as input. ''' # Back up config cfg_orig = deepcopy(cfg) method = cfg_orig.method_dict # Add config options to the dict master_dict = OrderedDict() master_dict['config'] = method # Add date master_dict['properties'] = OrderedDict() master_dict['properties'][ 'processing_date'] = pack_helper.get_current_date() print('Adding processing date: {}'.format( master_dict['properties']['processing_date'])) # Add submission flag master_dict['properties']['is_submission'] = cfg.is_submission print('Flagging as user submission: {}'.format(cfg.is_submission)) # Add descriptor properties cfg_desc = deepcopy(cfg_orig) cfg_desc.dataset = 'phototourism' cfg_desc.scene = 'british_museum' try: descriptors_dict = load_h5(get_desc_file(cfg_desc)) desc_type, desc_size, desc_nbytes = pack_helper.get_descriptor_properties( cfg_desc, descriptors_dict) except Exception: desc_type = 'none' desc_size = 0 desc_nbytes = 0 master_dict['properties']['descriptor_type'] = desc_type master_dict['properties']['descriptor_size'] = desc_size master_dict['properties']['descriptor_nbytes'] = desc_nbytes print('Adding descriptor properties: {} {} ({} bytes)'.format( master_dict['properties']['descriptor_size'], master_dict['properties']['descriptor_type'], master_dict['properties']['descriptor_nbytes'])) deprecated_images_all = load_json(cfg.json_deprecated_images) if cfg.dataset in deprecated_images_all and cfg.scene in deprecated_images_all[ cfg.dataset]: deprecated_images = deprecated_images_all[cfg.dataset][cfg.scene] else: deprecated_images = [] # Read data and splits DATASET_LIST = ['phototourism', 'pragueparks', 'googleurban'] for dataset in DATASET_LIST: # Skip if not in config if 'config_{}_stereo'.format( dataset) not in method and 'config_{}_multiview'.format( dataset) not in method: continue # Create empty dictionary master_dict[dataset] = OrderedDict() res_dict = OrderedDict() master_dict[dataset]['results'] = res_dict # Save number of runs master_dict[dataset]['num_runs_stereo'] = getattr( cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) master_dict[dataset]['num_runs_multiview'] = getattr( cfg_orig, 'num_runs_{}_multiview'.format(cfg_orig.subset)) # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] bag_size_str = ['{}bag'.format(b) for b in bag_size_list] # Create empty dicts for scene in ['allseq'] + scene_list: res_dict[scene] = OrderedDict() for task in ['stereo', 'multiview']: res_dict[scene][task] = OrderedDict() res_dict[scene][task]['run_avg'] = OrderedDict() if task == 'multiview': for bag in bag_size_str + ['bag_avg']: res_dict[scene]['multiview']['run_avg'][ bag] = OrderedDict() # Stereo -- multiple runs t = time() cur_key = 'config_{}_stereo'.format(dataset) if cfg_orig.eval_stereo and cur_key in method and method[cur_key]: num_runs = getattr(cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'stereo' for scene in scene_list: cfg.scene = scene res_dict[scene]['stereo']['run_avg'] = OrderedDict() for run in range(num_runs): res_dict[scene]['stereo']['run_{}'.format( run)] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] # metric_list += ['matching_scores_epipolar'] metric_list += ['num_inliers'] if dataset != 'googleurban': metric_list += ['matching_scores_depth_projection'] metric_list += ['repeatability'] metric_list += ['qt_auc'] metric_list += ['timings'] for run in range(num_runs): # Compute and pack results cfg.run = run cur_dict = res_dict[scene]['stereo']['run_{}'.format(run)] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)(cur_dict, deprecated_images, cfg) print( ' -- Packing "{}"/"{}"/stereo, run: {}/{}, metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, metric, time() - t_cur)) # Compute average across runs, for stereo t_cur = time() pack_helper.average_stereo_over_runs(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for stereo t_cur = time() pack_helper.average_stereo_over_scenes(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing stereo in {:.01f} sec.'.format(time() - t)) else: print('Skipping "{}/stereo"'.format(dataset)) # Multiview -- multiple runs t = time() cur_key = 'config_{}_multiview'.format(dataset) if cfg_orig.eval_multiview and cur_key in method and method[cur_key]: num_runs = getattr(cfg, 'num_runs_{}_multiview'.format(cfg.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'multiview' for scene in scene_list: cfg.scene = scene for run in ['run_avg' ] + ['run_{}'.format(f) for f in range(num_runs)]: res_dict[scene]['multiview'][run] = OrderedDict() for bags_label in ['bag_avg'] + bag_size_str: res_dict[scene]['multiview'][run][ bags_label] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] metric_list += ['num_input_matches'] metric_list += ['qt_auc_colmap'] metric_list += ['ATE'] metric_list += ['colmap_stats'] for run in range(num_runs): for bag_size in bag_size_list: # Compute and pack results cfg.run = run cfg.bag_size = bag_size cur_dict = res_dict[scene]['multiview'] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)( cur_dict['run_{}'.format(run)]['{}bag'.format( bag_size)], deprecated_images, cfg) print( ' -- Packing "{}"/"{}"/multiview, run {}/{}, "{}", metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, '{}bag'.format(bag_size), metric, time() - t_cur)) # Compute average across bags any_key = random.choice([ key for key in cur_dict['run_{}'.format(run)] if ('bag' in key and key != 'bag_avg') ]) for metric in cur_dict['run_{}'.format(run)][any_key]: pack_helper.average_multiview_over_bags( cfg, cur_dict['run_{}'.format(run)], bag_size_list) # Compute average across runs, for multiview t_cur = time() pack_helper.average_multiview_over_runs(cfg, res_dict, num_runs, bag_size_str + ['bag_avg']) print( ' -- Packing "{}"/multiview: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for multiview t_cur = time() pack_helper.average_multiview_over_scenes( cfg, res_dict, num_runs, ['bag_avg'] + bag_size_str) print( ' -- Packing "{}"/multiview: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing multiview in {:.01f} sec.'.format( time() - t)) else: print('Skipping "{}/multiview"'.format(dataset)) # Add a unique identifier (equivalent to "submission id" in previous versions. if cfg.is_challenge: master_dict['uuid'] = get_uuid(cfg) # Dump packed result if not os.path.exists(cfg.path_pack): os.makedirs(cfg.path_pack) json_dump_file = os.path.join( cfg.path_pack, '{}.json'.format(cfg.method_dict['config_common']['json_label'])) print(' -- Saving to: "{}"'.format(json_dump_file)) with open(json_dump_file, 'w') as outfile: json.dump(master_dict, outfile, indent=2) # Add a short results summary. print() print('-- SUMMARY --') print('Subset: "{}"'.format(cfg.subset)) for dataset in DATASET_LIST: print() print('Dataset "{}"'.format(dataset)) if dataset in master_dict: # Stereo if 'stereo' in master_dict[dataset]['results'][ 'allseq'] and cfg.eval_stereo: print('-- Stereo mAA(10 deg): {:.05f}'.format( master_dict[dataset]['results']['allseq']['stereo'] ['run_avg']['qt_auc_10_th_0.1']['mean'])) for scene in master_dict[dataset]['results']: if scene != 'allseq': print('---- Scene "{}" -> Stereo mAA(10 deg): {:.05f}'. format( scene, master_dict[dataset]['results'][scene] ['stereo']['run_avg']['qt_auc_10_th_0.1'] ['mean'])) if 'multiview' in master_dict[dataset]['results'][ 'allseq'] and cfg.eval_multiview: print('-- Multiview mAA(10 deg): {:.05f}'.format( master_dict[dataset]['results']['allseq']['multiview'] ['run_avg']['bag_avg']['qt_auc_colmap_10']['mean'])) for scene in master_dict[dataset]['results']: if scene != 'allseq': print( '---- Scene "{}" -> Multiview mAA(10 deg): {:.05f}' .format( scene, master_dict[dataset]['results'][scene] ['multiview']['run_avg']['bag_avg'] ['qt_auc_colmap_10']['mean']))
def main(cfg): '''Main function. Takes config as input. ''' # Back up config cfg_orig = deepcopy(cfg) method = cfg_orig.method_dict # Add config options to the dict master_dict = OrderedDict() master_dict['config'] = method # Add date master_dict['properties'] = OrderedDict() master_dict['properties'][ 'processing_date'] = pack_helper.get_current_date() print('Adding processing date: {}'.format( master_dict['properties']['processing_date'])) # Add descriptor properties cfg_desc = deepcopy(cfg_orig) cfg_desc.dataset = 'phototourism' cfg_desc.scene = 'british_museum' try: descriptors_dict = load_h5(get_desc_file(cfg_desc)) desc_type, desc_size, desc_nbytes = pack_helper.get_descriptor_properties( cfg_desc, descriptors_dict) except Exception: desc_type = 'none' desc_size = 0 desc_nbytes = 0 master_dict['properties']['descriptor_type'] = desc_type master_dict['properties']['descriptor_size'] = desc_size master_dict['properties']['descriptor_nbytes'] = desc_nbytes print('Adding descriptor properties: {} {} ({} bytes)'.format( master_dict['properties']['descriptor_size'], master_dict['properties']['descriptor_type'], master_dict['properties']['descriptor_nbytes'])) # Read data and splits for dataset in ['phototourism']: setattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset), './json/data/{}_{}.json'.format(dataset, cfg_orig.subset)) setattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset), './json/bag_size/{}_{}.json'.format(dataset, cfg_orig.subset)) # Create empty dictionary master_dict[dataset] = OrderedDict() res_dict = OrderedDict() master_dict[dataset]['results'] = res_dict # Save number of runs master_dict[dataset]['num_runs_stereo'] = getattr( cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) master_dict[dataset]['num_runs_multiview'] = getattr( cfg_orig, 'num_runs_{}_multiview'.format(cfg_orig.subset)) # Load data config scene_list = load_json( getattr(cfg_orig, 'scenes_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_json = load_json( getattr(cfg_orig, 'splits_{}_{}'.format(dataset, cfg_orig.subset))) bag_size_list = [b['bag_size'] for b in bag_size_json] bag_size_num = [b['num_in_bag'] for b in bag_size_json] bag_size_str = ['{}bag'.format(b) for b in bag_size_list] # Create empty dicts for scene in ['allseq'] + scene_list: res_dict[scene] = OrderedDict() for task in ['stereo', 'multiview', 'relocalization']: res_dict[scene][task] = OrderedDict() res_dict[scene][task]['run_avg'] = OrderedDict() if task == 'multiview': for bag in bag_size_str + ['bag_avg']: res_dict[scene]['multiview']['run_avg'][ bag] = OrderedDict() # Stereo -- multiple runs t = time() cur_key = 'config_{}_stereo'.format(dataset) if cfg_orig.eval_stereo and cur_key in method and method[cur_key]: num_runs = getattr(cfg_orig, 'num_runs_{}_stereo'.format(cfg_orig.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'stereo' for scene in scene_list: cfg.scene = scene res_dict[scene]['stereo']['run_avg'] = OrderedDict() for run in range(num_runs): res_dict[scene]['stereo']['run_{}'.format( run)] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] # metric_list += ['matching_scores_epipolar'] metric_list += ['num_inliers'] metric_list += ['matching_scores_depth_projection'] metric_list += ['repeatability'] metric_list += ['qt_auc'] metric_list += ['timings'] for run in range(num_runs): # Compute and pack results cfg.run = run cur_dict = res_dict[scene]['stereo']['run_{}'.format(run)] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)(cur_dict, cfg) print( ' -- Packing "{}"/"{}"/stereo, run: {}/{}, metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, metric, time() - t_cur)) # Compute average across runs, for stereo t_cur = time() pack_helper.average_stereo_over_runs(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for stereo t_cur = time() pack_helper.average_stereo_over_scenes(cfg, res_dict, num_runs) print( ' -- Packing "{}"/stereo: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing stereo in {:.01f} sec.'.format(time() - t)) else: print('Skipping "{}/stereo"'.format(dataset)) # Multiview -- multiple runs t = time() cur_key = 'config_{}_multiview'.format(dataset) if cfg_orig.eval_multiview and cur_key in method and method[cur_key]: num_runs = getattr(cfg, 'num_runs_{}_multiview'.format(cfg.subset)) cfg = deepcopy(cfg_orig) cfg.dataset = dataset cfg.task = 'multiview' for scene in scene_list: cfg.scene = scene for run in ['run_avg' ] + ['run_{}'.format(f) for f in range(num_runs)]: res_dict[scene]['multiview'][run] = OrderedDict() for bags_label in ['bag_avg'] + bag_size_str: res_dict[scene]['multiview'][run][ bags_label] = OrderedDict() # Create list of things to gather metric_list = [] metric_list += ['avg_num_keypoints'] metric_list += ['num_input_matches'] metric_list += ['qt_auc_colmap'] metric_list += ['ATE'] metric_list += ['colmap_stats'] for run in range(num_runs): for bag_size in bag_size_list: # Compute and pack results cfg.run = run cfg.bag_size = bag_size cur_dict = res_dict[scene]['multiview'] for metric in metric_list: t_cur = time() getattr(pack_helper, 'compute_' + metric)( cur_dict['run_{}'.format(run)]['{}bag'.format( bag_size)], cfg) print( ' -- Packing "{}"/"{}"/multiview, run {}/{}, "{}", metric: {} [{:.02f} s]' .format(dataset, scene, run + 1, num_runs, '{}bag'.format(bag_size), metric, time() - t_cur)) # Compute average across bags for metric in cur_dict['run_{}'.format(run)]['25bag']: pack_helper.average_multiview_over_bags( cfg, cur_dict['run_{}'.format(run)], bag_size_list) # Compute average across runs, for multiview t_cur = time() pack_helper.average_multiview_over_runs(cfg, res_dict, num_runs, bag_size_str + ['bag_avg']) print( ' -- Packing "{}"/multiview: averaging over {} run(s) [{:.02f} s]' .format(dataset, num_runs, time() - t_cur)) # Compute average across scenes, for multiview t_cur = time() pack_helper.average_multiview_over_scenes( cfg, res_dict, num_runs, ['bag_avg'] + bag_size_str) print( ' -- Packing "{}"/multiview: averaging over {} scene(s) [{:.02f} s]' .format(dataset, len(scene_list), time() - t_cur)) print(' -- Finished packing multiview in {:.01f} sec.'.format( time() - t)) # Relocalization -- multiple runs # TODO else: print('Skipping "{}/multiview"'.format(dataset)) # Dump packed result print(' -- Saving to: "{}"'.format( cfg.method_dict['config_common']['json_label'])) if not os.path.exists(cfg.path_pack): os.makedirs(cfg.path_pack) json_dump_file = os.path.join( cfg.path_pack, '{}.json'.format(cfg.method_dict['config_common']['json_label'])) with open(json_dump_file, 'w') as outfile: json.dump(master_dict, outfile, indent=2)