def sync_evaluated_results(database, save_dir, exp_name, prefix=None): if prefix is not None: res_exp_name = prefix + '_' + exp_name else: res_exp_name = exp_name split = 'val' # Check whether results of experiment exists chk_dir = os.path.join(save_dir, exp_name, 'Results_' + database) if not os.path.exists(chk_dir): print('Experiment {} is not yet ready. Omitting this directory'.format( exp_name)) return chk_file = os.path.join( save_dir, exp_name, 'Results_' + database, database + '_' + split + '_' + exp_name + '_edge.json') if os.path.isfile(chk_file): with open(chk_file, 'r') as f: eval_results = json.load(f) else: print('Creating json: {}'.format(res_exp_name)) eval_results = {} for measure in {'ods_f', 'ois_f', 'ap'}: tmp_fname = os.path.join( Path.seism_root_dir(), 'results', 'pr_curves', database, database + '_' + split + '_fb_' + res_exp_name + '_' + measure + '.txt') if not os.path.isfile(tmp_fname): print('Result not available') continue with open(tmp_fname, 'r') as f: eval_results[measure] = float(f.read().strip()) # Create edge json file if eval_results: print('Saving into .json: {}'.format(chk_file)) with open(chk_file, 'w') as f: json.dump(eval_results, f) for measure in eval_results: print('{}: {}'.format(measure, eval_results[measure]))
def sync_and_evaluate_one_folder(database, save_dir, exp_name, prefix=None, all_tasks_present=False): # dataset specific parameters if database == 'BSDS500': num_req_files = 200 gt_set = '' elif database == 'PASCALContext': if all_tasks_present: num_req_files = 1853 gt_set = 'val_all_tasks_present' else: num_req_files = 5105 gt_set = 'val' elif database == 'NYUD': num_req_files = 654 gt_set = 'val' else: raise NotImplementedError if prefix is None: res_exp_name = exp_name else: res_exp_name = prefix + '_' + exp_name # Check whether results of experiments exist chk_dir = os.path.join(save_dir, exp_name, 'Results_' + database, 'edge') if not os.path.exists(chk_dir): print('Experiment {} is not yet ready. Omitting this directory'.format( exp_name)) return # Check for filenames fnames = sorted(glob.glob(os.path.join(chk_dir, '*'))) if len(fnames) < num_req_files: print('Something is wrong with this directory. Check required: {}'. format(exp_name)) return elif len(fnames) > num_req_files: print('Already synced: {}'.format(exp_name)) else: # Seism path seism_cluster_dir = Path.seism_root_dir() # rsync to seism rsync_str = 'rsync -aP {}/ '.format(chk_dir) rsync_str += '[email protected]:{}/datasets/{}/{} '.format( seism_cluster_dir, database, res_exp_name) rsync_str += '--exclude=models --exclude=*.txt' print(rsync_str) os.system(rsync_str) # Submit the job subm_job_str = 'ssh [email protected] "source /home/sgeadmin/BIWICELL/common/settings.sh;' \ 'source /home/sgeadmin/BIWICELL/common/settings.sh;' subm_job_str += 'cp {}/parameters/HED.txt {}/parameters/{}.txt; ' \ ''.format(seism_cluster_dir, seism_cluster_dir, res_exp_name) subm_job_str += 'qsub -N evalFb -t 1-102 {}/eval_in_cluster.py {} read_one_cont_png fb 1 102 {} {}"' \ ''.format(seism_cluster_dir, res_exp_name, database, gt_set) print(subm_job_str) os.system(subm_job_str) # Leave the proof of submission os.system('touch {}/SYNCED_TO_REINHOLD'.format(chk_dir))