This function must return a list of these. One could think of cases where one could extract multiple learning curves from these runs, e.g. if each run is an independent training run of a neural network on the data. """ sr = filter(lambda r: not r.loss is None, sorted(runs, key=lambda r: r.budget)) value = [ [(r.budget, r.info['test_error']) for r in sr], ] return (value) bohb_logs_dir = '{}/search_space_{}/run{}-seed{}'.format( args.working_directory, args.space, args.run_id, args.seed) res = hpres.logged_results_to_HB_result(bohb_logs_dir) lcs_temp = res.get_learning_curves(lc_extractor=extract_HB_learning_curves) lcs = dict(lcs_temp) for key, value in lcs_temp.items(): if value == [[]]: del lcs[key] tool_tips = hpvis.default_tool_tips(res, lcs) #embed() inc_id = res.get_incumbent_id() id2conf = res.get_id2config_mapping() inc_trajectory = res.get_incumbent_trajectory()
if __name__ == '__main__': dirs = [ '{}/search_space_{}/darts/'.format(args.working_directory, args.space), '{}/search_space_{}/pc_darts/'.format(args.working_directory, args.space), '{}/search_space_{}/gdas/'.format(args.working_directory, args.space) ] opt_dict = {} for m, path in zip(['BOHB-DARTS', 'BOHB-PC-DARTS', 'BOHB-GDAS'], dirs): if m == 'BOHB-DARTS': ids = [int(x) for x in args.darts_id.split()] if m == 'BOHB-PC-DARTS': ids = [int(x) for x in args.pcdarts_id.split()] if m == 'BOHB-GDAS': ids = [int(x) for x in args.gdas_id.split()] runs = [] for seed in range(1, len(ids) + 1): logs_dir = path + 'run{}-seed{}'.format(ids[seed - 1], seed) res = hpres.logged_results_to_HB_result(logs_dir) run = list( filter(lambda r: not (r.info is None or r.loss is None), res.get_all_runs())) runs.append(run) opt_dict[m] = runs trajectory_plot(opt_dict)
plt.savefig(fig_name) plt.show() if __name__=='__main__': darts_logs_dir = '{}/search_space_3/darts/run3703139-seed1'.format( args.working_directory ) pcdarts_logs_dir = '{}/search_space_3/pc_darts/run3703171-seed1'.format( args.working_directory ) gdas_logs_dir = '{}/search_space_3/gdas/run3703155-seed1'.format( args.working_directory ) res_1 = hpres.logged_results_to_HB_result(darts_logs_dir) res_2 = hpres.logged_results_to_HB_result(pcdarts_logs_dir) res_3 = hpres.logged_results_to_HB_result(gdas_logs_dir) runs_1 = list(filter(lambda r: not (r.info is None or r.loss is None), res_1.get_all_runs())) runs_2 = list(filter(lambda r: not (r.info is None or r.loss is None), res_2.get_all_runs())) runs_3 = list(filter(lambda r: not (r.info is None or r.loss is None), res_3.get_all_runs())) opt_dict = { 'BOHB-DARTS': runs_1, 'BOHB-PC-DARTS': runs_2, 'BOHB-GDAS': runs_3 }
args.space, args.darts_id, args.seed) plt.savefig(fig_name) #plt.show() if __name__ == '__main__': darts_logs_dir = '{}/search_space_{}/darts/run{}-seed1'.format( args.working_directory, args.space, args.darts_id) #pcdarts_logs_dir = '{}/search_space_{}/pc_darts/run{}-seed1'.format( # args.working_directory, args.space, args.pcdarts_id #) #gdas_logs_dir = '{}/search_space_{}/gdas/run{}-seed1'.format( # args.working_directory, args.space, args.gdas_id #) res_1 = hpres.logged_results_to_HB_result(darts_logs_dir) #res_2 = hpres.logged_results_to_HB_result(pcdarts_logs_dir) #res_3 = hpres.logged_results_to_HB_result(gdas_logs_dir) runs_1 = list( filter(lambda r: not (r.info is None or r.loss is None), res_1.get_all_runs())) #runs_2 = list(filter(lambda r: not (r.info is None or r.loss is None), # res_2.get_all_runs())) #runs_3 = list(filter(lambda r: not (r.info is None or r.loss is None), # res_3.get_all_runs())) opt_dict = { 'BOHB-DARTS': runs_1, # 'BOHB-PC-DARTS': runs_2, # 'BOHB-GDAS': runs_3