This function must return a list of these. One could think
of cases where one could extract multiple learning curves
from these runs, e.g. if each run is an independent training
run of a neural network on the data.
"""
sr = filter(lambda r: not r.loss is None,
sorted(runs, key=lambda r: r.budget))
value = [
[(r.budget, r.info['test_error']) for r in sr],
]
return (value)
bohb_logs_dir = '{}/search_space_{}/run{}-seed{}'.format(
args.working_directory, args.space, args.run_id, args.seed)
res = hpres.logged_results_to_HB_result(bohb_logs_dir)
lcs_temp = res.get_learning_curves(lc_extractor=extract_HB_learning_curves)
lcs = dict(lcs_temp)
for key, value in lcs_temp.items():
if value == [[]]:
del lcs[key]
tool_tips = hpvis.default_tool_tips(res, lcs)
#embed()
inc_id = res.get_incumbent_id()
id2conf = res.get_id2config_mapping()
inc_trajectory = res.get_incumbent_trajectory()
if __name__ == '__main__':
dirs = [
'{}/search_space_{}/darts/'.format(args.working_directory, args.space),
'{}/search_space_{}/pc_darts/'.format(args.working_directory,
args.space),
'{}/search_space_{}/gdas/'.format(args.working_directory, args.space)
]
opt_dict = {}
for m, path in zip(['BOHB-DARTS', 'BOHB-PC-DARTS', 'BOHB-GDAS'], dirs):
if m == 'BOHB-DARTS':
ids = [int(x) for x in args.darts_id.split()]
if m == 'BOHB-PC-DARTS':
ids = [int(x) for x in args.pcdarts_id.split()]
if m == 'BOHB-GDAS':
ids = [int(x) for x in args.gdas_id.split()]
runs = []
for seed in range(1, len(ids) + 1):
logs_dir = path + 'run{}-seed{}'.format(ids[seed - 1], seed)
res = hpres.logged_results_to_HB_result(logs_dir)
run = list(
filter(lambda r: not (r.info is None or r.loss is None),
res.get_all_runs()))
runs.append(run)
opt_dict[m] = runs
trajectory_plot(opt_dict)
plt.savefig(fig_name)
plt.show()
if __name__=='__main__':
darts_logs_dir = '{}/search_space_3/darts/run3703139-seed1'.format(
args.working_directory
)
pcdarts_logs_dir = '{}/search_space_3/pc_darts/run3703171-seed1'.format(
args.working_directory
)
gdas_logs_dir = '{}/search_space_3/gdas/run3703155-seed1'.format(
args.working_directory
)
res_1 = hpres.logged_results_to_HB_result(darts_logs_dir)
res_2 = hpres.logged_results_to_HB_result(pcdarts_logs_dir)
res_3 = hpres.logged_results_to_HB_result(gdas_logs_dir)
runs_1 = list(filter(lambda r: not (r.info is None or r.loss is None),
res_1.get_all_runs()))
runs_2 = list(filter(lambda r: not (r.info is None or r.loss is None),
res_2.get_all_runs()))
runs_3 = list(filter(lambda r: not (r.info is None or r.loss is None),
res_3.get_all_runs()))
opt_dict = {
'BOHB-DARTS': runs_1,
'BOHB-PC-DARTS': runs_2,
'BOHB-GDAS': runs_3
}
args.space, args.darts_id, args.seed)
plt.savefig(fig_name)
#plt.show()
if __name__ == '__main__':
darts_logs_dir = '{}/search_space_{}/darts/run{}-seed1'.format(
args.working_directory, args.space, args.darts_id)
#pcdarts_logs_dir = '{}/search_space_{}/pc_darts/run{}-seed1'.format(
# args.working_directory, args.space, args.pcdarts_id
#)
#gdas_logs_dir = '{}/search_space_{}/gdas/run{}-seed1'.format(
# args.working_directory, args.space, args.gdas_id
#)
res_1 = hpres.logged_results_to_HB_result(darts_logs_dir)
#res_2 = hpres.logged_results_to_HB_result(pcdarts_logs_dir)
#res_3 = hpres.logged_results_to_HB_result(gdas_logs_dir)
runs_1 = list(
filter(lambda r: not (r.info is None or r.loss is None),
res_1.get_all_runs()))
#runs_2 = list(filter(lambda r: not (r.info is None or r.loss is None),
# res_2.get_all_runs()))
#runs_3 = list(filter(lambda r: not (r.info is None or r.loss is None),
# res_3.get_all_runs()))
opt_dict = {
'BOHB-DARTS': runs_1,
# 'BOHB-PC-DARTS': runs_2,
# 'BOHB-GDAS': runs_3