def plot_hp_search_optimizer(
eval_dir,
plot_dir,
hp_plot_order,
plot_parameter_names,
metric_scales,
xscale,
style,
line_label_format,
select,
):
changing_hp = hp_plot_order[-1]
for_plotting = get_optimizer_evaluation_results(eval_dir, hp_plot_order,
AVERAGING_NUMBER)
pupil_names = sorted(list(for_plotting.keys()))
result_types = sorted(list(for_plotting[pupil_names[0]].keys()))
regimes = sorted(list(
for_plotting[pupil_names[0]][result_types[0]].keys()))
fixed_hp_tmpl = create_plot_hp_layout(plot_dir, hp_plot_order, changing_hp)
# print("(plot_hp_search)plot_parameter_names:", plot_parameter_names)
xlabel = plot_parameter_names[changing_hp]
for pupil_name in pupil_names:
for res_type in result_types:
ylabel, yscale = get_y_specs(res_type, plot_parameter_names,
metric_scales)
for regime in sorted(regimes):
path = os.path.join(plot_dir, pupil_name, res_type, regime)
create_path(path)
data = for_plotting[pupil_name][res_type][regime]
launch_plotting(data, line_label_format, fixed_hp_tmpl, path,
xlabel, ylabel, xscale, yscale, style, select)
else:
initial_experiment_counter_value = biggest_idx + 1
env.grid_search_for_meta(
evaluation,
kwargs_for_pupil_building,
kwargs_for_optimizer_building,
build_pupil_hyperparameters=build_pupil_hyperparameters,
build_optimizer_hyperparameters=build_optimizer_hyperparameters,
other_hyperparameters=other_hyperparameters,
initial_experiment_counter_value=initial_experiment_counter_value,
**launch_kwargs
)
hp_names = get_hp_names_from_conf_file(parameter_set_file_name)
for_plotting = get_optimizer_evaluation_results(save_path, hp_names, AVERAGING_NUMBER)
best = get_best(for_plotting, 'optimizer')
metric_res = best['adam_prep']['loss']
best_on_valid = metric_res['validation']
print(' ' * 2 + 'loss' + ':', best_on_valid[1])
print_hps(hp_names, best_on_valid[0], 4)
best_conf = dict(list(zip(hp_names, best_on_valid[0])))
env.build_pupil(
batch_size=BATCH_SIZE,
**LSTM_SIZE,
regime='training_with_meta_optimizer',
additional_metrics=add_metrics,
going_to_limit_memory=True,
hp_names, metrics = get_hp_and_metric_names_from_pupil_eval_dir(eval_dir)
dataset_names = get_dataset_names_from_eval_dir(eval_dir)
if args.dataset_names is not None:
dataset_names = args.dataset_names.split(',')
for_plotting = get_pupil_evaluation_results(args.eval_dir, hp_names)
elif model == 'optimizer':
hp_names = get_hp_names_from_optimizer_eval_dir(eval_dir)
metrics, regimes = get_metric_names_and_regimes_from_optimizer_eval_dir(
eval_dir)
if args.regimes is not None:
regimes = args.regimes.split(',')
pupil_names = get_pupil_names_from_eval_dir(eval_dir)
if args.pupil_names is not None:
pupil_names = args.pupil_names.split(',')
for_plotting = get_optimizer_evaluation_results(eval_dir, hp_names,
AVERAGING_NUMBER)
if args.target_metrics is not None:
metrics = args.target_metrics.split(',')
best = get_best(for_plotting, model, value_filter=args.value_filter)
indents = [4, 8, 12]
print(best)
if model == 'pupil':
for dataset_name in dataset_names:
print('dataset:', dataset_name)
for metric in metrics:
b = best[dataset_name][metric]
print(' ' * indents[0] + metric + ':', b[1])
print_hps(hp_names, b[0], indents[1])
else:
