def plot_hp_search_pupil(
eval_dir,
plot_dir,
hp_plot_order,
plot_parameter_names,
metric_scales,
xscale,
style,
line_label_format,
select,
):
changing_hp = hp_plot_order[-1]
for_plotting = get_pupil_evaluation_results(eval_dir, hp_plot_order)
dataset_names = sorted(list(for_plotting.keys()))
result_types = sorted(list(for_plotting[dataset_names[0]].keys()))
fixed_hp_tmpl = create_plot_hp_layout(plot_dir, hp_plot_order, changing_hp)
xlabel = plot_parameter_names[changing_hp]
for dataset_name in dataset_names:
for res_type in result_types:
ylabel, yscale = get_y_specs(res_type, plot_parameter_names,
metric_scales)
path = os.path.join(plot_dir, dataset_name, res_type)
create_path(path)
data = for_plotting[dataset_name][res_type]
launch_plotting(data, line_label_format, fixed_hp_tmpl, path,
xlabel, ylabel, xscale, yscale, style, select)
def plot_lines_from_diff_hp_searches(
line_retrieve_inf,
plot_dir,
changing_hp,
plot_parameter_names,
metric_scales,
xscale,
style,
x_select,
model,
):
# print(line_retrieve_inf)
lines = retrieve_lines(line_retrieve_inf, x_select, model,
AVERAGING_NUMBER)
xlabel = plot_parameter_names[changing_hp]
create_path(plot_dir)
plot_description_file = os.path.join(plot_dir, 'description.txt')
with open(plot_description_file, 'w') as f:
f.write(nested2string(line_retrieve_inf))
for res_type, plot_data in lines.items():
ylabel, yscale = get_y_specs(res_type, plot_parameter_names,
metric_scales)
file_name_without_ext = add_index_to_filename_if_needed(
os.path.join(plot_dir, res_type))
plot_outer_legend(plot_data, None, xlabel, ylabel, xscale, yscale,
file_name_without_ext, style)
def plot_hp_search_optimizer(
eval_dir,
plot_dir,
hp_plot_order,
plot_parameter_names,
metric_scales,
xscale,
style,
line_label_format,
select,
):
changing_hp = hp_plot_order[-1]
for_plotting = get_optimizer_evaluation_results(eval_dir, hp_plot_order,
AVERAGING_NUMBER)
pupil_names = sorted(list(for_plotting.keys()))
result_types = sorted(list(for_plotting[pupil_names[0]].keys()))
regimes = sorted(list(
for_plotting[pupil_names[0]][result_types[0]].keys()))
fixed_hp_tmpl = create_plot_hp_layout(plot_dir, hp_plot_order, changing_hp)
# print("(plot_hp_search)plot_parameter_names:", plot_parameter_names)
xlabel = plot_parameter_names[changing_hp]
for pupil_name in pupil_names:
for res_type in result_types:
ylabel, yscale = get_y_specs(res_type, plot_parameter_names,
metric_scales)
for regime in sorted(regimes):
path = os.path.join(plot_dir, pupil_name, res_type, regime)
create_path(path)
data = for_plotting[pupil_name][res_type][regime]
launch_plotting(data, line_label_format, fixed_hp_tmpl, path,
xlabel, ylabel, xscale, yscale, style, select)
sys.path.remove(str(parent))
except ValueError: # Already removed
pass
from learning_to_learn.useful_functions import create_path, InvalidArgumentError
COLORS = ['r', 'g', 'b', 'k', 'c', 'magenta', 'brown', 'darkviolet', 'pink', 'yellow', 'gray', 'orange', 'olive']
DPI = 900
FORMATS = ['pdf', 'png']
FONT = {'family': 'Verdana',
'weight': 'normal'}
rc('font', **FONT)
save_path = sys.argv[1] # no extension!
create_path(save_path, file_name_is_in_path=True)
xlabel = sys.argv[2]
ylabel = sys.argv[3]
xscale = sys.argv[4]
yscale = sys.argv[5]
limit = sys.argv[6]
if limit == 'None':
limit = None
else:
limit = int(limit)
shift = sys.argv[7]
if limit == 'None':
shift = None
else:
shift = int(shift)
plot_data = sys.argv[8:]
def plot_outer_legend(plot_data, description, xlabel, ylabel, xscale, yscale,
file_name_without_ext, style):
# print("(plot_helpers.plot_outer_legend)xlabel:", xlabel)
# print("(plot_helpers.plot_outer_legend)plot_data:", plot_data)
rc('font', **FONT)
plt.clf()
plt.subplot(111)
for_plotlib = [list(), list()]
for label, line_data in plot_data.items():
for_plotlib[0].append(label)
for_plotlib[1].append(line_data)
for_plotlib = synchronous_sort(for_plotlib,
0,
lambda_func=lambda x: float(x)
if isnumber(x) else x)
lines = list()
labels = list()
if style['no_line']:
linestyle = 'None'
else:
linestyle = 'solid'
# print("(plot_helpers.plot_outer_legend)linestyle:", linestyle)
for idx, (label, line_data) in enumerate(zip(*for_plotlib)):
# if idx == 0:
# print("(plot_helpers.plot_outer_legend)line_data:", line_data)
if label is None or label == 'None':
label = ''
labels.append(label)
if idx > len(COLORS) - 1:
color = [
random.uniform(0, 1),
random.uniform(0, 1),
random.uniform(0, 1)
]
else:
color = COLORS[idx]
if len(line_data) > 2:
errors = line_data[2]
errors = [0. if e is None else e for e in errors]
else:
errors = None
if style['error'] == 'fill':
yerr = None
ym = [y - e for y, e in zip(line_data[1], errors)]
yp = [y + e for y, e in zip(line_data[1], errors)]
plt.fill_between(
line_data[0],
ym,
yp,
alpha=.4,
color=color,
)
elif style['error'] == 'bar':
yerr = errors
else:
yerr = None
# print("(plot_helpers.plot_outer_legend)yerr:", yerr)
# print("(plot_helpers.plot_outer_legend)line_data:", line_data)
lines.append(
plt.errorbar(
line_data[0],
line_data[1],
yerr=yerr,
marker=style['marker'],
color=color,
label=label,
ls=linestyle,
)[0])
# print("(plot_helpers.plot_outer_legend)labels:", labels)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
scale_kwargs = dict()
if xscale == 'symlog':
linthreshx = get_linthreshx(for_plotlib[1], )
scale_kwargs['linthreshx'] = linthreshx
plt.xscale(xscale, **scale_kwargs)
plt.yscale(yscale)
there_is_labels = False
for label in labels:
if len(label) > 0:
there_is_labels = there_is_labels or True
if there_is_labels:
handler_map = dict(
list(
zip(lines,
[HandlerLine2D(numpoints=1) for _ in range(len(lines))])))
# print("(plot_helpers.plot_outer_legend)handler_map:", handler_map)
ax = plt.gca()
handles, labels = ax.get_legend_handles_labels()
handles = [
h[0] if isinstance(h, container.ErrorbarContainer) else h
for h in handles
]
lgd = ax.legend(
handles,
labels,
bbox_to_anchor=(1.05, 1),
loc=2,
borderaxespad=0.,
handler_map=handler_map,
)
bbox_extra_artists = [lgd]
else:
bbox_extra_artists = ()
# lgd = plt.legend(
# bbox_to_anchor=(1.05, 1),
# loc=2,
# borderaxespad=0.,
# handler_map=handler_map,
# )
for format in FORMATS:
if format == 'pdf':
fig_path = os.path.join(file_name_without_ext + '.pdf')
elif format == 'png':
fig_path = os.path.join(file_name_without_ext + '.png')
else:
fig_path = None
create_path(fig_path, file_name_is_in_path=True)
r = plt.savefig(fig_path,
bbox_extra_artists=bbox_extra_artists,
bbox_inches='tight')
# print("%s %s %s %s:" % (pupil_name, res_type, regime, format), r)
if description is not None:
description_file = os.path.join(file_name_without_ext + '.txt')
with open(description_file, 'w') as f:
f.write(description)
lines_by_metrics[metric][label + '_' + dataset] = select_by_x(
extract_line_from_file(metric_data[dataset]), x_select)
else:
lines_by_metrics[metric][label] = select_by_x(
extract_line_from_file(metric_data[datasets_present[0]]),
x_select)
style = dict(
no_line=args.no_line,
error=args.error,
marker=args.marker,
)
plot_parameter_names = get_parameter_names(args.hp_names_file)
xscale = args.xscale
metric_scales = parse_metric_scales_str(args.metric_scales)
create_path(args.plot_name, file_name_is_in_path=True)
for metric, lines_for_metric in lines_by_metrics.items():
file_name = args.plot_name + '_' + metric
ylabel = plot_parameter_names[metric]
if metric in metric_scales:
yscale = metric_scales[metric]
else:
yscale = 'linear'
xlabel = args.xlabel
description = nested2string(lines_for_metric)
plot_outer_legend(
lines_for_metric,
description,
xlabel,
ylabel,
xscale,
types = list()
for s in hp_names_with_types:
spl = s.split(',')
hp_names.append(spl[0])
types.append(spl[1])
span = args.span.split(':')
points = list()
for string in span:
points.append(get_points_from_range(string))
indent = 4
for conf in confs:
path_to_conf = os.path.join(base_path, conf)
create_path(path_to_conf, file_name_is_in_path=True)
tmpl = get_tmpl(hp_names)
file_string = ''
file_string += tmpl % tuple(hp_names) + '\n'
file_string += tmpl % tuple(types) + '\n'
for values in points:
file_string += get_tmpl(values) % tuple(values) + '\n'
file_string += '%s' % args.num_repeats
with open(path_to_conf, 'w') as f:
f.write(file_string)
if args.verbose:
print('\n' + ' ' * indent + conf)
print(file_string)
if not args.no_git:
metric_scales = parse_metric_scales_str(args.metric_scales)
hp_plot_order = args.hp_order.split(',')
# abspath = os.path.abspath(__file__)
# dname = os.path.dirname(abspath)
# os.chdir(dname)
plot_parameter_names = get_parameter_names(
os.path.join(*(os.path.split(os.path.abspath(__file__))[:-1] +
(args.hp_names_file, ))))
xscale = args.xscale
for eval_dir in eval_dirs:
print(eval_dir)
plot_dir = os.path.join(*list(os.path.split(eval_dir)[:-1]) +
[args.plot_dir])
create_path(plot_dir)
if args.model == 'optimizer':
try:
plot_hp_search_optimizer(
eval_dir,
plot_dir,
hp_plot_order,
plot_parameter_names,
metric_scales,
args.xscale,
style,
args.line_label_format,
select,
)
except MissingHPError as e:
print(
from shutil import copyfile, copytree
from pathlib import Path # if you haven't already done so
file = Path(__file__).resolve()
parent, root = file.parent, file.parents[2]
sys.path.append(str(root))
try:
sys.path.remove(str(parent))
except ValueError: # Already removed
pass
from learning_to_learn.useful_functions import is_int, create_path
source_dir = sys.argv[1]
dest_dir = sys.argv[2]
create_path(dest_dir)
contents = os.listdir(source_dir)
exp_description_files = list()
for entry in contents:
if entry[-4:] == '.txt' and is_int(entry[:-4]):
exp_description_files.append(entry)
filtered = list()
hp_sets = list()
for file_name in exp_description_files:
with open(os.path.join(source_dir, file_name), 'r') as f:
hp_set = tuple(f.read().split('\n')[0].split())
if hp_set not in hp_sets:
hp_sets.append(hp_set)
help="Output file name. Path relative to this script.")
parser.add_argument(
'-i',
"--inter_path",
help="Path between model folder and results folder. default is 'test'",
default='test',
)
parser.add_argument("-v",
"--verbose",
help="Regulate verbosity",
action='store_true')
args = parser.parse_args()
model_paths = parse_path_comb(args.model_paths, filter_=False)
results = dict()
for model_path in model_paths:
results[model_path] = pupil_test_results_summarize(model_path,
args.inter_path)
string = stats_string_for_all_models(results, 2)
create_path(args.output, file_name_is_in_path=True)
with open(args.output, 'w') as f:
f.write(string)
if args.verbose:
print(string)
hp_names = list()
types = list()
for s in hp_names_with_types:
spl = s.split(',')
hp_names.append(spl[0])
types.append(spl[1])
span = args.span.split(':')
points = list()
for string in span:
points.append(get_points_from_range(string))
indent = 4
for conf in confs:
create_path(conf, file_name_is_in_path=True)
tmpl = get_tmpl(hp_names)
file_string = ''
file_string += tmpl % tuple(hp_names) + '\n'
file_string += tmpl % tuple(types) + '\n'
for values in points:
file_string += get_tmpl(values) % tuple(values) + '\n'
file_string += '%s' % args.num_repeats
with open(conf, 'w') as f:
f.write(file_string)
if args.verbose:
print('\n' + ' ' * indent + conf)
print(file_string)
if not args.no_git:
def plot_outer_legend(
plot_data,
description,
xlabel,
ylabel,
xscale,
yscale,
file_name_without_ext,
style,
shifts=None,
legend_pos='outside',
only_color_as_marker_in_legend=False,
labels_of_drawn_lines=None,
formats=None,
save=True,
show=False,
axes_to_invert=(),
select=None,
bottom=None,
dpi=300,
size_factor=1,
grid=False,
which_grid='major',
additional_artists=None,
):
if shifts is None:
shifts = [0, 0]
if style['no_line']:
linestyle = 'None'
else:
linestyle = 'solid'
if 'linewidth' in style and style['linewidth'] is not None:
lw = style['linewidth']
else:
lw = 1
if select is not None:
plot_data.filter_points(select)
plot_data.shift_lines('x', shifts[0])
plot_data.shift_lines('y', shifts[1])
if labels_of_drawn_lines is not None:
plot_data = python.filter_dict_by_keys(plot_data,
labels_of_drawn_lines)
if formats is None:
formats = FORMATS
rc('font', **FONT)
plt.clf()
plt.subplot(111)
if plot_data.all_labels_are_numbers():
plot_data.set_float_sorting_key()
lines = list()
for idx, (label, line_data) in enumerate(plot_data.items()):
if label is None or label == 'None':
label = ''
if idx > len(COLORS) - 1:
color = [
random.uniform(0, 1),
random.uniform(0, 1),
random.uniform(0, 1)
]
else:
color = COLORS[idx]
if style['error'] == 'fill':
bounds = line_data.get_bounds()
x_err, y_err = None, None
plt.fill_between(
bounds[0][0],
bounds[0][1],
bounds[1][1],
alpha=.4,
color=color,
)
elif style['error'] == 'bar':
x_err, y_err = line_data.get('x_err'), line_data.get('y_err')
else:
x_err, y_err = None, None
lines.append(
plt.errorbar(
line_data['x'],
line_data['y'],
yerr=y_err,
xerr=x_err,
marker=style['marker'],
color=color,
label=label,
ls=linestyle,
lw=lw,
)[0])
if 'x' in axes_to_invert:
plt.gca().invert_xaxis()
if 'y' in axes_to_invert:
plt.gca().invert_yaxis()
if bottom is not None:
plt.gca().set_ylim(bottom=bottom)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
if select is not None:
plt.xlim(select)
scale_kwargs = dict()
if xscale == 'symlog':
linthreshx = get_linthresh(
plot_data.get_all_values('x') +
plot_data.get_all_values('x_err'), )
scale_kwargs['linthreshx'] = linthreshx
plt.xscale(xscale, **scale_kwargs)
plt.yscale(yscale)
if which_grid in ['minor', 'both']:
plt.minorticks_on()
plt.grid(b=grid, which=which_grid)
if additional_artists is not None:
add_special_artists(additional_artists)
if plot_data.labels_are_provided() and legend_pos != 'None':
bbox_extra_artists = [
add_legend(lines, legend_pos, only_color_as_marker_in_legend)
]
else:
bbox_extra_artists = ()
fig = plt.gcf()
size = fig.get_size_inches()
fig.set_size_inches(size[0] * size_factor, size[1] * size_factor)
if save:
for format in formats:
if format == 'pdf':
fig_path = file_name_without_ext + '.pdf'
elif format == 'png':
fig_path = file_name_without_ext + '.png'
else:
fig_path = None
create_path(fig_path, file_name_is_in_path=True)
r = plt.savefig(
fig_path,
bbox_extra_artists=bbox_extra_artists,
bbox_inches='tight',
dpi=dpi,
)
if show:
plt.show()
if description is not None:
description_file = os.path.join(file_name_without_ext + '.txt')
with open(description_file, 'w') as f:
f.write(description)
handler_map = dict(
zip(sorted(handles, key=lambda x: x.get_label()), [
PatchHandler(color=color) for _, color in zip(
sorted(handles, key=lambda x: x.get_label()), COLORS)
]))
lgd = ax.legend(
# handles,
# labels,
**pos_dict,
borderaxespad=0.,
handler_map=handler_map)
bbox_extra_artists = [lgd]
else:
bbox_extra_artists = []
if not args.show:
for format_ in args.formats:
if format_ == 'pdf':
fig_path = args.output + '.pdf'
elif format_ == 'png':
fig_path = args.output + '.png'
else:
fig_path = None
create_path(fig_path, file_name_is_in_path=True)
r = plt.savefig(fig_path,
bbox_extra_artists=bbox_extra_artists,
bbox_inches='tight')
if args.show:
plt.show()
