def generate_parameter_diff_plots(exp_dir,
output_dir,
plot,
comparison_dir=None,
start_iter=1,
latex_report=None):
# Parameter changes
assert start_iter >= 1
comparison_dir = [] if comparison_dir is None else comparison_dir
dirs = [exp_dir] + comparison_dir
index = 0
stats_per_dir = {}
key_file = {
"Parameter differences": "parameter.diff",
"Relative parameter differences": "relative_parameter.diff"
}
stats_per_dir = {}
for dir in dirs:
stats_per_dir[dir] = {}
for key in key_file:
stats_per_dir[dir][key] = (
log_parse.parse_progress_logs_for_param_diff(dir, key))
# write down the stats for the main experiment directory
for diff_type in key_file:
with open("{0}/{1}".format(output_dir, key_file[diff_type]), "w") as f:
diff_per_component_per_iter = (
stats_per_dir[exp_dir][diff_type]['progress_per_component'])
component_names = (
stats_per_dir[exp_dir][diff_type]['component_names'])
max_iter = stats_per_dir[exp_dir][diff_type]['max_iter']
f.write(" ".join(["Iteration"] + component_names) + "\n")
total_missing_iterations = 0
gave_user_warning = False
for iter in range(max_iter + 1):
iter_data = [str(iter)]
for c in component_names:
try:
iter_data.append(
str(diff_per_component_per_iter[c][iter]))
except KeyError:
total_missing_iterations += 1
iter_data.append("NA")
if (total_missing_iterations / len(component_names) > 20
and not gave_user_warning):
logger.warning(
"There are more than {0} missing "
"iterations per component. "
"Something might be wrong.".format(
total_missing_iterations / len(component_names)))
gave_user_warning = True
f.write(" ".join(iter_data) + "\n")
if plot:
# get the component names
diff_type = key_file.keys()[0]
main_component_names = stats_per_dir[exp_dir][diff_type][
'progress_per_component'].keys()
main_component_names.sort()
plot_component_names = set(main_component_names)
for dir in dirs:
try:
component_names = set(stats_per_dir[dir][diff_type]
['progress_per_component'].keys())
plot_component_names = plot_component_names.intersection(
component_names)
except KeyError:
continue
plot_component_names = list(plot_component_names)
plot_component_names.sort()
if plot_component_names != main_component_names:
logger.warning("The components in all the neural networks in the "
"given experiment dirs are not the same, "
"so comparison plots are provided only for common "
"component names. "
"Make sure that these are comparable experiments "
"before analyzing these plots.")
assert main_component_names
fig = plt.figure()
logger.info("Generating parameter-difference plots for the "
"following components:{0}".format(
', '.join(main_component_names)))
for component_name in main_component_names:
fig.clf()
index = 0
plots = []
for dir in dirs:
color_val = g_plot_colors[index]
index += 1
iter_stats = []
try:
for diff_type in [
'Parameter differences',
'Relative parameter differences'
]:
iter_stats.append(
np.array(
sorted(stats_per_dir[dir][diff_type]
['progress_per_component']
[component_name].items())))
except KeyError as e:
# this component is not available in this network so lets
# not just plot it
if dir == exp_dir:
raise Exception("No parameter differences were "
"available even in the main "
"experiment dir for the component "
"{0}. Something went wrong: "
"{1}.".format(component_name, str(e)))
continue
ax = plt.subplot(211)
mp, = ax.plot(iter_stats[0][:, 0],
iter_stats[0][:, 1],
color=color_val,
label="Parameter Differences {0}".format(dir))
plots.append(mp)
ax.set_ylabel('Parameter Differences')
ax.grid(True)
ax = plt.subplot(212)
mp, = ax.plot(iter_stats[1][:, 0],
iter_stats[1][:, 1],
color=color_val,
label="Relative Parameter "
"Differences {0}".format(dir))
ax.set_xlabel('Iteration')
ax.set_ylabel('Relative Parameter Differences')
ax.grid(True)
lgd = plt.legend(handles=plots,
loc='lower center',
bbox_to_anchor=(0.5, -0.5 + len(dirs) * -0.2),
ncol=1,
borderaxespad=0.)
plt.grid(True)
fig.suptitle("Parameter differences at {comp_name}".format(
comp_name=component_name))
comp_name = latex_compliant_name(component_name)
figfile_name = '{dir}/param_diff_{comp_name}.pdf'.format(
dir=output_dir, comp_name=comp_name)
fig.savefig(figfile_name,
bbox_extra_artists=(lgd, ),
bbox_inches='tight')
if latex_report is not None:
latex_report.add_figure(
figfile_name,
"Parameter differences at {0}".format(component_name))
def generate_parameter_diff_plots(exp_dir, output_dir, plot,
comparison_dir=None, start_iter=1,
latex_report=None):
# Parameter changes
assert start_iter >= 1
comparison_dir = [] if comparison_dir is None else comparison_dir
dirs = [exp_dir] + comparison_dir
index = 0
stats_per_dir = {}
key_file = {"Parameter differences": "parameter.diff",
"Relative parameter differences": "relative_parameter.diff"}
stats_per_dir = {}
for dir in dirs:
stats_per_dir[dir] = {}
for key in key_file:
stats_per_dir[dir][key] = (
log_parse.parse_progress_logs_for_param_diff(dir, key))
# write down the stats for the main experiment directory
for diff_type in key_file:
with open("{0}/{1}".format(output_dir, key_file[diff_type]), "w") as f:
diff_per_component_per_iter = (
stats_per_dir[exp_dir][diff_type]['progress_per_component'])
component_names = (
stats_per_dir[exp_dir][diff_type]['component_names'])
max_iter = stats_per_dir[exp_dir][diff_type]['max_iter']
f.write(" ".join(["Iteration"] + component_names)+"\n")
total_missing_iterations = 0
gave_user_warning = False
for iter in range(max_iter + 1):
iter_data = [str(iter)]
for c in component_names:
try:
iter_data.append(
str(diff_per_component_per_iter[c][iter]))
except KeyError:
total_missing_iterations += 1
iter_data.append("NA")
if (total_missing_iterations/len(component_names) > 20
and not gave_user_warning):
logger.warning("There are more than {0} missing "
"iterations per component. "
"Something might be wrong.".format(
total_missing_iterations
/ len(component_names)))
gave_user_warning = True
f.write(" ".join(iter_data)+"\n")
if plot:
# get the component names
diff_type = key_file.keys()[0]
main_component_names = stats_per_dir[exp_dir][diff_type][
'progress_per_component'].keys()
main_component_names.sort()
plot_component_names = set(main_component_names)
for dir in dirs:
try:
component_names = set(stats_per_dir[dir][diff_type][
'progress_per_component'].keys())
plot_component_names = plot_component_names.intersection(
component_names)
except KeyError:
continue
plot_component_names = list(plot_component_names)
plot_component_names.sort()
if plot_component_names != main_component_names:
logger.warning("The components in all the neural networks in the "
"given experiment dirs are not the same, "
"so comparison plots are provided only for common "
"component names. "
"Make sure that these are comparable experiments "
"before analyzing these plots.")
assert main_component_names
fig = plt.figure()
logger.info("Generating parameter-difference plots for the "
"following components:{0}".format(
', '.join(main_component_names)))
for component_name in main_component_names:
fig.clf()
index = 0
plots = []
for dir in dirs:
color_val = g_plot_colors[index]
index += 1
iter_stats = []
try:
for diff_type in ['Parameter differences',
'Relative parameter differences']:
iter_stats.append(np.array(
sorted(stats_per_dir[dir][diff_type][
'progress_per_component'][
component_name].items())))
except KeyError as e:
# this component is not available in this network so lets
# not just plot it
if dir == exp_dir:
raise Exception("No parameter differences were "
"available even in the main "
"experiment dir for the component "
"{0}. Something went wrong: "
"{1}.".format(
component_name, str(e)))
continue
ax = plt.subplot(211)
mp, = ax.plot(iter_stats[0][:, 0], iter_stats[0][:, 1],
color=color_val,
label="Parameter Differences {0}".format(dir))
plots.append(mp)
ax.set_ylabel('Parameter Differences')
ax.grid(True)
ax = plt.subplot(212)
mp, = ax.plot(iter_stats[1][:, 0], iter_stats[1][:, 1],
color=color_val,
label="Relative Parameter "
"Differences {0}".format(dir))
ax.set_xlabel('Iteration')
ax.set_ylabel('Relative Parameter Differences')
ax.grid(True)
lgd = plt.legend(handles=plots, loc='lower center',
bbox_to_anchor=(0.5, -0.5 + len(dirs) * -0.2),
ncol=1, borderaxespad=0.)
plt.grid(True)
fig.suptitle("Parameter differences at {comp_name}".format(
comp_name=component_name))
comp_name = latex_compliant_name(component_name)
figfile_name = '{dir}/param_diff_{comp_name}.pdf'.format(
dir=output_dir, comp_name=comp_name)
fig.savefig(figfile_name, bbox_extra_artists=(lgd,),
bbox_inches='tight')
if latex_report is not None:
latex_report.add_figure(
figfile_name,
"Parameter differences at {0}".format(component_name))
