def generate_parameter_diff_plots(exp_dir, output_dir, plot, comparison_dir=None, start_iter=1, latex_report=None): # Parameter changes assert start_iter >= 1 comparison_dir = [] if comparison_dir is None else comparison_dir dirs = [exp_dir] + comparison_dir index = 0 stats_per_dir = {} key_file = { "Parameter differences": "parameter.diff", "Relative parameter differences": "relative_parameter.diff" } stats_per_dir = {} for dir in dirs: stats_per_dir[dir] = {} for key in key_file: stats_per_dir[dir][key] = ( log_parse.parse_progress_logs_for_param_diff(dir, key)) # write down the stats for the main experiment directory for diff_type in key_file: with open("{0}/{1}".format(output_dir, key_file[diff_type]), "w") as f: diff_per_component_per_iter = ( stats_per_dir[exp_dir][diff_type]['progress_per_component']) component_names = ( stats_per_dir[exp_dir][diff_type]['component_names']) max_iter = stats_per_dir[exp_dir][diff_type]['max_iter'] f.write(" ".join(["Iteration"] + component_names) + "\n") total_missing_iterations = 0 gave_user_warning = False for iter in range(max_iter + 1): iter_data = [str(iter)] for c in component_names: try: iter_data.append( str(diff_per_component_per_iter[c][iter])) except KeyError: total_missing_iterations += 1 iter_data.append("NA") if (total_missing_iterations / len(component_names) > 20 and not gave_user_warning): logger.warning( "There are more than {0} missing " "iterations per component. " "Something might be wrong.".format( total_missing_iterations / len(component_names))) gave_user_warning = True f.write(" ".join(iter_data) + "\n") if plot: # get the component names diff_type = key_file.keys()[0] main_component_names = stats_per_dir[exp_dir][diff_type][ 'progress_per_component'].keys() main_component_names.sort() plot_component_names = set(main_component_names) for dir in dirs: try: component_names = set(stats_per_dir[dir][diff_type] ['progress_per_component'].keys()) plot_component_names = plot_component_names.intersection( component_names) except KeyError: continue plot_component_names = list(plot_component_names) plot_component_names.sort() if plot_component_names != main_component_names: logger.warning("The components in all the neural networks in the " "given experiment dirs are not the same, " "so comparison plots are provided only for common " "component names. " "Make sure that these are comparable experiments " "before analyzing these plots.") assert main_component_names fig = plt.figure() logger.info("Generating parameter-difference plots for the " "following components:{0}".format( ', '.join(main_component_names))) for component_name in main_component_names: fig.clf() index = 0 plots = [] for dir in dirs: color_val = g_plot_colors[index] index += 1 iter_stats = [] try: for diff_type in [ 'Parameter differences', 'Relative parameter differences' ]: iter_stats.append( np.array( sorted(stats_per_dir[dir][diff_type] ['progress_per_component'] [component_name].items()))) except KeyError as e: # this component is not available in this network so lets # not just plot it if dir == exp_dir: raise Exception("No parameter differences were " "available even in the main " "experiment dir for the component " "{0}. Something went wrong: " "{1}.".format(component_name, str(e))) continue ax = plt.subplot(211) mp, = ax.plot(iter_stats[0][:, 0], iter_stats[0][:, 1], color=color_val, label="Parameter Differences {0}".format(dir)) plots.append(mp) ax.set_ylabel('Parameter Differences') ax.grid(True) ax = plt.subplot(212) mp, = ax.plot(iter_stats[1][:, 0], iter_stats[1][:, 1], color=color_val, label="Relative Parameter " "Differences {0}".format(dir)) ax.set_xlabel('Iteration') ax.set_ylabel('Relative Parameter Differences') ax.grid(True) lgd = plt.legend(handles=plots, loc='lower center', bbox_to_anchor=(0.5, -0.5 + len(dirs) * -0.2), ncol=1, borderaxespad=0.) plt.grid(True) fig.suptitle("Parameter differences at {comp_name}".format( comp_name=component_name)) comp_name = latex_compliant_name(component_name) figfile_name = '{dir}/param_diff_{comp_name}.pdf'.format( dir=output_dir, comp_name=comp_name) fig.savefig(figfile_name, bbox_extra_artists=(lgd, ), bbox_inches='tight') if latex_report is not None: latex_report.add_figure( figfile_name, "Parameter differences at {0}".format(component_name))
def generate_parameter_diff_plots(exp_dir, output_dir, plot, comparison_dir=None, start_iter=1, latex_report=None): # Parameter changes assert start_iter >= 1 comparison_dir = [] if comparison_dir is None else comparison_dir dirs = [exp_dir] + comparison_dir index = 0 stats_per_dir = {} key_file = {"Parameter differences": "parameter.diff", "Relative parameter differences": "relative_parameter.diff"} stats_per_dir = {} for dir in dirs: stats_per_dir[dir] = {} for key in key_file: stats_per_dir[dir][key] = ( log_parse.parse_progress_logs_for_param_diff(dir, key)) # write down the stats for the main experiment directory for diff_type in key_file: with open("{0}/{1}".format(output_dir, key_file[diff_type]), "w") as f: diff_per_component_per_iter = ( stats_per_dir[exp_dir][diff_type]['progress_per_component']) component_names = ( stats_per_dir[exp_dir][diff_type]['component_names']) max_iter = stats_per_dir[exp_dir][diff_type]['max_iter'] f.write(" ".join(["Iteration"] + component_names)+"\n") total_missing_iterations = 0 gave_user_warning = False for iter in range(max_iter + 1): iter_data = [str(iter)] for c in component_names: try: iter_data.append( str(diff_per_component_per_iter[c][iter])) except KeyError: total_missing_iterations += 1 iter_data.append("NA") if (total_missing_iterations/len(component_names) > 20 and not gave_user_warning): logger.warning("There are more than {0} missing " "iterations per component. " "Something might be wrong.".format( total_missing_iterations / len(component_names))) gave_user_warning = True f.write(" ".join(iter_data)+"\n") if plot: # get the component names diff_type = key_file.keys()[0] main_component_names = stats_per_dir[exp_dir][diff_type][ 'progress_per_component'].keys() main_component_names.sort() plot_component_names = set(main_component_names) for dir in dirs: try: component_names = set(stats_per_dir[dir][diff_type][ 'progress_per_component'].keys()) plot_component_names = plot_component_names.intersection( component_names) except KeyError: continue plot_component_names = list(plot_component_names) plot_component_names.sort() if plot_component_names != main_component_names: logger.warning("The components in all the neural networks in the " "given experiment dirs are not the same, " "so comparison plots are provided only for common " "component names. " "Make sure that these are comparable experiments " "before analyzing these plots.") assert main_component_names fig = plt.figure() logger.info("Generating parameter-difference plots for the " "following components:{0}".format( ', '.join(main_component_names))) for component_name in main_component_names: fig.clf() index = 0 plots = [] for dir in dirs: color_val = g_plot_colors[index] index += 1 iter_stats = [] try: for diff_type in ['Parameter differences', 'Relative parameter differences']: iter_stats.append(np.array( sorted(stats_per_dir[dir][diff_type][ 'progress_per_component'][ component_name].items()))) except KeyError as e: # this component is not available in this network so lets # not just plot it if dir == exp_dir: raise Exception("No parameter differences were " "available even in the main " "experiment dir for the component " "{0}. Something went wrong: " "{1}.".format( component_name, str(e))) continue ax = plt.subplot(211) mp, = ax.plot(iter_stats[0][:, 0], iter_stats[0][:, 1], color=color_val, label="Parameter Differences {0}".format(dir)) plots.append(mp) ax.set_ylabel('Parameter Differences') ax.grid(True) ax = plt.subplot(212) mp, = ax.plot(iter_stats[1][:, 0], iter_stats[1][:, 1], color=color_val, label="Relative Parameter " "Differences {0}".format(dir)) ax.set_xlabel('Iteration') ax.set_ylabel('Relative Parameter Differences') ax.grid(True) lgd = plt.legend(handles=plots, loc='lower center', bbox_to_anchor=(0.5, -0.5 + len(dirs) * -0.2), ncol=1, borderaxespad=0.) plt.grid(True) fig.suptitle("Parameter differences at {comp_name}".format( comp_name=component_name)) comp_name = latex_compliant_name(component_name) figfile_name = '{dir}/param_diff_{comp_name}.pdf'.format( dir=output_dir, comp_name=comp_name) fig.savefig(figfile_name, bbox_extra_artists=(lgd,), bbox_inches='tight') if latex_report is not None: latex_report.add_figure( figfile_name, "Parameter differences at {0}".format(component_name))