def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): document.add(Heading(self.template_params.experiment_name)) for title in measurement_manager.single_run_measurements[ 0].get_item_names(): if title not in [ "initial training error", "initial testing error", "untraining error", "retraining error", "testing error" ]: continue data = measurement_manager.get_values_from_all_runs(title) n_runs = len(data) if self.template_params.every_second_is_baseline: # TODO MS labels = (n_runs // 2) * ["experiment", "baseline"] else: labels = n_runs * [""] plot_multiple_runs(list(range(len(data[0]))), data, ylabel="error", xlabel="steps", labels=labels, title=title, smoothing_window_size=21, path=path.join(docs_folder, title), doc=document)
def _prepare_document(self) -> (Document, str, os.path): """Prepares the document for writing the results""" doc = Document() date = get_stamp() doc.add(self._get_heading(date)) return doc, date
def add_fig_to_doc(fig, path_name, doc: Document): name = to_safe_path(path_name) # save figure also as pickle if not os.path.exists(os.path.dirname(name)): os.makedirs(os.path.dirname(name)) with open(name + '.fpk', 'wb+') as file: pickle.dump(fig, file, protocol=2) name_svg = name + '.svg' fig.savefig(name_svg, format='svg') im = Image(os.path.basename(name_svg)) doc.add(im)
def _publish_results(self): """Plot and save the results.""" doc = Document() date = get_stamp() labels = ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list) title = 'Mutual Information labels vs ' + self._experiment_name self.plot_save(title, self._mutual_info, self._baseline_mutual_info, 'Norm. mutual information', labels, date, self._docs_folder, doc) title = 'Weak classifier accuracy labels vs ' + self._experiment_name self.plot_save(title, self._classifier_accuracy, self._baseline_classifier_accuracy, 'Classifier accuracy', labels, date, self._docs_folder, doc) #, smoothing_size=3) title = 'average delta' f = plot_multiple_runs( self._different_steps[0], # here the X axes are identical self._average_delta, title=title, ylabel='log(delta)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'average boosting duration' f = plot_multiple_runs(self._different_steps[0], self._average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) doc.write_file( path.join(self._docs_folder, to_safe_name(self._complete_name() + date + ".html"))) print('done')
def __init__(self, template: ExperimentTemplateBase, docs_folder: str, template_class: Type, default_topology_parameters: Dict[str, Any], topology_params: List[Dict[str, Any]], runner_params: ExperimentParams): """Prepares the document for writing the results""" self._document = Document() self._template = template self._docs_folder = docs_folder self._template_class = template_class self._default_topology_parameters = default_topology_parameters self._topology_params = topology_params self._runner_params = runner_params parameter_extractor = ParameterExtractor( self._default_topology_parameters) self._common_parameters, self._differing_parameters = parameter_extractor.extract( self._topology_params) if not self._docs_folder: raise RuntimeError("Docs folder not defined")
def test_doc_creation(): document = Document().\ add(Heading("Look at his heading")). \ add("This is a very nice document.<br><br>").\ add(Figure.from_params('test_image.png', "Behold our powerful visualization", height=200, width=400)).\ add(Matrix(torch.rand((2, 3)), m_labels=['a', 'b'], n_labels=['10', '20', '30'], format_func=lambda x: "%.5f" % x)) # The following line will write to a non-temporary file. # Uncomment if you want to see what the html file looks like. # document.write_file("test_file.html") with tempfile.TemporaryFile() as document_file: doc_text = document.as_text() document_file.write(doc_text.encode('UTF-8')) document_file.seek(0) read_text = document_file.readline() assert read_text.startswith(b"<!DOCTYPE html>")
def _publish_results(self): """Plot and optionally save the results.""" doc = Document() date = get_stamp() labels = ExperimentTemplateBase.parameters_to_string(self._topology_parameters_list) title = f'FPS vs. {self._experiment_name} (smoothing {self._smoothing_window_size})' self.plot_save(title, self._fps, 'FPS', labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size) smoothing = len(self._fps[0]) // 100 # adaptive smoothing if smoothing % 2 == 0: smoothing += 1 # make odd title = 'FPS vs. ' + self._experiment_name + f' (smoothing {smoothing})' self.plot_save(title, self._fps, 'FPS', labels, date, self._docs_folder, doc, smoothing_window_size=smoothing) title = f'max_memory_allocated() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})' self.plot_save(title, self._max_mem, 'Max memory', labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size) title = f'memory_allocated() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})' self.plot_save(title, self._current_mem, 'Current memory', labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size) title = f'max_cached() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})' self.plot_save(title, self._max_cached, 'Max cached mem', labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size) title = f'memory_cached() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})' self.plot_save(title, self._current_cached, 'Current cached mem', labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size) self._publish_aux_results(labels, date, self._docs_folder, doc) doc.write_file(path.join(self._docs_folder, f"{self._topology_class.__name__}_" + date + ".html")) print('done')
def _print_statistics(self, doc: Document): doc.add(Heading("Overall statistics")) for run in range(len(self._tasks_solved)): doc.add(Heading(f" Run {run}:", 2)) experiment_statistics = self._tasks_solved[run] tasks_solved_statuses = [[stats.task_id, stats.task_solved] for stats in experiment_statistics] doc.add(f"Tasks solved: {tasks_solved_statuses}.<br />") for task_stats in experiment_statistics: self._print_task_stats(doc, task_stats)
def _print_task_stats(self, doc: Document, task_stats: TaskExperimentStatistics): doc.add(Heading(f"Task {task_stats.task_id}<br />", 3)) doc.add(f"Solved: {task_stats.task_solved}<br />") if task_stats.instances_seen_training != 0: ratio = task_stats.instances_solved_training / task_stats.instances_seen_training else: ratio = 0 doc.add( f"Instances solved / seen during training: {task_stats.instances_solved_training} / " f"{task_stats.instances_seen_training} = {ratio*100:.1f} %<br />") if task_stats.instances_seen_testing != 0: ratio = task_stats.instances_solved_testing / task_stats.instances_seen_testing else: ratio = 0 doc.add( f"Instances solved / seen during testing: {task_stats.instances_solved_testing} / " f"{task_stats.instances_seen_testing} = {ratio*100:.1f} %<br />")
def publish_results_for_layer(document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str], layer_id: int, num_layers: int, show_conv_agreements: bool, is_train_test_classifier_computed: bool): """Publish results for each layer separately This uses the data measured and computed for each run by the Task0TaAnalysisLayerComponent and aggregated and stored in the measurement_manager. """ logger.info(f'publishing results for layer {layer_id}...') num_boosted_clusters = measurement_manager.get_values_from_all_runs( f'num_boosted_getter_{layer_id}') average_boosting_dur = measurement_manager.get_values_from_all_runs( f'average_boosting_dur_{layer_id}') average_deltas = measurement_manager.get_values_from_all_runs( f'average_delta_{layer_id}') base_weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs( f'base_weak_class_accuracy_{layer_id}') clustering_agreements = measurement_manager.get_custom_data_from_all_runs( f'clustering_agreements_{layer_id}') average_steps_deltas = measurement_manager.get_items_from_all_runs( f'average_delta_{layer_id}') sp_evaluation_steps = [key for key, value in average_steps_deltas[0]] labels = topology_parameters document.add(f"<br><br><br><b>Results for layer {layer_id}</b><br>") prefix = 'L' + str(layer_id) + '--' testing_phase_ids = list(range(0, len( clustering_agreements[0][0]))) # x-axis values if is_train_test_classifier_computed: weak_class_accuracy_train = measurement_manager.get_custom_data_from_all_runs( f'weak_class_accuracy_train_{layer_id}') weak_class_accuracy_test = measurement_manager.get_custom_data_from_all_runs( f'weak_class_accuracy_test_{layer_id}') title = prefix + ' Weak classifier accuracy (trained on train, tested on train data)' plot_multiple_runs_with_baselines(testing_phase_ids, weak_class_accuracy_train, base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join( docs_folder, title), doc=document) title = prefix + ' Weak classifier accuracy (trained on train, tested on test data)' plot_multiple_runs_with_baselines(testing_phase_ids, weak_class_accuracy_test, base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join( docs_folder, title), doc=document) weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs( f'weak_class_accuracy_{layer_id}') title = prefix + ' Weak classifier accuracy (trained on test, tested on test)' plot_multiple_runs_with_baselines(testing_phase_ids, weak_class_accuracy, base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join(docs_folder, title), doc=document) title = prefix + '- Average boosting duration' plot_multiple_runs(sp_evaluation_steps, average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels, hide_legend=True, path=path.join(docs_folder, title), doc=document) title = prefix + '- Num boosted clusters' plot_multiple_runs(sp_evaluation_steps, num_boosted_clusters, title=title, ylabel='Num boosted clusters / total clusters', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join(docs_folder, title), doc=document) title = prefix + '- Average_deltas' plot_multiple_runs( sp_evaluation_steps, average_deltas, title=title, ylabel='average_deltas', xlabel='steps', labels=labels, disable_ascii_labels=True, hide_legend=True, # use_scatter=True, path=path.join(docs_folder, title), doc=document) # if this is not the top layer, show conv agreements only if required if show_conv_agreements or layer_id == (num_layers - 1): agreements = clustering_agreements for run_id, run_agreements in enumerate(agreements): for expert_id, expert_agreements in enumerate(run_agreements): Task0TaAnalysisTemplate._plot_agreement( prefix, expert_id, run_id, testing_phase_ids, expert_agreements, document, docs_folder) logger.info('done')
def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): """An alternative to the _publish_results method, this is called from _publish_results now Draw and add your plots to the document here. """ steps = measurement_manager.get_values_from_all_runs('current_step') plotted_training_phase_id = measurement_manager.get_values_from_all_runs( 'training_phase_id') plotted_testing_phase_id = measurement_manager.get_values_from_all_runs( 'testing_phase_id') plotted_is_learning = measurement_manager.get_values_from_all_runs( 'is_learning') labels = topology_parameters document.add( f"<br><br><br><b>Common results from the TopExpert</b><br>") title = 'training_phase_id' plot_multiple_runs(steps, plotted_training_phase_id, title=title, ylabel='training_phase_id', xlabel='steps', labels=labels, path=path.join(docs_folder, title), doc=document) title = 'testing_phase_id' plot_multiple_runs(steps, plotted_testing_phase_id, title=title, ylabel='testing_phase_id', xlabel='steps', labels=labels, path=path.join(docs_folder, title), doc=document) title = 'is_learning' plot_multiple_runs(steps, plotted_is_learning, title=title, ylabel='is_learning', xlabel='steps', labels=labels, path=path.join(docs_folder, title), doc=document) predicted_labels_mse = measurement_manager.get_custom_data_from_all_runs( 'predicted_labels_mse') testing_phases_x = list(range(0, len(predicted_labels_mse[0]))) model_accuracy = measurement_manager.get_custom_data_from_all_runs( 'model_accuracy') baseline_accuracy = measurement_manager.get_custom_data_from_all_runs( 'baseline_accuracy') # plot the classification accuracy title = 'Label reconstruction accuracy (step-wise)' plot_multiple_runs_with_baselines(testing_phases_x, model_accuracy, baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True, path=path.join(docs_folder, title), doc=document) model_se_accuracy = measurement_manager.get_custom_data_from_all_runs( 'model_se_accuracy') baseline_se_accuracy = measurement_manager.get_custom_data_from_all_runs( 'baseline_se_accuracy') # plot the classification accuracy title = 'Label reconstruction SE accuracy (object-wise)' plot_multiple_runs_with_baselines(testing_phases_x, model_se_accuracy, baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True, path=path.join(docs_folder, title), doc=document) baseline_labels_mse = measurement_manager.get_custom_data_from_all_runs( 'baseline_labels_mse') # plot the MSE title = 'Mean Square Error of label reconstruction' plot_multiple_runs_with_baselines( testing_phases_x, predicted_labels_mse, baseline_labels_mse, title=title, ylabel='MSE', xlabel='testing phase ID', labels=labels, hide_legend=True, ylim=[-0.1, 0.2], # just for better resolution path=path.join(docs_folder, title), doc=document) for layer_id in reversed(range(self._experiment_params.num_layers)): self.publish_results_for_layer( document, docs_folder, measurement_manager, topology_parameters, layer_id, self._experiment_params.num_layers, self._experiment_params.show_conv_agreements, self._experiment_params.is_train_test_classifier_computed)
class DocumentPublisher: """A class responsible for the publishing of results based on a template.""" _header_table_width = 60 def __init__(self, template: ExperimentTemplateBase, docs_folder: str, template_class: Type, default_topology_parameters: Dict[str, Any], topology_params: List[Dict[str, Any]], runner_params: ExperimentParams): """Prepares the document for writing the results""" self._document = Document() self._template = template self._docs_folder = docs_folder self._template_class = template_class self._default_topology_parameters = default_topology_parameters self._topology_params = topology_params self._runner_params = runner_params parameter_extractor = ParameterExtractor( self._default_topology_parameters) self._common_parameters, self._differing_parameters = parameter_extractor.extract( self._topology_params) if not self._docs_folder: raise RuntimeError("Docs folder not defined") def publish_results(self, timestamp: str, measurement_manager: MeasurementManager): """Publishes the result and returns the path of the main html document.""" self._document.add(self._get_heading(timestamp)) self._template.publish_results( self._document, self._docs_folder, measurement_manager, self.parameters_to_string(self._differing_parameters)) logger.info(f"Results documents generated in {self._docs_folder}") return self._write_document(self._docs_folder) def _write_document(self, docs_folder: str) -> str: """Writes the document to the disk.""" if not os.path.isdir(docs_folder): os.makedirs(docs_folder) doc_path = os.path.join( docs_folder, to_safe_name(self._template.experiment_name + ".html")) self._document.write_file(doc_path) logger.info('done') return doc_path @classmethod def _param_to_html(cls, name: str, value: Any, indent: bool = False, bold: bool = False) -> str: if hasattr(value, '_asdict'): value = value._asdict() if isinstance(value, dict): row = cls._param_to_html(name, '', bold=True) + ''.join([ cls._param_to_html(key, val, indent=True) for key, val in value.items() ]) else: style = ' style="padding-left: 20px"' if indent else '' name = f'<strong>{name}</strong>' if bold else name row = "<tr>" + \ f"<td{style}>{name}</td>" + \ f"<td>{cls._param_value_to_string(value)}</td>" + \ "</tr>" return row def _runner_configuration_to_html(self) -> str: """Convert the runner's configuration to the string for the html heading.""" result = f"\n<b>Experiment and runner configuration</b>:<br> " result += self._get_table_header() # values in columns result += self._param_to_html("runner", self._runner_params) for key, value in self._template.get_additional_parameters().items(): result += self._param_to_html(key, value) result += "</table>" return result def _get_table_header(self): header = f"<table style=\"width:{self._header_table_width}%\">" + \ "<tr>" + \ "<th style=\"text-align: left\">Param name</th>" + \ "<th style=\"text-align:left\">Param value</th>" + \ "</tr>" return header @classmethod def parameters_to_string(cls, parameters: List[Dict[str, Any]]) -> List[str]: return [ ", ".join(f"{param}: {cls._param_value_to_string(value)}" for param, value in parameter.items()) for parameter in parameters ] @classmethod def _param_value_to_string(cls, value): """Lists of normal values are parsed OK, param value can be also list of classes, parse to readable string.""" if type(value) in (list, tuple): return [cls._param_value_to_string(x) for x in value] elif isinstance(value, type): return value.__name__ return value def _get_heading(self, date: str): """Get heading of the html file with the experiment description""" info = f"<p><b>Template</b>: {self._template_class.__name__}<br>" + \ f"\n<b>Experiment_name</b>: {self._template.experiment_name}<br>" + \ f"\n<b>Date:</b> {date[1:]}</p>" info += '<table style="width: 100%">' info += '<tr>' info += '<td style="width: 50%">' info += f"\n<b>List of common parameters</b>:<br> " # create table with the params info += self._get_table_header() for key, value in self._common_parameters.items(): info += self._param_to_html(key, value) info += "</table>" info += '</td>' info += '<td>' # add the description of the template configuration info += self._runner_configuration_to_html() info += '</td>' info += '</tr>' info += '</table>' return info
def _publish_results(self): """Plot and save the results.""" doc = Document() date = get_stamp() labels = ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list) for manager in self._layer_measurement_managers: manager.publish_results(labels=labels, date=date, docs_folder=self._docs_folder, doc=doc) # plot the running MSE title = 'Mean Square Error of TA classification' f = plot_multiple_runs_with_baselines(self._steps, self._predicted_labels_mse, self._baseline_labels_mse, title=title, ylabel='mse', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'TA classification accuracy' f = plot_multiple_runs_with_baselines( self._steps, self._classification_accuracy, self._random_classification_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'TA classification accuracy - (SE-metric)' f = plot_multiple_runs_with_baselines( self._steps, self._se_classification_accuracy, self._se_random_classification_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'is_learning' f = plot_multiple_runs( self._steps, self._is_learning_info, title=title, ylabel='is learning', xlabel='steps', ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) doc.write_file( path.join(self._docs_folder, to_safe_name(self._complete_name() + date + ".html"))) print('done')
def _publish_results(self): doc = Document() doc.add(self._get_heading('_not set')) params_str = pprint.pformat( Task0TrainTestTemplateRelearning.pprint_parameters_to_string(self._topology_parameters_list), 1, 160 ) doc.add(f"<p><b>Parameters:</b><br /><pre>learning rate: {self._learning_rate},\n" + params_str + "</pre></p>") labels = ExperimentTemplateBase.parameters_to_string(self._topology_parameters_list) testing_phases = list(range(0, len(self._clustering_agreement_list[0][0]))) for i, run_label in enumerate(self._run_labels): title = 'Clustering agreement ' + run_label f = plot_multiple_runs( testing_phases, self._clustering_agreement_list[i], title=title, ylabel='agreement', xlabel='testing training_phases', disable_ascii_labels=True, hide_legend=True, ylim=[ClusterAgreementMeasurement.NO_VALUE - 0.1, 1.1] ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Model classification accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phases, self._model_accuracy, self._baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase', ylim=[-0.1, 1.1], labels=self._run_labels ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Model SE classification accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phases, self._model_se_accuracy, self._baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase', ylim=[-0.1, 1.1], labels=self._run_labels ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Weak classifier accuracy trained on SP outputs to labels' f = plot_multiple_runs_with_baselines( testing_phases, self._weak_class_accuracy, self._base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Average Deltas Train' f = plot_multiple_runs( [self._sp_evaluation_period * i for i in range(0, len(self._average_delta_train[0]))], self._average_delta_train, title=title, ylabel='average_deltas', xlabel='steps', labels=self._run_labels, disable_ascii_labels=True, use_scatter=False ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Average Deltas Test' f = plot_multiple_runs( [self._sp_evaluation_period * i for i in range(0, len(self._average_delta_test[0]))], self._average_delta_test, title=title, ylabel='average_deltas', xlabel='steps', labels=self._run_labels, disable_ascii_labels=True, use_scatter=False ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Average boosting duration' f = plot_multiple_runs( [self._sp_evaluation_period * i for i in range(0, len(self._average_boosting_dur[0]))], self._average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) doc.write_file(path.join(self._docs_folder, "main.html")) print('done')
def _publish_results(self): doc = Document() xlabel = "steps" ylabel = "number of SP forward executions" title_fe_dt = "smoothed derivation of SP forward executions and TP forward executions (SP O QD)" figsize = (18, 12) date = get_stamp() layer_names = ['L0', 'L1', 'L2'] nr_layers = len(layer_names) labels = list(map(lambda x: x + " forward execution", layer_names)) + \ list(map(lambda x: x + " qualitative difference", layer_names)) colors = ['b', 'orange', 'g', 'r', 'p'] color_params = [{'c': color} for color in colors[:nr_layers]] color_ls_params = [{ 'c': color, 'ls': '--' } for color in colors[:nr_layers]] other_params = color_params + color_ls_params params_description = ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list) for run in range(len(self.sp_executions)): sp_execution_dt = list( map(compute_derivations, self.sp_executions[run])) sp_output_dt = self.sp_output_stability[run] title = title_fe_dt + f" run {run} " fig = plot_multiple_runs(x_values=self.training_steps, y_values=sp_execution_dt + sp_output_dt, ylim=[0, 1], labels=labels, smoothing_window_size=501, xlabel=xlabel, ylabel=ylabel, title=title + params_description[run], figsize=figsize, hide_legend=False, other_params=other_params) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "classification accuracy from reconstructed labels" fig = plot_multiple_runs(x_values=list(range( self._num_testing_phases)), y_values=self.classification_accuracy, ylim=[0, 1], labels=params_description, smoothing_window_size=None, xlabel="accuracy", ylabel="phases", title=title, figsize=figsize, hide_legend=False) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "SE classification accuracy from reconstructed labels" fig = plot_multiple_runs(x_values=list(range( self._num_testing_phases)), y_values=self.classification_accuracy_se, ylim=[0, 1], labels=params_description, smoothing_window_size=None, xlabel="SE accuracy", ylabel="phases", title=title, figsize=figsize, hide_legend=False) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) doc.write_file(path.join(self._docs_folder, f"main.html"))
def _write_document(self, doc: Document, date: str, docs_folder: os.path): """ Writes the document to the hdd after adding the graph (see the _publish_results()).""" doc.write_file( path.join(docs_folder, to_safe_name(self._complete_name() + date + ".html"))) print('done')
def _publish_results(self): """Plot and optionally save the results.""" mse = np.array(self._mse) mse_testing = np.array(self._mse_testing) memory_used = torch.tensor(self._memory_used, dtype=torch.float32) window_size = 201 memory_used = (memory_used.view(-1, 1).expand(mse_testing.shape) / (1024**2)) error_memory_ratio = torch.tensor(mse_testing, dtype=torch.float32) * memory_used accuracy_memory_ratio = [] for run_acc, run_mem in zip(self._weak_classifier_results, memory_used): accuracy_memory_ratio_run = [] for acc, mem in zip(run_acc, run_mem): accuracy_memory_ratio_run.append((1 - acc) * mem) accuracy_memory_ratio.append(accuracy_memory_ratio_run) accuracy_memory_ratio = torch.tensor(accuracy_memory_ratio) doc = Document() figs = [] xlabel = "steps" ylabel = "mean reconstruction error" title = "Influence of hyperparameters on reconstruction error (training)" figsize = (18, 12) date = get_stamp() fig = plot_multiple_runs( x_values=np.arange(len(mse[0])), y_values=mse, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), smoothing_window_size=window_size, xlabel=xlabel, ylabel=ylabel, title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) fig = plot_multiple_runs( x_values=np.arange(len(mse[0])), y_values=mse, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel=ylabel, title=title, figsize=figsize, hide_legend=True) add_fig_to_doc( fig, path.join(self._docs_folder, to_safe_name(title) + "_smooth"), doc) title = "Influence of hyperparameters on reconstruction error (testing)" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=mse_testing, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel=ylabel, title=title, figsize=figsize, ) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Weak classifier accuracy" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=np.array(self._weak_classifier_results), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="steps", ylabel="accuracy", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Memory * reconstruction tradeoff (testing)" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=error_memory_ratio.numpy(), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel= "Mean Reconstruction Error times required meogabyte of memory", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Memory * error (= 1 - acc) tradeoff" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=accuracy_memory_ratio.numpy(), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel= "Mean Reconstruction Error times required meogabyte of memory", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Entropy and mean reconstruction error" fig = plot_multiple_runs( x_values=np.array(self._code_entropy), y_values=mse_testing, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="entropy", ylabel="mean reconstruction error", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Mean reconstruction error and classifier accuracy" fig = plot_multiple_runs( x_values=mse_testing, y_values=np.array(self._weak_classifier_results), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="mean reconstruction error", ylabel="accuracy", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Entropy and classifier accuracy" fig = plot_multiple_runs( x_values=np.array(self._code_entropy), y_values=np.array(self._weak_classifier_results), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="entropy", ylabel="accuracy", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) doc.write_file( path.join(self._docs_folder, f"{self._topology_class.__name__}_" + date + ".html")) print(self._memory_used)
def _publish_results_to_doc(self, doc: Document, date: str, docs_folder: path): """Adds my results to the results produced by the base class""" super()._publish_results_to_doc(doc, date, docs_folder) doc.add(f"<br><br><br><b>Common results</b><br>") labels = ExperimentTemplateBase.extract_params_for_legend(self._topology_parameters_list) title = 'training_phase_id' f = plot_multiple_runs( self._steps, self._plotted_training_phase_id, title=title, ylabel='training_phase_id', xlabel='steps', labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = 'testing_phase_id' f = plot_multiple_runs( self._steps, self._plotted_testing_phase_id, title=title, ylabel='testing_phase_id', xlabel='steps', labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) if self._plot_is_learning: # plot the classification accuracy title = 'Is Learning' f = plot_multiple_runs( self._steps, self._is_learning_info, title=title, ylabel='learning=True?', xlabel='steps', ylim=[-0.1, 1.1], labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) testing_phase_ids = list(range(0, len(self._mutual_info[0]))) # x-axis values title = 'Top-layer L' + str(self._top_layer_id()) + ' Mutual information of SP output with labels' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._mutual_info, self._base_mutual_info, title=title, ylabel='Normalized mutual information', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the classification accuracy title = 'Label reconstruction accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._model_accuracy, self._baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the classification accuracy title = 'Label reconstruction SE accuracy' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._model_se_accuracy, self._baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the MSE title = 'Mean Square Error of label reconstruction' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._predicted_labels_mse, self._baseline_labels_mse, title=title, ylabel='MSE', xlabel='testing phase ID', labels=labels, hide_legend=True, ylim=[-0.1, 0.2] # just for better resolution ) add_fig_to_doc(f, path.join(docs_folder, title), doc) for manager in reversed(self._layer_measurement_managers): manager.publish_results(labels=title, date=date, docs_folder=docs_folder, doc=doc)