def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): document.add(Heading(self.template_params.experiment_name)) for title in measurement_manager.single_run_measurements[ 0].get_item_names(): if title not in [ "initial training error", "initial testing error", "untraining error", "retraining error", "testing error" ]: continue data = measurement_manager.get_values_from_all_runs(title) n_runs = len(data) if self.template_params.every_second_is_baseline: # TODO MS labels = (n_runs // 2) * ["experiment", "baseline"] else: labels = n_runs * [""] plot_multiple_runs(list(range(len(data[0]))), data, ylabel="error", xlabel="steps", labels=labels, title=title, smoothing_window_size=21, path=path.join(docs_folder, title), doc=document)
def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): steps = measurement_manager.get_values_from_all_runs('current_step') labels = topology_parameters multi_config_rewards = measurement_manager.get_custom_data_from_all_runs( 'average_rewards') for single_config_multi_run_rewards in multi_config_rewards: num_runs = len(single_config_multi_run_rewards[0]) single_runs = [[] for _ in range(num_runs)] for timestep in single_config_multi_run_rewards: for i in range(num_runs): single_runs[i].append(timestep[i]) max_value = max(max(run) for run in single_runs) for i, timeseries in enumerate(single_runs): title = f"average_reward_run_{i}" plot_multiple_runs(steps, timeseries, ylabel=f"average_reward_run_{i}", xlabel="steps", ylim=[0, max_value], labels=labels, title=title, path=path.join(docs_folder, title), doc=document) means = [] mins = [] maxes = [] for timestep in single_config_multi_run_rewards: means.append(np.mean(timestep)) mins.append(np.min(timestep)) maxes.append(np.max(timestep)) title = "all_average_rewards" plot_multiple_runs(steps, means, y_lower=mins, y_upper=maxes, ylabel="average_reward", xlabel="steps", labels=labels, title=title, path=path.join(docs_folder, title), doc=document)
def different_plots(x_values, y_values, baseline, legend): """Tests potentially problematic plot setups.""" plot_multiple_runs(x_values, y_values, labels=legend) plot_multiple_runs(x_values, y_values, labels=legend, smoothing_window_size=3) plot_multiple_runs_with_baselines(x_values, y_values, baseline, labels=legend) plot_multiple_runs_with_baselines(x_values, y_values, baseline, labels=legend, smoothing_window_size=3)
def _plot_agreement(prefix: str, expert_id: int, run_id: int, testing_phase_ids: List[int], agreements: List[List[int]], doc, docs_folder): """Plot cluster agreements for one expert in a layer""" title = prefix + f'- E{expert_id}-Run{run_id} - Clustering agreements' plot_multiple_runs( testing_phase_ids, agreements, title=title, ylabel='agreement', xlabel=f'testing phases', disable_ascii_labels=True, hide_legend=True, ylim=[ClusterAgreementMeasurement.NO_VALUE - 0.1, 1.1], path=path.join(docs_folder, title), doc=doc)
def _publish_results(self): """Plot and save the results.""" doc = Document() date = get_stamp() labels = ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list) title = 'Mutual Information labels vs ' + self._experiment_name self.plot_save(title, self._mutual_info, self._baseline_mutual_info, 'Norm. mutual information', labels, date, self._docs_folder, doc) title = 'Weak classifier accuracy labels vs ' + self._experiment_name self.plot_save(title, self._classifier_accuracy, self._baseline_classifier_accuracy, 'Classifier accuracy', labels, date, self._docs_folder, doc) #, smoothing_size=3) title = 'average delta' f = plot_multiple_runs( self._different_steps[0], # here the X axes are identical self._average_delta, title=title, ylabel='log(delta)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'average boosting duration' f = plot_multiple_runs(self._different_steps[0], self._average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) doc.write_file( path.join(self._docs_folder, to_safe_name(self._complete_name() + date + ".html"))) print('done')
def get_data(self) -> plt: plt.close('all') data = self._matrix.cpu().numpy() n_steps = data.shape[1] x_vlaues = np.arange(0, n_steps) figure = plot_multiple_runs(x_vlaues, data) return figure
def publish_one_run(self, single_run_measurements: SingleRunMeasurements, document: Document, docs_folder: str, topology_parameters: List[str]): steps = single_run_measurements.get_items('current_step') accuracy_1 = single_run_measurements.get_items(ACCURACY_1) accuracy_per_flock_1 = single_run_measurements.get_items( ACCURACY_PER_FLOCK_1) accuracy_2 = single_run_measurements.get_items(ACCURACY_2) accuracy_per_flock_2 = single_run_measurements.get_items( ACCURACY_PER_FLOCK_2) labels = topology_parameters multiple_flocks_alpha = 0.05 title = 'accuracy_per_flock_1' f = plot_multiple_runs([steps], [accuracy_per_flock_1], title=title, ylabel='accuracy_per_flock_1', xlabel='steps', labels=labels, hide_legend=True, other_params=[{ 'alpha': multiple_flocks_alpha }]) add_fig_to_doc(f, os.path.join(docs_folder, title), document) title = 'accuracy_per_flock_2' f = plot_multiple_runs([steps], [accuracy_per_flock_2], title=title, ylabel='accuracy_per_flock_2', xlabel='steps', labels=labels, hide_legend=True, other_params=[{ 'alpha': multiple_flocks_alpha }]) add_fig_to_doc(f, os.path.join(docs_folder, title), document) a1 = np.array(accuracy_1) a2 = np.array(accuracy_2)
def plot_save(self, title, series, ylabel, labels, date, docs_folder, doc, smoothing_window_size=None): f = plot_multiple_runs( self._steps, series, title=title, xlabel='Simulation step', ylabel=ylabel, labels=labels, smoothing_window_size=smoothing_window_size) add_fig_to_doc(f, path.join(docs_folder, to_safe_name(title)), doc) plt.close(f)
def get_data(self) -> plt: plt.close('all') self._buffer = np.append(self._buffer, self._matrix.cpu().numpy(), axis=1) n_steps = self._buffer.shape[1] x_vlaues = np.arange(0, n_steps) figure = plot_multiple_runs(x_vlaues, self._buffer) return figure
def publish_results(self, labels, date, docs_folder, doc): prefix = 'L' + str(self._layer_id) + '--' title = prefix + self._available( ) + 'Mutual Information labels vs ' + self._experiment_name Task0OnlineLearningTemplate.plot_save(title, self._steps, self._mutual_info, self._baseline_mutual_info, 'Norm. mutual information', labels, date, docs_folder, doc) title = prefix + self._available( ) + 'Weak classifier accuracy labels vs ' + self._experiment_name Task0OnlineLearningTemplate.plot_save( title, self._steps, self._classifier_accuracy, self._baseline_classifier_accuracy, 'Classifier accuracy', labels, date, docs_folder, doc) # , smoothing_size=3) # TODO why this is different than the two above? title = prefix + 'average delta' f = plot_multiple_runs(self._different_steps, self._average_delta, title=title, ylabel='log(delta)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = prefix + 'average boosting duration' f = plot_multiple_runs(self._different_steps, self._average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(docs_folder, title), doc)
def plot_testing_accuracy(steps: np.ndarray, d1: np.ndarray, d2: np.ndarray, window_size: int, name: str, trim_size: int, labels: List[str], document: Document, docs_folder: str): a1 = compute_moving_average(d1, window_size) a2 = compute_moving_average(d2, window_size) title = f'{name}, window size: {window_size}' f = plot_multiple_runs( np.expand_dims(steps[trim_size:-trim_size - 1], axis=0), np.stack((np.expand_dims(a1[trim_size:-trim_size - 1], axis=0), np.expand_dims(a2[trim_size:-trim_size - 1], axis=0)), axis=-1), title=title, ylabel='accuracy', xlabel='steps', labels=labels, other_params=[{ 'color': None }]) add_fig_to_doc(f, os.path.join(docs_folder, title), document)
def _plot_agreement(self, prefix: str, expert_id: int, run_id: int, testing_phase_ids: List[int], agreements: List[List[int]], docs_folder, doc): """Plot cluster agreements for one expert in a layer""" run_params = ExperimentTemplateBase.parameters_to_string([self._topology_parameters_list[run_id]]) title = prefix + f'- E{expert_id}-Run{run_id} - Clustering agreements' f = plot_multiple_runs( testing_phase_ids, agreements, title=title, ylabel='agreement', # xlabel=f'params: {run_params}', xlabel=f'testing phases', disable_ascii_labels=True, hide_legend=True, ylim=[ClusterAgreementMeasurement.NO_VALUE - 0.1, 1.1] ) add_fig_to_doc(f, path.join(docs_folder, title), doc)
def _publish_results(self): """Plot and save the results.""" doc = Document() date = get_stamp() labels = ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list) for manager in self._layer_measurement_managers: manager.publish_results(labels=labels, date=date, docs_folder=self._docs_folder, doc=doc) # plot the running MSE title = 'Mean Square Error of TA classification' f = plot_multiple_runs_with_baselines(self._steps, self._predicted_labels_mse, self._baseline_labels_mse, title=title, ylabel='mse', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'TA classification accuracy' f = plot_multiple_runs_with_baselines( self._steps, self._classification_accuracy, self._random_classification_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'TA classification accuracy - (SE-metric)' f = plot_multiple_runs_with_baselines( self._steps, self._se_classification_accuracy, self._se_random_classification_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'is_learning' f = plot_multiple_runs( self._steps, self._is_learning_info, title=title, ylabel='is learning', xlabel='steps', ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) doc.write_file( path.join(self._docs_folder, to_safe_name(self._complete_name() + date + ".html"))) print('done')
def _publish_results(self): doc = Document() doc.add(self._get_heading('_not set')) params_str = pprint.pformat( Task0TrainTestTemplateRelearning.pprint_parameters_to_string(self._topology_parameters_list), 1, 160 ) doc.add(f"<p><b>Parameters:</b><br /><pre>learning rate: {self._learning_rate},\n" + params_str + "</pre></p>") labels = ExperimentTemplateBase.parameters_to_string(self._topology_parameters_list) testing_phases = list(range(0, len(self._clustering_agreement_list[0][0]))) for i, run_label in enumerate(self._run_labels): title = 'Clustering agreement ' + run_label f = plot_multiple_runs( testing_phases, self._clustering_agreement_list[i], title=title, ylabel='agreement', xlabel='testing training_phases', disable_ascii_labels=True, hide_legend=True, ylim=[ClusterAgreementMeasurement.NO_VALUE - 0.1, 1.1] ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Model classification accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phases, self._model_accuracy, self._baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase', ylim=[-0.1, 1.1], labels=self._run_labels ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Model SE classification accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phases, self._model_se_accuracy, self._baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase', ylim=[-0.1, 1.1], labels=self._run_labels ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Weak classifier accuracy trained on SP outputs to labels' f = plot_multiple_runs_with_baselines( testing_phases, self._weak_class_accuracy, self._base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Average Deltas Train' f = plot_multiple_runs( [self._sp_evaluation_period * i for i in range(0, len(self._average_delta_train[0]))], self._average_delta_train, title=title, ylabel='average_deltas', xlabel='steps', labels=self._run_labels, disable_ascii_labels=True, use_scatter=False ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Average Deltas Test' f = plot_multiple_runs( [self._sp_evaluation_period * i for i in range(0, len(self._average_delta_test[0]))], self._average_delta_test, title=title, ylabel='average_deltas', xlabel='steps', labels=self._run_labels, disable_ascii_labels=True, use_scatter=False ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) title = 'Average boosting duration' f = plot_multiple_runs( [self._sp_evaluation_period * i for i in range(0, len(self._average_boosting_dur[0]))], self._average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(self._docs_folder, title), doc) doc.write_file(path.join(self._docs_folder, "main.html")) print('done')
def publish_results_for_layer(document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str], layer_id: int, num_layers: int, show_conv_agreements: bool, is_train_test_classifier_computed: bool): """Publish results for each layer separately This uses the data measured and computed for each run by the Task0TaAnalysisLayerComponent and aggregated and stored in the measurement_manager. """ logger.info(f'publishing results for layer {layer_id}...') num_boosted_clusters = measurement_manager.get_values_from_all_runs( f'num_boosted_getter_{layer_id}') average_boosting_dur = measurement_manager.get_values_from_all_runs( f'average_boosting_dur_{layer_id}') average_deltas = measurement_manager.get_values_from_all_runs( f'average_delta_{layer_id}') base_weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs( f'base_weak_class_accuracy_{layer_id}') clustering_agreements = measurement_manager.get_custom_data_from_all_runs( f'clustering_agreements_{layer_id}') average_steps_deltas = measurement_manager.get_items_from_all_runs( f'average_delta_{layer_id}') sp_evaluation_steps = [key for key, value in average_steps_deltas[0]] labels = topology_parameters document.add(f"<br><br><br><b>Results for layer {layer_id}</b><br>") prefix = 'L' + str(layer_id) + '--' testing_phase_ids = list(range(0, len( clustering_agreements[0][0]))) # x-axis values if is_train_test_classifier_computed: weak_class_accuracy_train = measurement_manager.get_custom_data_from_all_runs( f'weak_class_accuracy_train_{layer_id}') weak_class_accuracy_test = measurement_manager.get_custom_data_from_all_runs( f'weak_class_accuracy_test_{layer_id}') title = prefix + ' Weak classifier accuracy (trained on train, tested on train data)' plot_multiple_runs_with_baselines(testing_phase_ids, weak_class_accuracy_train, base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join( docs_folder, title), doc=document) title = prefix + ' Weak classifier accuracy (trained on train, tested on test data)' plot_multiple_runs_with_baselines(testing_phase_ids, weak_class_accuracy_test, base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join( docs_folder, title), doc=document) weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs( f'weak_class_accuracy_{layer_id}') title = prefix + ' Weak classifier accuracy (trained on test, tested on test)' plot_multiple_runs_with_baselines(testing_phase_ids, weak_class_accuracy, base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join(docs_folder, title), doc=document) title = prefix + '- Average boosting duration' plot_multiple_runs(sp_evaluation_steps, average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels, hide_legend=True, path=path.join(docs_folder, title), doc=document) title = prefix + '- Num boosted clusters' plot_multiple_runs(sp_evaluation_steps, num_boosted_clusters, title=title, ylabel='Num boosted clusters / total clusters', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True, path=path.join(docs_folder, title), doc=document) title = prefix + '- Average_deltas' plot_multiple_runs( sp_evaluation_steps, average_deltas, title=title, ylabel='average_deltas', xlabel='steps', labels=labels, disable_ascii_labels=True, hide_legend=True, # use_scatter=True, path=path.join(docs_folder, title), doc=document) # if this is not the top layer, show conv agreements only if required if show_conv_agreements or layer_id == (num_layers - 1): agreements = clustering_agreements for run_id, run_agreements in enumerate(agreements): for expert_id, expert_agreements in enumerate(run_agreements): Task0TaAnalysisTemplate._plot_agreement( prefix, expert_id, run_id, testing_phase_ids, expert_agreements, document, docs_folder) logger.info('done')
def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): """An alternative to the _publish_results method, this is called from _publish_results now Draw and add your plots to the document here. """ steps = measurement_manager.get_values_from_all_runs('current_step') plotted_training_phase_id = measurement_manager.get_values_from_all_runs( 'training_phase_id') plotted_testing_phase_id = measurement_manager.get_values_from_all_runs( 'testing_phase_id') labels = topology_parameters title = 'training_phase_id' f = plot_multiple_runs(steps, plotted_training_phase_id, title=title, ylabel='training_phase_id', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(docs_folder, title), document) title = 'testing_phase_id' f = plot_multiple_runs(steps, plotted_testing_phase_id, title=title, ylabel='testing_phase_id', xlabel='steps', labels=labels) add_fig_to_doc(f, path.join(docs_folder, title), document) max_x = max(steps[0]) average_delta_train_0 = measurement_manager.get_items_from_all_runs( 'average_delta0_train') test_sp_steps = [key for key, value in average_delta_train_0[0]] title = 'average_delta_train_layer0' f = plot_multiple_runs(test_sp_steps, [[value for key, value in sequence] for sequence in average_delta_train_0], title=title, ylabel='average_deltas', xlabel='steps', labels=labels, xlim=[0, max_x], disable_ascii_labels=True, hide_legend=True # use_scatter=True ) add_fig_to_doc(f, path.join(docs_folder, title), document) average_delta_train_1 = measurement_manager.get_items_from_all_runs( 'average_delta1_train') title = 'average_delta_train_layer1' f = plot_multiple_runs(test_sp_steps, [[value for key, value in sequence] for sequence in average_delta_train_1], title=title, ylabel='average_deltas', xlabel='steps', labels=labels, xlim=[0, max_x], disable_ascii_labels=True, hide_legend=True # use_scatter=True ) add_fig_to_doc(f, path.join(docs_folder, title), document) predicted_labels_mse = measurement_manager.get_custom_data_from_all_runs( 'predicted_labels_mse') testing_phases_x = list(range(0, len(predicted_labels_mse[0]))) model_accuracy = measurement_manager.get_custom_data_from_all_runs( 'model_accuracy') baseline_accuracy = measurement_manager.get_custom_data_from_all_runs( 'baseline_accuracy') # plot the classification accuracy title = 'Label reconstruction accuracy (step-wise)' f = plot_multiple_runs_with_baselines(testing_phases_x, model_accuracy, baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True) add_fig_to_doc(f, path.join(docs_folder, title), document) model_se_accuracy = measurement_manager.get_custom_data_from_all_runs( 'model_se_accuracy') baseline_se_accuracy = measurement_manager.get_custom_data_from_all_runs( 'baseline_se_accuracy') # plot the classification accuracy title = 'Label reconstruction SE accuracy (step-wise)' f = plot_multiple_runs_with_baselines(testing_phases_x, model_se_accuracy, baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True) add_fig_to_doc(f, path.join(docs_folder, title), document) baseline_labels_mse = measurement_manager.get_custom_data_from_all_runs( 'baseline_labels_mse') # plot the MSE title = 'Mean Square Error of label reconstruction' f = plot_multiple_runs_with_baselines( testing_phases_x, predicted_labels_mse, baseline_labels_mse, title=title, ylabel='MSE', xlabel='testing phase ID', labels=labels, hide_legend=True, ylim=[-0.1, 0.2] # just for better resolution ) add_fig_to_doc(f, path.join(docs_folder, title), document)
def _publish_results_to_doc(self, doc: Document, date: str, docs_folder: path): """Adds my results to the results produced by the base class""" super()._publish_results_to_doc(doc, date, docs_folder) doc.add(f"<br><br><br><b>Common results</b><br>") labels = ExperimentTemplateBase.extract_params_for_legend(self._topology_parameters_list) title = 'training_phase_id' f = plot_multiple_runs( self._steps, self._plotted_training_phase_id, title=title, ylabel='training_phase_id', xlabel='steps', labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = 'testing_phase_id' f = plot_multiple_runs( self._steps, self._plotted_testing_phase_id, title=title, ylabel='testing_phase_id', xlabel='steps', labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) if self._plot_is_learning: # plot the classification accuracy title = 'Is Learning' f = plot_multiple_runs( self._steps, self._is_learning_info, title=title, ylabel='learning=True?', xlabel='steps', ylim=[-0.1, 1.1], labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) testing_phase_ids = list(range(0, len(self._mutual_info[0]))) # x-axis values title = 'Top-layer L' + str(self._top_layer_id()) + ' Mutual information of SP output with labels' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._mutual_info, self._base_mutual_info, title=title, ylabel='Normalized mutual information', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the classification accuracy title = 'Label reconstruction accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._model_accuracy, self._baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the classification accuracy title = 'Label reconstruction SE accuracy' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._model_se_accuracy, self._baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the MSE title = 'Mean Square Error of label reconstruction' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._predicted_labels_mse, self._baseline_labels_mse, title=title, ylabel='MSE', xlabel='testing phase ID', labels=labels, hide_legend=True, ylim=[-0.1, 0.2] # just for better resolution ) add_fig_to_doc(f, path.join(docs_folder, title), doc) for manager in reversed(self._layer_measurement_managers): manager.publish_results(labels=title, date=date, docs_folder=docs_folder, doc=doc)
def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): """An alternative to the _publish_results method, this is called from _publish_results now Draw and add your plots to the document here. """ steps = measurement_manager.get_values_from_all_runs('current_step') accuracy_1 = measurement_manager.get_values_from_all_runs(ACCURACY_1) accuracy_per_flock_1 = measurement_manager.get_values_from_all_runs( ACCURACY_PER_FLOCK_1) accuracy_2 = measurement_manager.get_values_from_all_runs(ACCURACY_2) accuracy_per_flock_2 = measurement_manager.get_values_from_all_runs( ACCURACY_PER_FLOCK_2) multiple_flocks_alpha = 0.05 # measurement_manager.single_run_measurements[0].get_custom_data('custom') labels = topology_parameters a1 = np.array(accuracy_1) a2 = np.array(accuracy_2) title = 'accuracy_1_2' f = plot_multiple_runs(steps, np.stack((a1, a2), axis=-1), title=title, ylabel='accuracy_1_2', xlabel='steps', labels=labels, other_params=[{ 'color': None }]) add_fig_to_doc(f, os.path.join(docs_folder, title), document) experiment_params = measurement_manager.single_run_measurements[ 0].model_parameters['params']['experiment_params'] experiment_params.component.publish_results(document, docs_folder, measurement_manager, topology_parameters) title = 'accuracy_per_flock_1' f = plot_multiple_runs(steps, accuracy_per_flock_1, title=title, ylabel='accuracy_per_flock_1', xlabel='steps', labels=labels, hide_legend=True, other_params=[{ 'alpha': multiple_flocks_alpha }]) add_fig_to_doc(f, os.path.join(docs_folder, title), document) # title = 'accuracy_2' # f = plot_multiple_runs( # steps, # accuracy_2, # title=title, # ylabel='accuracy_2', # xlabel='steps', # labels=labels # ) # add_fig_to_doc(f, os.path.join(docs_folder, title), document) title = 'accuracy_per_flock_2' f = plot_multiple_runs(steps, accuracy_per_flock_2, title=title, ylabel='accuracy_per_flock_2', xlabel='steps', labels=labels, hide_legend=True, other_params=[{ 'alpha': multiple_flocks_alpha }]) add_fig_to_doc(f, os.path.join(docs_folder, title), document) # Add table with MSE single-step values # prediction_mse_values = [f'{v:.5f}' for v in prediction_mse[0]] # document.add_table(['step', 'prediction_mse'], list(zip(steps[0], prediction_mse_values)), # attribs={'style': 'font-size:0.8em;'}) # for single_run_measurements in measurement_manager.single_run_measurements: # self.publish_one_run(single_run_measurements, document, docs_folder, topology_parameters) doc_path = os.path.join(docs_folder, to_safe_name(self.experiment_name + ".html")) if doc_path.startswith("\\\\?\\"): doc_path = doc_path[len("\\\\?\\"):] # Note not logging to UI now logger.info( f'Results published <a href="file:///{doc_path}">{doc_path}</a>') logger.info(f'Results published {doc_path}')
def _publish_results(self): """Plot and optionally save the results.""" mse = np.array(self._mse) mse_testing = np.array(self._mse_testing) memory_used = torch.tensor(self._memory_used, dtype=torch.float32) window_size = 201 memory_used = (memory_used.view(-1, 1).expand(mse_testing.shape) / (1024**2)) error_memory_ratio = torch.tensor(mse_testing, dtype=torch.float32) * memory_used accuracy_memory_ratio = [] for run_acc, run_mem in zip(self._weak_classifier_results, memory_used): accuracy_memory_ratio_run = [] for acc, mem in zip(run_acc, run_mem): accuracy_memory_ratio_run.append((1 - acc) * mem) accuracy_memory_ratio.append(accuracy_memory_ratio_run) accuracy_memory_ratio = torch.tensor(accuracy_memory_ratio) doc = Document() figs = [] xlabel = "steps" ylabel = "mean reconstruction error" title = "Influence of hyperparameters on reconstruction error (training)" figsize = (18, 12) date = get_stamp() fig = plot_multiple_runs( x_values=np.arange(len(mse[0])), y_values=mse, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), smoothing_window_size=window_size, xlabel=xlabel, ylabel=ylabel, title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) fig = plot_multiple_runs( x_values=np.arange(len(mse[0])), y_values=mse, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel=ylabel, title=title, figsize=figsize, hide_legend=True) add_fig_to_doc( fig, path.join(self._docs_folder, to_safe_name(title) + "_smooth"), doc) title = "Influence of hyperparameters on reconstruction error (testing)" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=mse_testing, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel=ylabel, title=title, figsize=figsize, ) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Weak classifier accuracy" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=np.array(self._weak_classifier_results), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="steps", ylabel="accuracy", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Memory * reconstruction tradeoff (testing)" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=error_memory_ratio.numpy(), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel= "Mean Reconstruction Error times required meogabyte of memory", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Memory * error (= 1 - acc) tradeoff" fig = plot_multiple_runs( x_values=np.array(self._training_steps_before_testing[0]), y_values=accuracy_memory_ratio.numpy(), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel=xlabel, ylabel= "Mean Reconstruction Error times required meogabyte of memory", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Entropy and mean reconstruction error" fig = plot_multiple_runs( x_values=np.array(self._code_entropy), y_values=mse_testing, labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="entropy", ylabel="mean reconstruction error", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Mean reconstruction error and classifier accuracy" fig = plot_multiple_runs( x_values=mse_testing, y_values=np.array(self._weak_classifier_results), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="mean reconstruction error", ylabel="accuracy", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "Entropy and classifier accuracy" fig = plot_multiple_runs( x_values=np.array(self._code_entropy), y_values=np.array(self._weak_classifier_results), labels=ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list), xlabel="entropy", ylabel="accuracy", title=title, figsize=figsize, hide_legend=True) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) doc.write_file( path.join(self._docs_folder, f"{self._topology_class.__name__}_" + date + ".html")) print(self._memory_used)
def publish_results(self, document: Document, docs_folder: str, measurement_manager: MeasurementManager, topology_parameters: List[str]): """An alternative to the _publish_results method, this is called from _publish_results now Draw and add your plots to the document here. """ steps = measurement_manager.get_values_from_all_runs('current_step') plotted_training_phase_id = measurement_manager.get_values_from_all_runs( 'training_phase_id') plotted_testing_phase_id = measurement_manager.get_values_from_all_runs( 'testing_phase_id') plotted_is_learning = measurement_manager.get_values_from_all_runs( 'is_learning') labels = topology_parameters document.add( f"<br><br><br><b>Common results from the TopExpert</b><br>") title = 'training_phase_id' plot_multiple_runs(steps, plotted_training_phase_id, title=title, ylabel='training_phase_id', xlabel='steps', labels=labels, path=path.join(docs_folder, title), doc=document) title = 'testing_phase_id' plot_multiple_runs(steps, plotted_testing_phase_id, title=title, ylabel='testing_phase_id', xlabel='steps', labels=labels, path=path.join(docs_folder, title), doc=document) title = 'is_learning' plot_multiple_runs(steps, plotted_is_learning, title=title, ylabel='is_learning', xlabel='steps', labels=labels, path=path.join(docs_folder, title), doc=document) predicted_labels_mse = measurement_manager.get_custom_data_from_all_runs( 'predicted_labels_mse') testing_phases_x = list(range(0, len(predicted_labels_mse[0]))) model_accuracy = measurement_manager.get_custom_data_from_all_runs( 'model_accuracy') baseline_accuracy = measurement_manager.get_custom_data_from_all_runs( 'baseline_accuracy') # plot the classification accuracy title = 'Label reconstruction accuracy (step-wise)' plot_multiple_runs_with_baselines(testing_phases_x, model_accuracy, baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True, path=path.join(docs_folder, title), doc=document) model_se_accuracy = measurement_manager.get_custom_data_from_all_runs( 'model_se_accuracy') baseline_se_accuracy = measurement_manager.get_custom_data_from_all_runs( 'baseline_se_accuracy') # plot the classification accuracy title = 'Label reconstruction SE accuracy (object-wise)' plot_multiple_runs_with_baselines(testing_phases_x, model_se_accuracy, baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True, path=path.join(docs_folder, title), doc=document) baseline_labels_mse = measurement_manager.get_custom_data_from_all_runs( 'baseline_labels_mse') # plot the MSE title = 'Mean Square Error of label reconstruction' plot_multiple_runs_with_baselines( testing_phases_x, predicted_labels_mse, baseline_labels_mse, title=title, ylabel='MSE', xlabel='testing phase ID', labels=labels, hide_legend=True, ylim=[-0.1, 0.2], # just for better resolution path=path.join(docs_folder, title), doc=document) for layer_id in reversed(range(self._experiment_params.num_layers)): self.publish_results_for_layer( document, docs_folder, measurement_manager, topology_parameters, layer_id, self._experiment_params.num_layers, self._experiment_params.show_conv_agreements, self._experiment_params.is_train_test_classifier_computed)
def _publish_results(self): doc = Document() xlabel = "steps" ylabel = "number of SP forward executions" title_fe_dt = "smoothed derivation of SP forward executions and TP forward executions (SP O QD)" figsize = (18, 12) date = get_stamp() layer_names = ['L0', 'L1', 'L2'] nr_layers = len(layer_names) labels = list(map(lambda x: x + " forward execution", layer_names)) + \ list(map(lambda x: x + " qualitative difference", layer_names)) colors = ['b', 'orange', 'g', 'r', 'p'] color_params = [{'c': color} for color in colors[:nr_layers]] color_ls_params = [{ 'c': color, 'ls': '--' } for color in colors[:nr_layers]] other_params = color_params + color_ls_params params_description = ExperimentTemplateBase.parameters_to_string( self._topology_parameters_list) for run in range(len(self.sp_executions)): sp_execution_dt = list( map(compute_derivations, self.sp_executions[run])) sp_output_dt = self.sp_output_stability[run] title = title_fe_dt + f" run {run} " fig = plot_multiple_runs(x_values=self.training_steps, y_values=sp_execution_dt + sp_output_dt, ylim=[0, 1], labels=labels, smoothing_window_size=501, xlabel=xlabel, ylabel=ylabel, title=title + params_description[run], figsize=figsize, hide_legend=False, other_params=other_params) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "classification accuracy from reconstructed labels" fig = plot_multiple_runs(x_values=list(range( self._num_testing_phases)), y_values=self.classification_accuracy, ylim=[0, 1], labels=params_description, smoothing_window_size=None, xlabel="accuracy", ylabel="phases", title=title, figsize=figsize, hide_legend=False) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) title = "SE classification accuracy from reconstructed labels" fig = plot_multiple_runs(x_values=list(range( self._num_testing_phases)), y_values=self.classification_accuracy_se, ylim=[0, 1], labels=params_description, smoothing_window_size=None, xlabel="SE accuracy", ylabel="phases", title=title, figsize=figsize, hide_legend=False) add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)), doc) doc.write_file(path.join(self._docs_folder, f"main.html"))
def _publish_results_to_doc(self, doc: Document, date: str, docs_folder: os.path): """An alternative to the _publish_results method, this is called from _publish_results now Draw and add your topologies to the document here. """ labels = ExperimentTemplateBase.extract_params_for_legend(self._topology_parameters_list) title = 'training_phase_id' f = plot_multiple_runs( self._steps, self._plotted_training_phase_id, title=title, ylabel='training_phase_id', xlabel='steps', labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = 'testing_phase_id' f = plot_multiple_runs( self._steps, self._plotted_testing_phase_id, title=title, ylabel='testing_phase_id', xlabel='steps', labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the classification accuracy title = 'Is Learning' f = plot_multiple_runs( self._steps, self._is_learning_info, title=title, ylabel='learning=True?', xlabel='steps', ylim=[-0.1, 1.1], labels=labels ) add_fig_to_doc(f, path.join(docs_folder, title), doc) max_x = max(self._steps[0]) title = 'average_delta_train_layer0' f = plot_multiple_runs( self._test_sp_steps[0], self._average_delta_train_0, title=title, ylabel='average_deltas', xlabel='steps', labels=labels, xlim=[0, max_x], disable_ascii_labels=True, hide_legend=True # use_scatter=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = 'average_delta_train_layer1' f = plot_multiple_runs( self._test_sp_steps[0], self._average_delta_train_1, title=title, ylabel='average_deltas', xlabel='steps', labels=labels, xlim=[0, max_x], disable_ascii_labels=True, hide_legend=True # use_scatter=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) testing_phases_x = list(range(0, len(self._predicted_labels_mse[0]))) # plot the classification accuracy title = 'Label reconstruction accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phases_x, self._model_accuracy, self._baseline_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the classification accuracy title = 'Label reconstruction SE accuracy (step-wise)' f = plot_multiple_runs_with_baselines( testing_phases_x, self._model_se_accuracy, self._baseline_se_accuracy, title=title, ylabel='accuracy (1 ~ 100%)', xlabel='testing_phase ID', ylim=[-0.1, 1.1], labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # plot the MSE title = 'Mean Square Error of label reconstruction' f = plot_multiple_runs_with_baselines( testing_phases_x, self._predicted_labels_mse, self._baseline_labels_mse, title=title, ylabel='MSE', xlabel='testing phase ID', labels=labels, hide_legend=True, ylim=[-0.1, 0.2] # just for better resolution ) add_fig_to_doc(f, path.join(docs_folder, title), doc)
def publish_results(self, labels, date, docs_folder, doc): doc.add(f"<br><br><br><b>Results for layer {self._layer_id}</b><br>") prefix = 'L' + str(self._layer_id) + '--' testing_phase_ids = list(range(0, len(self._clustering_agreements[0][0]))) # x-axis values title = prefix + ' Weak classifier accuracy trained on SP outputs to labels' f = plot_multiple_runs_with_baselines( testing_phase_ids, self._weak_class_accuracy, self._base_weak_class_accuracy, title=title, ylabel='Accuracy (1 ~ 100%)', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = prefix + '- Average boosting duration' f = plot_multiple_runs( self._sp_evaluation_steps[0], self._average_boosting_dur, title=title, ylabel='duration', xlabel='steps', labels=labels, hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = prefix + '- Num boosted clusters' f = plot_multiple_runs( self._sp_evaluation_steps[0], self._num_boosted_clusters, title=title, ylabel='Num boosted clusters / total clusters', xlabel='steps', labels=labels, ylim=[-0.1, 1.1], hide_legend=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) title = prefix + '- Average_deltas' f = plot_multiple_runs( self._sp_evaluation_steps[0], self._average_deltas, title=title, ylabel='average_deltas', xlabel='steps', labels=labels, disable_ascii_labels=True, hide_legend=True # use_scatter=True ) add_fig_to_doc(f, path.join(docs_folder, title), doc) # if this is not the top layer, show conv agreements only if required if self._show_conv_agreements or self._layer_id == (self._num_layers - 1): agreements = self._clustering_agreements for run_id, run_agreements in enumerate(agreements): for expert_id, expert_agreements in enumerate(run_agreements): self._plot_agreement( prefix, expert_id, run_id, testing_phase_ids, expert_agreements, docs_folder, doc )