def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
document.add(Heading(self.template_params.experiment_name))
for title in measurement_manager.single_run_measurements[
0].get_item_names():
if title not in [
"initial training error", "initial testing error",
"untraining error", "retraining error", "testing error"
]:
continue
data = measurement_manager.get_values_from_all_runs(title)
n_runs = len(data)
if self.template_params.every_second_is_baseline: # TODO MS
labels = (n_runs // 2) * ["experiment", "baseline"]
else:
labels = n_runs * [""]
plot_multiple_runs(list(range(len(data[0]))),
data,
ylabel="error",
xlabel="steps",
labels=labels,
title=title,
smoothing_window_size=21,
path=path.join(docs_folder, title),
doc=document)
def _prepare_document(self) -> (Document, str, os.path):
"""Prepares the document for writing the results"""
doc = Document()
date = get_stamp()
doc.add(self._get_heading(date))
return doc, date
def add_fig_to_doc(fig, path_name, doc: Document):
name = to_safe_path(path_name)
# save figure also as pickle
if not os.path.exists(os.path.dirname(name)):
os.makedirs(os.path.dirname(name))
with open(name + '.fpk', 'wb+') as file:
pickle.dump(fig, file, protocol=2)
name_svg = name + '.svg'
fig.savefig(name_svg, format='svg')
im = Image(os.path.basename(name_svg))
doc.add(im)
def _publish_results(self):
"""Plot and save the results."""
doc = Document()
date = get_stamp()
labels = ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list)
title = 'Mutual Information labels vs ' + self._experiment_name
self.plot_save(title, self._mutual_info, self._baseline_mutual_info,
'Norm. mutual information', labels, date,
self._docs_folder, doc)
title = 'Weak classifier accuracy labels vs ' + self._experiment_name
self.plot_save(title, self._classifier_accuracy,
self._baseline_classifier_accuracy,
'Classifier accuracy', labels, date, self._docs_folder,
doc) #, smoothing_size=3)
title = 'average delta'
f = plot_multiple_runs(
self._different_steps[0], # here the X axes are identical
self._average_delta,
title=title,
ylabel='log(delta)',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'average boosting duration'
f = plot_multiple_runs(self._different_steps[0],
self._average_boosting_dur,
title=title,
ylabel='duration',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
doc.write_file(
path.join(self._docs_folder,
to_safe_name(self._complete_name() + date + ".html")))
print('done')
def __init__(self, template: ExperimentTemplateBase, docs_folder: str,
template_class: Type, default_topology_parameters: Dict[str,
Any],
topology_params: List[Dict[str, Any]],
runner_params: ExperimentParams):
"""Prepares the document for writing the results"""
self._document = Document()
self._template = template
self._docs_folder = docs_folder
self._template_class = template_class
self._default_topology_parameters = default_topology_parameters
self._topology_params = topology_params
self._runner_params = runner_params
parameter_extractor = ParameterExtractor(
self._default_topology_parameters)
self._common_parameters, self._differing_parameters = parameter_extractor.extract(
self._topology_params)
if not self._docs_folder:
raise RuntimeError("Docs folder not defined")
def test_doc_creation():
document = Document().\
add(Heading("Look at his heading")). \
add("This is a very nice document.<br><br>").\
add(Figure.from_params('test_image.png',
"Behold our powerful visualization",
height=200,
width=400)).\
add(Matrix(torch.rand((2, 3)), m_labels=['a', 'b'], n_labels=['10', '20', '30'],
format_func=lambda x: "%.5f" % x))
# The following line will write to a non-temporary file.
# Uncomment if you want to see what the html file looks like.
# document.write_file("test_file.html")
with tempfile.TemporaryFile() as document_file:
doc_text = document.as_text()
document_file.write(doc_text.encode('UTF-8'))
document_file.seek(0)
read_text = document_file.readline()
assert read_text.startswith(b"<!DOCTYPE html>")
def _publish_results(self):
"""Plot and optionally save the results."""
doc = Document()
date = get_stamp()
labels = ExperimentTemplateBase.parameters_to_string(self._topology_parameters_list)
title = f'FPS vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._fps, 'FPS',
labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size)
smoothing = len(self._fps[0]) // 100 # adaptive smoothing
if smoothing % 2 == 0:
smoothing += 1 # make odd
title = 'FPS vs. ' + self._experiment_name + f' (smoothing {smoothing})'
self.plot_save(title, self._fps, 'FPS', labels, date, self._docs_folder, doc,
smoothing_window_size=smoothing)
title = f'max_memory_allocated() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._max_mem, 'Max memory', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
title = f'memory_allocated() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._current_mem, 'Current memory', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
title = f'max_cached() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._max_cached, 'Max cached mem', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
title = f'memory_cached() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._current_cached, 'Current cached mem', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
self._publish_aux_results(labels, date, self._docs_folder, doc)
doc.write_file(path.join(self._docs_folder, f"{self._topology_class.__name__}_" + date + ".html"))
print('done')
def _print_statistics(self, doc: Document):
doc.add(Heading("Overall statistics"))
for run in range(len(self._tasks_solved)):
doc.add(Heading(f" Run {run}:", 2))
experiment_statistics = self._tasks_solved[run]
tasks_solved_statuses = [[stats.task_id, stats.task_solved]
for stats in experiment_statistics]
doc.add(f"Tasks solved: {tasks_solved_statuses}.<br />")
for task_stats in experiment_statistics:
self._print_task_stats(doc, task_stats)
def _print_task_stats(self, doc: Document,
task_stats: TaskExperimentStatistics):
doc.add(Heading(f"Task {task_stats.task_id}<br />", 3))
doc.add(f"Solved: {task_stats.task_solved}<br />")
if task_stats.instances_seen_training != 0:
ratio = task_stats.instances_solved_training / task_stats.instances_seen_training
else:
ratio = 0
doc.add(
f"Instances solved / seen during training: {task_stats.instances_solved_training} / "
f"{task_stats.instances_seen_training} = {ratio*100:.1f} %<br />")
if task_stats.instances_seen_testing != 0:
ratio = task_stats.instances_solved_testing / task_stats.instances_seen_testing
else:
ratio = 0
doc.add(
f"Instances solved / seen during testing: {task_stats.instances_solved_testing} / "
f"{task_stats.instances_seen_testing} = {ratio*100:.1f} %<br />")
def publish_results_for_layer(document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str],
layer_id: int, num_layers: int,
show_conv_agreements: bool,
is_train_test_classifier_computed: bool):
"""Publish results for each layer separately
This uses the data measured and computed for each run by the Task0TaAnalysisLayerComponent and aggregated
and stored in the measurement_manager.
"""
logger.info(f'publishing results for layer {layer_id}...')
num_boosted_clusters = measurement_manager.get_values_from_all_runs(
f'num_boosted_getter_{layer_id}')
average_boosting_dur = measurement_manager.get_values_from_all_runs(
f'average_boosting_dur_{layer_id}')
average_deltas = measurement_manager.get_values_from_all_runs(
f'average_delta_{layer_id}')
base_weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs(
f'base_weak_class_accuracy_{layer_id}')
clustering_agreements = measurement_manager.get_custom_data_from_all_runs(
f'clustering_agreements_{layer_id}')
average_steps_deltas = measurement_manager.get_items_from_all_runs(
f'average_delta_{layer_id}')
sp_evaluation_steps = [key for key, value in average_steps_deltas[0]]
labels = topology_parameters
document.add(f"<br><br><br><b>Results for layer {layer_id}</b><br>")
prefix = 'L' + str(layer_id) + '--'
testing_phase_ids = list(range(0, len(
clustering_agreements[0][0]))) # x-axis values
if is_train_test_classifier_computed:
weak_class_accuracy_train = measurement_manager.get_custom_data_from_all_runs(
f'weak_class_accuracy_train_{layer_id}')
weak_class_accuracy_test = measurement_manager.get_custom_data_from_all_runs(
f'weak_class_accuracy_test_{layer_id}')
title = prefix + ' Weak classifier accuracy (trained on train, tested on train data)'
plot_multiple_runs_with_baselines(testing_phase_ids,
weak_class_accuracy_train,
base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(
docs_folder, title),
doc=document)
title = prefix + ' Weak classifier accuracy (trained on train, tested on test data)'
plot_multiple_runs_with_baselines(testing_phase_ids,
weak_class_accuracy_test,
base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(
docs_folder, title),
doc=document)
weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs(
f'weak_class_accuracy_{layer_id}')
title = prefix + ' Weak classifier accuracy (trained on test, tested on test)'
plot_multiple_runs_with_baselines(testing_phase_ids,
weak_class_accuracy,
base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
title = prefix + '- Average boosting duration'
plot_multiple_runs(sp_evaluation_steps,
average_boosting_dur,
title=title,
ylabel='duration',
xlabel='steps',
labels=labels,
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
title = prefix + '- Num boosted clusters'
plot_multiple_runs(sp_evaluation_steps,
num_boosted_clusters,
title=title,
ylabel='Num boosted clusters / total clusters',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
title = prefix + '- Average_deltas'
plot_multiple_runs(
sp_evaluation_steps,
average_deltas,
title=title,
ylabel='average_deltas',
xlabel='steps',
labels=labels,
disable_ascii_labels=True,
hide_legend=True,
# use_scatter=True,
path=path.join(docs_folder, title),
doc=document)
# if this is not the top layer, show conv agreements only if required
if show_conv_agreements or layer_id == (num_layers - 1):
agreements = clustering_agreements
for run_id, run_agreements in enumerate(agreements):
for expert_id, expert_agreements in enumerate(run_agreements):
Task0TaAnalysisTemplate._plot_agreement(
prefix, expert_id, run_id, testing_phase_ids,
expert_agreements, document, docs_folder)
logger.info('done')
def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
"""An alternative to the _publish_results method, this is called from _publish_results now
Draw and add your plots to the document here.
"""
steps = measurement_manager.get_values_from_all_runs('current_step')
plotted_training_phase_id = measurement_manager.get_values_from_all_runs(
'training_phase_id')
plotted_testing_phase_id = measurement_manager.get_values_from_all_runs(
'testing_phase_id')
plotted_is_learning = measurement_manager.get_values_from_all_runs(
'is_learning')
labels = topology_parameters
document.add(
f"<br><br><br><b>Common results from the TopExpert</b><br>")
title = 'training_phase_id'
plot_multiple_runs(steps,
plotted_training_phase_id,
title=title,
ylabel='training_phase_id',
xlabel='steps',
labels=labels,
path=path.join(docs_folder, title),
doc=document)
title = 'testing_phase_id'
plot_multiple_runs(steps,
plotted_testing_phase_id,
title=title,
ylabel='testing_phase_id',
xlabel='steps',
labels=labels,
path=path.join(docs_folder, title),
doc=document)
title = 'is_learning'
plot_multiple_runs(steps,
plotted_is_learning,
title=title,
ylabel='is_learning',
xlabel='steps',
labels=labels,
path=path.join(docs_folder, title),
doc=document)
predicted_labels_mse = measurement_manager.get_custom_data_from_all_runs(
'predicted_labels_mse')
testing_phases_x = list(range(0, len(predicted_labels_mse[0])))
model_accuracy = measurement_manager.get_custom_data_from_all_runs(
'model_accuracy')
baseline_accuracy = measurement_manager.get_custom_data_from_all_runs(
'baseline_accuracy')
# plot the classification accuracy
title = 'Label reconstruction accuracy (step-wise)'
plot_multiple_runs_with_baselines(testing_phases_x,
model_accuracy,
baseline_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
model_se_accuracy = measurement_manager.get_custom_data_from_all_runs(
'model_se_accuracy')
baseline_se_accuracy = measurement_manager.get_custom_data_from_all_runs(
'baseline_se_accuracy')
# plot the classification accuracy
title = 'Label reconstruction SE accuracy (object-wise)'
plot_multiple_runs_with_baselines(testing_phases_x,
model_se_accuracy,
baseline_se_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing_phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
baseline_labels_mse = measurement_manager.get_custom_data_from_all_runs(
'baseline_labels_mse')
# plot the MSE
title = 'Mean Square Error of label reconstruction'
plot_multiple_runs_with_baselines(
testing_phases_x,
predicted_labels_mse,
baseline_labels_mse,
title=title,
ylabel='MSE',
xlabel='testing phase ID',
labels=labels,
hide_legend=True,
ylim=[-0.1, 0.2], # just for better resolution
path=path.join(docs_folder, title),
doc=document)
for layer_id in reversed(range(self._experiment_params.num_layers)):
self.publish_results_for_layer(
document, docs_folder, measurement_manager,
topology_parameters, layer_id,
self._experiment_params.num_layers,
self._experiment_params.show_conv_agreements,
self._experiment_params.is_train_test_classifier_computed)
class DocumentPublisher:
"""A class responsible for the publishing of results based on a template."""
_header_table_width = 60
def __init__(self, template: ExperimentTemplateBase, docs_folder: str,
template_class: Type, default_topology_parameters: Dict[str,
Any],
topology_params: List[Dict[str, Any]],
runner_params: ExperimentParams):
"""Prepares the document for writing the results"""
self._document = Document()
self._template = template
self._docs_folder = docs_folder
self._template_class = template_class
self._default_topology_parameters = default_topology_parameters
self._topology_params = topology_params
self._runner_params = runner_params
parameter_extractor = ParameterExtractor(
self._default_topology_parameters)
self._common_parameters, self._differing_parameters = parameter_extractor.extract(
self._topology_params)
if not self._docs_folder:
raise RuntimeError("Docs folder not defined")
def publish_results(self, timestamp: str,
measurement_manager: MeasurementManager):
"""Publishes the result and returns the path of the main html document."""
self._document.add(self._get_heading(timestamp))
self._template.publish_results(
self._document, self._docs_folder, measurement_manager,
self.parameters_to_string(self._differing_parameters))
logger.info(f"Results documents generated in {self._docs_folder}")
return self._write_document(self._docs_folder)
def _write_document(self, docs_folder: str) -> str:
"""Writes the document to the disk."""
if not os.path.isdir(docs_folder):
os.makedirs(docs_folder)
doc_path = os.path.join(
docs_folder,
to_safe_name(self._template.experiment_name + ".html"))
self._document.write_file(doc_path)
logger.info('done')
return doc_path
@classmethod
def _param_to_html(cls,
name: str,
value: Any,
indent: bool = False,
bold: bool = False) -> str:
if hasattr(value, '_asdict'):
value = value._asdict()
if isinstance(value, dict):
row = cls._param_to_html(name, '', bold=True) + ''.join([
cls._param_to_html(key, val, indent=True)
for key, val in value.items()
])
else:
style = ' style="padding-left: 20px"' if indent else ''
name = f'<strong>{name}</strong>' if bold else name
row = "<tr>" + \
f"<td{style}>{name}</td>" + \
f"<td>{cls._param_value_to_string(value)}</td>" + \
"</tr>"
return row
def _runner_configuration_to_html(self) -> str:
"""Convert the runner's configuration to the string for the html heading."""
result = f"\n<b>Experiment and runner configuration</b>:<br> "
result += self._get_table_header()
# values in columns
result += self._param_to_html("runner", self._runner_params)
for key, value in self._template.get_additional_parameters().items():
result += self._param_to_html(key, value)
result += "</table>"
return result
def _get_table_header(self):
header = f"<table style=\"width:{self._header_table_width}%\">" + \
"<tr>" + \
"<th style=\"text-align: left\">Param name</th>" + \
"<th style=\"text-align:left\">Param value</th>" + \
"</tr>"
return header
@classmethod
def parameters_to_string(cls, parameters: List[Dict[str,
Any]]) -> List[str]:
return [
", ".join(f"{param}: {cls._param_value_to_string(value)}"
for param, value in parameter.items())
for parameter in parameters
]
@classmethod
def _param_value_to_string(cls, value):
"""Lists of normal values are parsed OK, param value can be also list of classes, parse to readable string."""
if type(value) in (list, tuple):
return [cls._param_value_to_string(x) for x in value]
elif isinstance(value, type):
return value.__name__
return value
def _get_heading(self, date: str):
"""Get heading of the html file with the experiment description"""
info = f"<p><b>Template</b>: {self._template_class.__name__}<br>" + \
f"\n<b>Experiment_name</b>: {self._template.experiment_name}<br>" + \
f"\n<b>Date:</b> {date[1:]}</p>"
info += '<table style="width: 100%">'
info += '<tr>'
info += '<td style="width: 50%">'
info += f"\n<b>List of common parameters</b>:<br> "
# create table with the params
info += self._get_table_header()
for key, value in self._common_parameters.items():
info += self._param_to_html(key, value)
info += "</table>"
info += '</td>'
info += '<td>'
# add the description of the template configuration
info += self._runner_configuration_to_html()
info += '</td>'
info += '</tr>'
info += '</table>'
return info
def _publish_results(self):
"""Plot and save the results."""
doc = Document()
date = get_stamp()
labels = ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list)
for manager in self._layer_measurement_managers:
manager.publish_results(labels=labels,
date=date,
docs_folder=self._docs_folder,
doc=doc)
# plot the running MSE
title = 'Mean Square Error of TA classification'
f = plot_multiple_runs_with_baselines(self._steps,
self._predicted_labels_mse,
self._baseline_labels_mse,
title=title,
ylabel='mse',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'TA classification accuracy'
f = plot_multiple_runs_with_baselines(
self._steps,
self._classification_accuracy,
self._random_classification_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'TA classification accuracy - (SE-metric)'
f = plot_multiple_runs_with_baselines(
self._steps,
self._se_classification_accuracy,
self._se_random_classification_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'is_learning'
f = plot_multiple_runs(
self._steps,
self._is_learning_info,
title=title,
ylabel='is learning',
xlabel='steps',
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
doc.write_file(
path.join(self._docs_folder,
to_safe_name(self._complete_name() + date + ".html")))
print('done')
def _publish_results(self):
doc = Document()
doc.add(self._get_heading('_not set'))
params_str = pprint.pformat(
Task0TrainTestTemplateRelearning.pprint_parameters_to_string(self._topology_parameters_list), 1, 160
)
doc.add(f"<p><b>Parameters:</b><br /><pre>learning rate: {self._learning_rate},\n" + params_str + "</pre></p>")
labels = ExperimentTemplateBase.parameters_to_string(self._topology_parameters_list)
testing_phases = list(range(0, len(self._clustering_agreement_list[0][0])))
for i, run_label in enumerate(self._run_labels):
title = 'Clustering agreement ' + run_label
f = plot_multiple_runs(
testing_phases,
self._clustering_agreement_list[i],
title=title,
ylabel='agreement',
xlabel='testing training_phases',
disable_ascii_labels=True,
hide_legend=True,
ylim=[ClusterAgreementMeasurement.NO_VALUE - 0.1, 1.1]
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'Model classification accuracy (step-wise)'
f = plot_multiple_runs_with_baselines(
testing_phases,
self._model_accuracy,
self._baseline_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing phase',
ylim=[-0.1, 1.1],
labels=self._run_labels
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'Model SE classification accuracy (step-wise)'
f = plot_multiple_runs_with_baselines(
testing_phases,
self._model_se_accuracy,
self._baseline_se_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing_phase',
ylim=[-0.1, 1.1],
labels=self._run_labels
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'Weak classifier accuracy trained on SP outputs to labels'
f = plot_multiple_runs_with_baselines(
testing_phases,
self._weak_class_accuracy,
self._base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'Average Deltas Train'
f = plot_multiple_runs(
[self._sp_evaluation_period * i for i in range(0, len(self._average_delta_train[0]))],
self._average_delta_train,
title=title,
ylabel='average_deltas',
xlabel='steps',
labels=self._run_labels,
disable_ascii_labels=True,
use_scatter=False
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'Average Deltas Test'
f = plot_multiple_runs(
[self._sp_evaluation_period * i for i in range(0, len(self._average_delta_test[0]))],
self._average_delta_test,
title=title,
ylabel='average_deltas',
xlabel='steps',
labels=self._run_labels,
disable_ascii_labels=True,
use_scatter=False
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'Average boosting duration'
f = plot_multiple_runs(
[self._sp_evaluation_period * i for i in range(0, len(self._average_boosting_dur[0]))],
self._average_boosting_dur,
title=title,
ylabel='duration',
xlabel='steps',
labels=labels,
hide_legend=True
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
doc.write_file(path.join(self._docs_folder, "main.html"))
print('done')
def _publish_results(self):
doc = Document()
xlabel = "steps"
ylabel = "number of SP forward executions"
title_fe_dt = "smoothed derivation of SP forward executions and TP forward executions (SP O QD)"
figsize = (18, 12)
date = get_stamp()
layer_names = ['L0', 'L1', 'L2']
nr_layers = len(layer_names)
labels = list(map(lambda x: x + " forward execution", layer_names)) + \
list(map(lambda x: x + " qualitative difference", layer_names))
colors = ['b', 'orange', 'g', 'r', 'p']
color_params = [{'c': color} for color in colors[:nr_layers]]
color_ls_params = [{
'c': color,
'ls': '--'
} for color in colors[:nr_layers]]
other_params = color_params + color_ls_params
params_description = ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list)
for run in range(len(self.sp_executions)):
sp_execution_dt = list(
map(compute_derivations, self.sp_executions[run]))
sp_output_dt = self.sp_output_stability[run]
title = title_fe_dt + f" run {run} "
fig = plot_multiple_runs(x_values=self.training_steps,
y_values=sp_execution_dt + sp_output_dt,
ylim=[0, 1],
labels=labels,
smoothing_window_size=501,
xlabel=xlabel,
ylabel=ylabel,
title=title + params_description[run],
figsize=figsize,
hide_legend=False,
other_params=other_params)
add_fig_to_doc(fig,
path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "classification accuracy from reconstructed labels"
fig = plot_multiple_runs(x_values=list(range(
self._num_testing_phases)),
y_values=self.classification_accuracy,
ylim=[0, 1],
labels=params_description,
smoothing_window_size=None,
xlabel="accuracy",
ylabel="phases",
title=title,
figsize=figsize,
hide_legend=False)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "SE classification accuracy from reconstructed labels"
fig = plot_multiple_runs(x_values=list(range(
self._num_testing_phases)),
y_values=self.classification_accuracy_se,
ylim=[0, 1],
labels=params_description,
smoothing_window_size=None,
xlabel="SE accuracy",
ylabel="phases",
title=title,
figsize=figsize,
hide_legend=False)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
doc.write_file(path.join(self._docs_folder, f"main.html"))
def _write_document(self, doc: Document, date: str, docs_folder: os.path):
""" Writes the document to the hdd after adding the graph (see the _publish_results())."""
doc.write_file(
path.join(docs_folder,
to_safe_name(self._complete_name() + date + ".html")))
print('done')
def _publish_results(self):
"""Plot and optionally save the results."""
mse = np.array(self._mse)
mse_testing = np.array(self._mse_testing)
memory_used = torch.tensor(self._memory_used, dtype=torch.float32)
window_size = 201
memory_used = (memory_used.view(-1, 1).expand(mse_testing.shape) /
(1024**2))
error_memory_ratio = torch.tensor(mse_testing,
dtype=torch.float32) * memory_used
accuracy_memory_ratio = []
for run_acc, run_mem in zip(self._weak_classifier_results,
memory_used):
accuracy_memory_ratio_run = []
for acc, mem in zip(run_acc, run_mem):
accuracy_memory_ratio_run.append((1 - acc) * mem)
accuracy_memory_ratio.append(accuracy_memory_ratio_run)
accuracy_memory_ratio = torch.tensor(accuracy_memory_ratio)
doc = Document()
figs = []
xlabel = "steps"
ylabel = "mean reconstruction error"
title = "Influence of hyperparameters on reconstruction error (training)"
figsize = (18, 12)
date = get_stamp()
fig = plot_multiple_runs(
x_values=np.arange(len(mse[0])),
y_values=mse,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
smoothing_window_size=window_size,
xlabel=xlabel,
ylabel=ylabel,
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
fig = plot_multiple_runs(
x_values=np.arange(len(mse[0])),
y_values=mse,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=ylabel,
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(
fig, path.join(self._docs_folder,
to_safe_name(title) + "_smooth"), doc)
title = "Influence of hyperparameters on reconstruction error (testing)"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=mse_testing,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=ylabel,
title=title,
figsize=figsize,
)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Weak classifier accuracy"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=np.array(self._weak_classifier_results),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="steps",
ylabel="accuracy",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Memory * reconstruction tradeoff (testing)"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=error_memory_ratio.numpy(),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=
"Mean Reconstruction Error times required meogabyte of memory",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Memory * error (= 1 - acc) tradeoff"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=accuracy_memory_ratio.numpy(),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=
"Mean Reconstruction Error times required meogabyte of memory",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Entropy and mean reconstruction error"
fig = plot_multiple_runs(
x_values=np.array(self._code_entropy),
y_values=mse_testing,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="entropy",
ylabel="mean reconstruction error",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Mean reconstruction error and classifier accuracy"
fig = plot_multiple_runs(
x_values=mse_testing,
y_values=np.array(self._weak_classifier_results),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="mean reconstruction error",
ylabel="accuracy",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Entropy and classifier accuracy"
fig = plot_multiple_runs(
x_values=np.array(self._code_entropy),
y_values=np.array(self._weak_classifier_results),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="entropy",
ylabel="accuracy",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
doc.write_file(
path.join(self._docs_folder,
f"{self._topology_class.__name__}_" + date + ".html"))
print(self._memory_used)
def _publish_results_to_doc(self, doc: Document, date: str, docs_folder: path):
"""Adds my results to the results produced by the base class"""
super()._publish_results_to_doc(doc, date, docs_folder)
doc.add(f"<br><br><br><b>Common results</b><br>")
labels = ExperimentTemplateBase.extract_params_for_legend(self._topology_parameters_list)
title = 'training_phase_id'
f = plot_multiple_runs(
self._steps,
self._plotted_training_phase_id,
title=title,
ylabel='training_phase_id',
xlabel='steps',
labels=labels
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
title = 'testing_phase_id'
f = plot_multiple_runs(
self._steps,
self._plotted_testing_phase_id,
title=title,
ylabel='testing_phase_id',
xlabel='steps',
labels=labels
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
if self._plot_is_learning:
# plot the classification accuracy
title = 'Is Learning'
f = plot_multiple_runs(
self._steps,
self._is_learning_info,
title=title,
ylabel='learning=True?',
xlabel='steps',
ylim=[-0.1, 1.1],
labels=labels
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
testing_phase_ids = list(range(0, len(self._mutual_info[0]))) # x-axis values
title = 'Top-layer L' + str(self._top_layer_id()) + ' Mutual information of SP output with labels'
f = plot_multiple_runs_with_baselines(
testing_phase_ids,
self._mutual_info,
self._base_mutual_info,
title=title,
ylabel='Normalized mutual information',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
# plot the classification accuracy
title = 'Label reconstruction accuracy (step-wise)'
f = plot_multiple_runs_with_baselines(
testing_phase_ids,
self._model_accuracy,
self._baseline_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
# plot the classification accuracy
title = 'Label reconstruction SE accuracy'
f = plot_multiple_runs_with_baselines(
testing_phase_ids,
self._model_se_accuracy,
self._baseline_se_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing_phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
# plot the MSE
title = 'Mean Square Error of label reconstruction'
f = plot_multiple_runs_with_baselines(
testing_phase_ids,
self._predicted_labels_mse,
self._baseline_labels_mse,
title=title,
ylabel='MSE',
xlabel='testing phase ID',
labels=labels,
hide_legend=True,
ylim=[-0.1, 0.2] # just for better resolution
)
add_fig_to_doc(f, path.join(docs_folder, title), doc)
for manager in reversed(self._layer_measurement_managers):
manager.publish_results(labels=title, date=date, docs_folder=docs_folder, doc=doc)