def run(self, runner: ExperimentRunner, auto_start: bool = True):
"""A generator which performs one step of the experiment per iteration.
One step here means one simulation step of the topology of a single experiment run. If no run exists, the next
run in the queue of all runs is created and a step is done there. If all single experiment runs were already
finished, the simulation is stopped and the results are generated.
"""
self._timestamp_start = get_stamp()
self.measurement_manager = MeasurementManager(
self._experiment_folder, self._measurement_manager_params)
if self._params.clear_cache:
self.measurement_manager.clear_cache()
elif self._params.load_cache:
# This can only happen in the cache is not cleared.
self.measurement_manager.load_cache_index()
garbage_collect(
) # empty the memory from previous experiments so that they do not influence this one
logger.info(
f"Starting experiment with {len(self._topology_parameters)} runs")
self._run_all(runner, auto_start)
if not self._params.calculate_statistics:
# No results get published.
return
self.results_path = self.produce_results()
def serialize_to(self, folder_path: str):
"""Creates `.crm` metadata file and `.drm` (or '.zrm' for compressed data) data file in specified path."""
# experiment_id = self.model_name + '_' + str(list(self.model_parameters.values())) + '_' + get_stamp()
file_name = self.model_name + get_stamp()
if self.use_zip:
file_extension = '.zrm'
else:
file_extension = '.drm'
drm_file_name = file_name + file_extension
crm_file_name = file_name + '.ycrm'
drm_file_path = path.join(folder_path, drm_file_name)
crm_file_path = path.join(folder_path, crm_file_name)
drm_file_path = to_safe_path(drm_file_path)
crm_file_path = to_safe_path(crm_file_path)
params_to_serialize = self.to_serializable_params(
self.model_parameters)
if self.use_zip:
save_zip(self, drm_file_path, protocol=2)
else:
with open(drm_file_path, 'wb') as drm_file:
pickle.dump(self, drm_file, protocol=2)
with open(crm_file_path, 'w') as crm_file:
yaml = YAML()
yaml.dump(
{
'model_name': self.model_name,
'model_parameters': params_to_serialize,
'data_file': drm_file_name
}, crm_file)
def _prepare_document(self) -> (Document, str, os.path):
"""Prepares the document for writing the results"""
doc = Document()
date = get_stamp()
doc.add(self._get_heading(date))
return doc, date
def _create_cache_folders(self,
clear_old_cache: bool = True,
alternative_folder=None):
if alternative_folder is None:
experiment_folder = path.join(get_experiment_results_folder(),
self._experiment_template_name())
else:
experiment_folder = alternative_folder
if clear_old_cache:
try:
for file_name in os.listdir(experiment_folder):
if file_name.endswith('.drm') or file_name.endswith(
'.crm'):
os.unlink(path.join(experiment_folder, file_name))
except FileNotFoundError as e:
logger.warning("cache folder not found: \n" + str(e))
pass
if self._save_cache and not os.path.exists(experiment_folder):
os.makedirs(experiment_folder)
date = get_stamp()
docs_folder = path.join(experiment_folder, "docs",
self._complete_name() + date)
if self._save_cache and not os.path.exists(docs_folder):
os.makedirs(docs_folder)
return experiment_folder, docs_folder
def _publish_results(self):
"""Plot and save the results."""
doc = Document()
date = get_stamp()
labels = ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list)
title = 'Mutual Information labels vs ' + self._experiment_name
self.plot_save(title, self._mutual_info, self._baseline_mutual_info,
'Norm. mutual information', labels, date,
self._docs_folder, doc)
title = 'Weak classifier accuracy labels vs ' + self._experiment_name
self.plot_save(title, self._classifier_accuracy,
self._baseline_classifier_accuracy,
'Classifier accuracy', labels, date, self._docs_folder,
doc) #, smoothing_size=3)
title = 'average delta'
f = plot_multiple_runs(
self._different_steps[0], # here the X axes are identical
self._average_delta,
title=title,
ylabel='log(delta)',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'average boosting duration'
f = plot_multiple_runs(self._different_steps[0],
self._average_boosting_dur,
title=title,
ylabel='duration',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
doc.write_file(
path.join(self._docs_folder,
to_safe_name(self._complete_name() + date + ".html")))
print('done')
def _publish_results(self):
"""Plot and optionally save the results."""
doc = Document()
date = get_stamp()
labels = ExperimentTemplateBase.parameters_to_string(self._topology_parameters_list)
title = f'FPS vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._fps, 'FPS',
labels, date, self._docs_folder, doc, smoothing_window_size=self._smoothing_window_size)
smoothing = len(self._fps[0]) // 100 # adaptive smoothing
if smoothing % 2 == 0:
smoothing += 1 # make odd
title = 'FPS vs. ' + self._experiment_name + f' (smoothing {smoothing})'
self.plot_save(title, self._fps, 'FPS', labels, date, self._docs_folder, doc,
smoothing_window_size=smoothing)
title = f'max_memory_allocated() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._max_mem, 'Max memory', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
title = f'memory_allocated() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._current_mem, 'Current memory', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
title = f'max_cached() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._max_cached, 'Max cached mem', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
title = f'memory_cached() vs. {self._experiment_name} (smoothing {self._smoothing_window_size})'
self.plot_save(title, self._current_cached, 'Current cached mem', labels, date, self._docs_folder, doc,
smoothing_window_size=self._smoothing_window_size)
self._publish_aux_results(labels, date, self._docs_folder, doc)
doc.write_file(path.join(self._docs_folder, f"{self._topology_class.__name__}_" + date + ".html"))
print('done')
def run_experiment_with_ui(experiment: Experiment, auto_start: bool = True):
with observer_system_context() as observer_system:
log_filename = experiment.setup_logging_path(get_stamp())
setup_logging_ui(LogObservable(), observer_system, log_filename)
runner = UiExperimentRunner(observer_system)
experiment.run(runner, auto_start=auto_start)
def run_experiment(experiment: Experiment):
setup_logging_no_ui(filename=experiment.setup_logging_path(get_stamp()))
runner = BasicExperimentRunner()
experiment.run(runner)
def _publish_results(self):
"""Plot and save the results."""
doc = Document()
date = get_stamp()
labels = ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list)
for manager in self._layer_measurement_managers:
manager.publish_results(labels=labels,
date=date,
docs_folder=self._docs_folder,
doc=doc)
# plot the running MSE
title = 'Mean Square Error of TA classification'
f = plot_multiple_runs_with_baselines(self._steps,
self._predicted_labels_mse,
self._baseline_labels_mse,
title=title,
ylabel='mse',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'TA classification accuracy'
f = plot_multiple_runs_with_baselines(
self._steps,
self._classification_accuracy,
self._random_classification_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'TA classification accuracy - (SE-metric)'
f = plot_multiple_runs_with_baselines(
self._steps,
self._se_classification_accuracy,
self._se_random_classification_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
title = 'is_learning'
f = plot_multiple_runs(
self._steps,
self._is_learning_info,
title=title,
ylabel='is learning',
xlabel='steps',
)
add_fig_to_doc(f, path.join(self._docs_folder, title), doc)
doc.write_file(
path.join(self._docs_folder,
to_safe_name(self._complete_name() + date + ".html")))
print('done')
def _publish_results(self):
doc = Document()
xlabel = "steps"
ylabel = "number of SP forward executions"
title_fe_dt = "smoothed derivation of SP forward executions and TP forward executions (SP O QD)"
figsize = (18, 12)
date = get_stamp()
layer_names = ['L0', 'L1', 'L2']
nr_layers = len(layer_names)
labels = list(map(lambda x: x + " forward execution", layer_names)) + \
list(map(lambda x: x + " qualitative difference", layer_names))
colors = ['b', 'orange', 'g', 'r', 'p']
color_params = [{'c': color} for color in colors[:nr_layers]]
color_ls_params = [{
'c': color,
'ls': '--'
} for color in colors[:nr_layers]]
other_params = color_params + color_ls_params
params_description = ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list)
for run in range(len(self.sp_executions)):
sp_execution_dt = list(
map(compute_derivations, self.sp_executions[run]))
sp_output_dt = self.sp_output_stability[run]
title = title_fe_dt + f" run {run} "
fig = plot_multiple_runs(x_values=self.training_steps,
y_values=sp_execution_dt + sp_output_dt,
ylim=[0, 1],
labels=labels,
smoothing_window_size=501,
xlabel=xlabel,
ylabel=ylabel,
title=title + params_description[run],
figsize=figsize,
hide_legend=False,
other_params=other_params)
add_fig_to_doc(fig,
path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "classification accuracy from reconstructed labels"
fig = plot_multiple_runs(x_values=list(range(
self._num_testing_phases)),
y_values=self.classification_accuracy,
ylim=[0, 1],
labels=params_description,
smoothing_window_size=None,
xlabel="accuracy",
ylabel="phases",
title=title,
figsize=figsize,
hide_legend=False)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "SE classification accuracy from reconstructed labels"
fig = plot_multiple_runs(x_values=list(range(
self._num_testing_phases)),
y_values=self.classification_accuracy_se,
ylim=[0, 1],
labels=params_description,
smoothing_window_size=None,
xlabel="SE accuracy",
ylabel="phases",
title=title,
figsize=figsize,
hide_legend=False)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
doc.write_file(path.join(self._docs_folder, f"main.html"))
def _publish_results(self):
"""Plot and optionally save the results."""
mse = np.array(self._mse)
mse_testing = np.array(self._mse_testing)
memory_used = torch.tensor(self._memory_used, dtype=torch.float32)
window_size = 201
memory_used = (memory_used.view(-1, 1).expand(mse_testing.shape) /
(1024**2))
error_memory_ratio = torch.tensor(mse_testing,
dtype=torch.float32) * memory_used
accuracy_memory_ratio = []
for run_acc, run_mem in zip(self._weak_classifier_results,
memory_used):
accuracy_memory_ratio_run = []
for acc, mem in zip(run_acc, run_mem):
accuracy_memory_ratio_run.append((1 - acc) * mem)
accuracy_memory_ratio.append(accuracy_memory_ratio_run)
accuracy_memory_ratio = torch.tensor(accuracy_memory_ratio)
doc = Document()
figs = []
xlabel = "steps"
ylabel = "mean reconstruction error"
title = "Influence of hyperparameters on reconstruction error (training)"
figsize = (18, 12)
date = get_stamp()
fig = plot_multiple_runs(
x_values=np.arange(len(mse[0])),
y_values=mse,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
smoothing_window_size=window_size,
xlabel=xlabel,
ylabel=ylabel,
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
fig = plot_multiple_runs(
x_values=np.arange(len(mse[0])),
y_values=mse,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=ylabel,
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(
fig, path.join(self._docs_folder,
to_safe_name(title) + "_smooth"), doc)
title = "Influence of hyperparameters on reconstruction error (testing)"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=mse_testing,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=ylabel,
title=title,
figsize=figsize,
)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Weak classifier accuracy"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=np.array(self._weak_classifier_results),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="steps",
ylabel="accuracy",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Memory * reconstruction tradeoff (testing)"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=error_memory_ratio.numpy(),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=
"Mean Reconstruction Error times required meogabyte of memory",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Memory * error (= 1 - acc) tradeoff"
fig = plot_multiple_runs(
x_values=np.array(self._training_steps_before_testing[0]),
y_values=accuracy_memory_ratio.numpy(),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel=xlabel,
ylabel=
"Mean Reconstruction Error times required meogabyte of memory",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Entropy and mean reconstruction error"
fig = plot_multiple_runs(
x_values=np.array(self._code_entropy),
y_values=mse_testing,
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="entropy",
ylabel="mean reconstruction error",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Mean reconstruction error and classifier accuracy"
fig = plot_multiple_runs(
x_values=mse_testing,
y_values=np.array(self._weak_classifier_results),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="mean reconstruction error",
ylabel="accuracy",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
title = "Entropy and classifier accuracy"
fig = plot_multiple_runs(
x_values=np.array(self._code_entropy),
y_values=np.array(self._weak_classifier_results),
labels=ExperimentTemplateBase.parameters_to_string(
self._topology_parameters_list),
xlabel="entropy",
ylabel="accuracy",
title=title,
figsize=figsize,
hide_legend=True)
add_fig_to_doc(fig, path.join(self._docs_folder, to_safe_name(title)),
doc)
doc.write_file(
path.join(self._docs_folder,
f"{self._topology_class.__name__}_" + date + ".html"))
print(self._memory_used)