def run(self, runner: ExperimentRunner, auto_start: bool = True):
"""A generator which performs one step of the experiment per iteration.
One step here means one simulation step of the topology of a single experiment run. If no run exists, the next
run in the queue of all runs is created and a step is done there. If all single experiment runs were already
finished, the simulation is stopped and the results are generated.
"""
self._timestamp_start = get_stamp()
self.measurement_manager = MeasurementManager(
self._experiment_folder, self._measurement_manager_params)
if self._params.clear_cache:
self.measurement_manager.clear_cache()
elif self._params.load_cache:
# This can only happen in the cache is not cleared.
self.measurement_manager.load_cache_index()
garbage_collect(
) # empty the memory from previous experiments so that they do not influence this one
logger.info(
f"Starting experiment with {len(self._topology_parameters)} runs")
self._run_all(runner, auto_start)
if not self._params.calculate_statistics:
# No results get published.
return
self.results_path = self.produce_results()
def test_measurement_manager():
manager = MeasurementManager(None, MeasurementManagerParams())
run = manager.create_new_run('foo model', {'param': 'value'})
run._measurements.add(1, 'measurement', 42)
manager.add_results(run)
assert 42 == manager.single_run_measurements[0].get_item('measurement',
step=1)
def publish_results(cls, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
experiment_params = measurement_manager.single_run_measurements[
0].model_parameters['params']['experiment_params']
gl_params = cast(GLExperimentParams, experiment_params)
not_forgetting_params = cast(NotForgettingExperimentParams,
gl_params.params)
start_of_testing_phase = not_forgetting_params.phase_1_steps + not_forgetting_params.phase_2_steps
start_of_testing_phase -= 1000 # start 500 steps backwards to initialize moving average
steps = np.array(
measurement_manager.get_values_from_all_runs('current_step'))[
0, start_of_testing_phase:]
sequence_ids = np.array(
measurement_manager.get_values_from_all_runs(
DATASET_2_SEQUENCE_ID))[0, start_of_testing_phase:]
accuracy_per_flock_1_single = np.array(
measurement_manager.get_values_from_all_runs(
ACCURACY_PER_FLOCK_SINGLE_1))[0, start_of_testing_phase:]
accuracy_per_flock_2_single = np.array(
measurement_manager.get_values_from_all_runs(
ACCURACY_PER_FLOCK_SINGLE_2))[0, start_of_testing_phase:]
labels = topology_parameters
# title = 'sequence_id'
# f = plot_multiple_runs(
# np.expand_dims(steps, axis=0),
# np.expand_dims(sequence_id, axis=0),
# title=title,
# ylabel='sequence_id',
# xlabel='steps',
# labels=labels
# )
# add_fig_to_doc(f, os.path.join(docs_folder, title), document)
d1 = np.mean(accuracy_per_flock_1_single, axis=-1)
d2 = np.mean(accuracy_per_flock_2_single, axis=-1)
plot_testing_accuracy(steps,
d1,
d2,
window_size=180,
name='accuracy_testing_1_2',
trim_size=180,
labels=labels,
document=document,
docs_folder=docs_folder)
seq_filter = (sequence_ids == 2) | (sequence_ids == 5)
def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
steps = measurement_manager.get_values_from_all_runs('current_step')
labels = topology_parameters
multi_config_rewards = measurement_manager.get_custom_data_from_all_runs(
'average_rewards')
for single_config_multi_run_rewards in multi_config_rewards:
num_runs = len(single_config_multi_run_rewards[0])
single_runs = [[] for _ in range(num_runs)]
for timestep in single_config_multi_run_rewards:
for i in range(num_runs):
single_runs[i].append(timestep[i])
max_value = max(max(run) for run in single_runs)
for i, timeseries in enumerate(single_runs):
title = f"average_reward_run_{i}"
plot_multiple_runs(steps,
timeseries,
ylabel=f"average_reward_run_{i}",
xlabel="steps",
ylim=[0, max_value],
labels=labels,
title=title,
path=path.join(docs_folder, title),
doc=document)
means = []
mins = []
maxes = []
for timestep in single_config_multi_run_rewards:
means.append(np.mean(timestep))
mins.append(np.min(timestep))
maxes.append(np.max(timestep))
title = "all_average_rewards"
plot_multiple_runs(steps,
means,
y_lower=mins,
y_upper=maxes,
ylabel="average_reward",
xlabel="steps",
labels=labels,
title=title,
path=path.join(docs_folder, title),
doc=document)
def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
document.add(Heading(self.template_params.experiment_name))
for title in measurement_manager.single_run_measurements[
0].get_item_names():
if title not in [
"initial training error", "initial testing error",
"untraining error", "retraining error", "testing error"
]:
continue
data = measurement_manager.get_values_from_all_runs(title)
n_runs = len(data)
if self.template_params.every_second_is_baseline: # TODO MS
labels = (n_runs // 2) * ["experiment", "baseline"]
else:
labels = n_runs * [""]
plot_multiple_runs(list(range(len(data[0]))),
data,
ylabel="error",
xlabel="steps",
labels=labels,
title=title,
smoothing_window_size=21,
path=path.join(docs_folder, title),
doc=document)
def test_state_corner_cases(self):
observer_system = ObserverSystemVoid()
topology_parameters = {'device': 'cpu'}
params = ExperimentParams(max_steps=0)
template = BasicExperimentTemplate("Template")
topology_factory = BasicTopologyFactory(Topology)
measurement_manager = MeasurementManager(None, None)
manager = SingleExperimentManager(template, topology_factory,
topology_parameters,
measurement_manager,
params.create_run_params())
run = manager.create_run()
simulation = UiExperimentRunner(observer_system)
simulation.init_run(run)
simulation.start()
simulation.start()
simulation.pause()
simulation.pause()
simulation.stop()
simulation.stop()
simulation.pause()
simulation.pause()
assert RunState.STOPPED == simulation._state
simulation.step()
simulation.step()
simulation.start()
simulation.stop()
def _create_basic_run_manager(topology_factory: Callable[..., Topology],
topology_params: Dict[str, Any],
seed: Optional[int] = None,
max_steps: int = 0,
save_model_after_run: bool = True):
template = BasicExperimentTemplate("Template")
topology_factory = BasicTopologyFactory(topology_factory)
measurement_manager = MeasurementManager(None, None)
run_params = ExperimentParams(max_steps,
seed=seed,
calculate_statistics=False,
save_models_after_run=save_model_after_run)
return SingleExperimentManager(template, topology_factory, topology_params,
measurement_manager,
run_params.create_run_params())
def test_train_test_experiment():
topology_params = {'device': 'cpu'}
train_test_params = TrainTestComponentParams(num_testing_phases=2,
num_testing_steps=2,
overall_training_steps=4)
run_params = SingleExperimentRunParams(
load_cache=False,
save_cache=False,
calculate_statistics=True,
max_steps=train_test_params.max_steps,
save_model_after_run=False)
template = TemplateStub(train_test_params)
topology_factory = BasicTopologyFactory(
lambda device: TrainTestTopologyStub(device))
measurement_manager = MeasurementManager(None, None)
# single_run = SingleExperimentManager(topology, controller, topology_params, run_measurement_manager, run_params)
single_run = SingleExperimentManager(template, topology_factory,
topology_params, measurement_manager,
run_params)
runner = BasicExperimentRunner()
runner.run(single_run.create_run())
run_measurements = measurement_manager.single_run_measurements[0]
assert all(len(item) > 0 for item in run_measurements._values)
partitioning = TrainTestMeasurementPartitioning(run_measurements)
training_phases = partitioning.partition_to_training_phases('item')
training_phases_model = partitioning.partition_to_training_phases(
'model_item')
testing_phases = partitioning.partition_to_testing_phases('item')
testing_phases_model = partitioning.partition_to_testing_phases(
'model_item')
assert [(0, [(0, 1), (1, 2)]), (1, [(4, 5), (5, 6)])] == training_phases
assert [(0, [(0, 1), (1, 1)]), (1, [(4, 1),
(5, 1)])] == training_phases_model
assert [(0, [(2, 3), (3, 4)]), (1, [(6, 7), (7, 8)])] == testing_phases
assert [(0, [(2, 2), (3, 2)]), (1, [(6, 2),
(7, 2)])] == testing_phases_model
def test_state(self, caplog):
caplog.set_level(logging.DEBUG)
observer_system = ObserverSystemVoid()
topology_crashing = False
def should_crash():
return topology_crashing
topology_params = {'should_crash': should_crash}
params = ExperimentParams(max_steps=0)
# single_run = SingleExperimentManager(TopologyStub(should_crash), controller, topology_params,
# run_measurement_manager, params.create_run_params())
template = BasicExperimentTemplate("Template")
topology_factory = BasicTopologyFactory(TopologyStub)
measurement_manager = MeasurementManager(None, None)
manager = SingleExperimentManager(template, topology_factory,
topology_params, measurement_manager,
params.create_run_params())
single_run = manager.create_run()
runner = UiExperimentRunner(observer_system)
runner.init_run(single_run)
assert RunState.STOPPED == runner._state
runner.start()
assert RunState.RUNNING == runner._state
runner.pause()
assert RunState.PAUSED == runner._state
runner.start()
assert RunState.RUNNING == runner._state
runner.stop()
assert RunState.STOPPED == runner._state
topology_crashing = True
runner.step()
assert RunState.CRASHED == runner._state
runner.wait()
def publish_results_for_layer(document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str],
layer_id: int, num_layers: int,
show_conv_agreements: bool,
is_train_test_classifier_computed: bool):
"""Publish results for each layer separately
This uses the data measured and computed for each run by the Task0TaAnalysisLayerComponent and aggregated
and stored in the measurement_manager.
"""
logger.info(f'publishing results for layer {layer_id}...')
num_boosted_clusters = measurement_manager.get_values_from_all_runs(
f'num_boosted_getter_{layer_id}')
average_boosting_dur = measurement_manager.get_values_from_all_runs(
f'average_boosting_dur_{layer_id}')
average_deltas = measurement_manager.get_values_from_all_runs(
f'average_delta_{layer_id}')
base_weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs(
f'base_weak_class_accuracy_{layer_id}')
clustering_agreements = measurement_manager.get_custom_data_from_all_runs(
f'clustering_agreements_{layer_id}')
average_steps_deltas = measurement_manager.get_items_from_all_runs(
f'average_delta_{layer_id}')
sp_evaluation_steps = [key for key, value in average_steps_deltas[0]]
labels = topology_parameters
document.add(f"<br><br><br><b>Results for layer {layer_id}</b><br>")
prefix = 'L' + str(layer_id) + '--'
testing_phase_ids = list(range(0, len(
clustering_agreements[0][0]))) # x-axis values
if is_train_test_classifier_computed:
weak_class_accuracy_train = measurement_manager.get_custom_data_from_all_runs(
f'weak_class_accuracy_train_{layer_id}')
weak_class_accuracy_test = measurement_manager.get_custom_data_from_all_runs(
f'weak_class_accuracy_test_{layer_id}')
title = prefix + ' Weak classifier accuracy (trained on train, tested on train data)'
plot_multiple_runs_with_baselines(testing_phase_ids,
weak_class_accuracy_train,
base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(
docs_folder, title),
doc=document)
title = prefix + ' Weak classifier accuracy (trained on train, tested on test data)'
plot_multiple_runs_with_baselines(testing_phase_ids,
weak_class_accuracy_test,
base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(
docs_folder, title),
doc=document)
weak_class_accuracy = measurement_manager.get_custom_data_from_all_runs(
f'weak_class_accuracy_{layer_id}')
title = prefix + ' Weak classifier accuracy (trained on test, tested on test)'
plot_multiple_runs_with_baselines(testing_phase_ids,
weak_class_accuracy,
base_weak_class_accuracy,
title=title,
ylabel='Accuracy (1 ~ 100%)',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
title = prefix + '- Average boosting duration'
plot_multiple_runs(sp_evaluation_steps,
average_boosting_dur,
title=title,
ylabel='duration',
xlabel='steps',
labels=labels,
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
title = prefix + '- Num boosted clusters'
plot_multiple_runs(sp_evaluation_steps,
num_boosted_clusters,
title=title,
ylabel='Num boosted clusters / total clusters',
xlabel='steps',
labels=labels,
ylim=[-0.1, 1.1],
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
title = prefix + '- Average_deltas'
plot_multiple_runs(
sp_evaluation_steps,
average_deltas,
title=title,
ylabel='average_deltas',
xlabel='steps',
labels=labels,
disable_ascii_labels=True,
hide_legend=True,
# use_scatter=True,
path=path.join(docs_folder, title),
doc=document)
# if this is not the top layer, show conv agreements only if required
if show_conv_agreements or layer_id == (num_layers - 1):
agreements = clustering_agreements
for run_id, run_agreements in enumerate(agreements):
for expert_id, expert_agreements in enumerate(run_agreements):
Task0TaAnalysisTemplate._plot_agreement(
prefix, expert_id, run_id, testing_phase_ids,
expert_agreements, document, docs_folder)
logger.info('done')
def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
"""An alternative to the _publish_results method, this is called from _publish_results now
Draw and add your plots to the document here.
"""
steps = measurement_manager.get_values_from_all_runs('current_step')
plotted_training_phase_id = measurement_manager.get_values_from_all_runs(
'training_phase_id')
plotted_testing_phase_id = measurement_manager.get_values_from_all_runs(
'testing_phase_id')
plotted_is_learning = measurement_manager.get_values_from_all_runs(
'is_learning')
labels = topology_parameters
document.add(
f"<br><br><br><b>Common results from the TopExpert</b><br>")
title = 'training_phase_id'
plot_multiple_runs(steps,
plotted_training_phase_id,
title=title,
ylabel='training_phase_id',
xlabel='steps',
labels=labels,
path=path.join(docs_folder, title),
doc=document)
title = 'testing_phase_id'
plot_multiple_runs(steps,
plotted_testing_phase_id,
title=title,
ylabel='testing_phase_id',
xlabel='steps',
labels=labels,
path=path.join(docs_folder, title),
doc=document)
title = 'is_learning'
plot_multiple_runs(steps,
plotted_is_learning,
title=title,
ylabel='is_learning',
xlabel='steps',
labels=labels,
path=path.join(docs_folder, title),
doc=document)
predicted_labels_mse = measurement_manager.get_custom_data_from_all_runs(
'predicted_labels_mse')
testing_phases_x = list(range(0, len(predicted_labels_mse[0])))
model_accuracy = measurement_manager.get_custom_data_from_all_runs(
'model_accuracy')
baseline_accuracy = measurement_manager.get_custom_data_from_all_runs(
'baseline_accuracy')
# plot the classification accuracy
title = 'Label reconstruction accuracy (step-wise)'
plot_multiple_runs_with_baselines(testing_phases_x,
model_accuracy,
baseline_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
model_se_accuracy = measurement_manager.get_custom_data_from_all_runs(
'model_se_accuracy')
baseline_se_accuracy = measurement_manager.get_custom_data_from_all_runs(
'baseline_se_accuracy')
# plot the classification accuracy
title = 'Label reconstruction SE accuracy (object-wise)'
plot_multiple_runs_with_baselines(testing_phases_x,
model_se_accuracy,
baseline_se_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing_phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True,
path=path.join(docs_folder, title),
doc=document)
baseline_labels_mse = measurement_manager.get_custom_data_from_all_runs(
'baseline_labels_mse')
# plot the MSE
title = 'Mean Square Error of label reconstruction'
plot_multiple_runs_with_baselines(
testing_phases_x,
predicted_labels_mse,
baseline_labels_mse,
title=title,
ylabel='MSE',
xlabel='testing phase ID',
labels=labels,
hide_legend=True,
ylim=[-0.1, 0.2], # just for better resolution
path=path.join(docs_folder, title),
doc=document)
for layer_id in reversed(range(self._experiment_params.num_layers)):
self.publish_results_for_layer(
document, docs_folder, measurement_manager,
topology_parameters, layer_id,
self._experiment_params.num_layers,
self._experiment_params.show_conv_agreements,
self._experiment_params.is_train_test_classifier_computed)
class Experiment(Generic[TTopology]):
"""A runner for a set of experiments."""
def __init__(self,
template: ExperimentTemplateBase,
topology_factory: TopologyFactory[TTopology],
topology_parameters: List[Dict[str, Any]],
params: ExperimentParams,
measurement_manager_params: Optional[
MeasurementManagerParams] = None):
"""Initializes the runner.
Args:
template: An experiment template.
topology_factory: A factory that creates a topology given one item from the topology_parameters list.
It also provides default parameter values.
topology_parameters: A set of parameters for the topologies that the experiment will run.
params: The parameters for this experiment runner.
measurement_manager_params: The parameters for the MeasurementManager.
"""
self._template = template
self._topology_factory = topology_factory
self._topology_parameters = topology_parameters
self._params = params
self._experiment_folder = self._get_experiment_folder_path(
self._params.experiment_folder)
self._measurement_manager_params = measurement_manager_params or MeasurementManagerParams(
)
self.measurement_manager = None
self._run_params = params.create_run_params()
self._init_seed(params.seed)
self._timestamp_start = None
# If not None, the log will be copied during publish_results().
self._log_file = None
# This stores the path to the results file produced by the whole experiment run.
self.results_path = None
@staticmethod
def _get_docs_folder_path(experiment_folder: str, experiment_name: str,
stamp: str):
return to_safe_path(
os.path.join(experiment_folder, "docs",
to_safe_name(experiment_name + stamp)))
def _get_experiment_folder_path(self, experiment_folder):
if experiment_folder is None:
return to_safe_path(
os.path.join(get_experiment_results_folder(),
to_safe_name(self._template.experiment_name)))
return experiment_folder
@staticmethod
def _get_default_topology_params(
topology_factory: Callable[..., TTopology]) -> Dict[str, Any]:
"""Get default params of the topology factory.
Returns: dictionary {param_name: param_default_value}
"""
my_params = dict(inspect.signature(topology_factory).parameters)
params_dict = dict(
(key, value.default)
for key, value in zip(my_params.keys(), list(my_params.values()))
if key is not 'self')
return params_dict
def _init_seed(self, seed: int):
"""Determines whether these measurements will be deterministic (across different runs)."""
self.rand = np.random.RandomState()
self.rand.seed(seed=seed)
set_global_seeds(seed)
def setup_logging_path(self, timestamp) -> str:
if not os.path.isdir(self._experiment_folder):
os.makedirs(self._experiment_folder)
self._log_file = os.path.join(self._experiment_folder,
f'run_{timestamp}.log')
return self._log_file
def run(self, runner: ExperimentRunner, auto_start: bool = True):
"""A generator which performs one step of the experiment per iteration.
One step here means one simulation step of the topology of a single experiment run. If no run exists, the next
run in the queue of all runs is created and a step is done there. If all single experiment runs were already
finished, the simulation is stopped and the results are generated.
"""
self._timestamp_start = get_stamp()
self.measurement_manager = MeasurementManager(
self._experiment_folder, self._measurement_manager_params)
if self._params.clear_cache:
self.measurement_manager.clear_cache()
elif self._params.load_cache:
# This can only happen in the cache is not cleared.
self.measurement_manager.load_cache_index()
garbage_collect(
) # empty the memory from previous experiments so that they do not influence this one
logger.info(
f"Starting experiment with {len(self._topology_parameters)} runs")
self._run_all(runner, auto_start)
if not self._params.calculate_statistics:
# No results get published.
return
self.results_path = self.produce_results()
def _run_all(self, runner: ExperimentRunner, auto_start: bool = True):
for topology_idx, topology_parameters in enumerate(
self._topology_parameters):
garbage_collect()
parameters_print = DocumentPublisher.parameters_to_string(
[topology_parameters])[0]
logger.info(
f'Creating run no. {topology_idx} with parameters: \n{parameters_print}'
)
single_experiment_manager = SingleExperimentManager(
self._template, self._topology_factory, topology_parameters,
self.measurement_manager, self._run_params, topology_idx)
if not single_experiment_manager.try_load():
run = single_experiment_manager.create_run()
runner.init_run(run)
if auto_start:
runner.start()
runner.wait()
def produce_results(self):
logger.info(f"Processing results")
default_topology_parameters = self._topology_factory.get_default_parameters(
)
docs_folder = self._params.docs_folder or self._get_docs_folder_path(
self._experiment_folder, self._template.experiment_name,
self._timestamp_start)
topology_parameters = [
dict(params) for params in self._topology_parameters
]
for params in topology_parameters:
# This is used to distinguish several runs with the same parameters if needed.
if '__id' in params:
del params['__id']
document_publisher = DocumentPublisher(self._template, docs_folder,
type(self._template),
default_topology_parameters,
topology_parameters,
self._params)
docs_file = document_publisher.publish_results(
self._timestamp_start, self.measurement_manager)
logger.info(f"Done")
if self._log_file is not None:
flush_logs()
log_file_destination = os.path.join(docs_folder, 'run.log')
copyfile(self._log_file, log_file_destination)
return docs_file
def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
"""An alternative to the _publish_results method, this is called from _publish_results now
Draw and add your plots to the document here.
"""
steps = measurement_manager.get_values_from_all_runs('current_step')
accuracy_1 = measurement_manager.get_values_from_all_runs(ACCURACY_1)
accuracy_per_flock_1 = measurement_manager.get_values_from_all_runs(
ACCURACY_PER_FLOCK_1)
accuracy_2 = measurement_manager.get_values_from_all_runs(ACCURACY_2)
accuracy_per_flock_2 = measurement_manager.get_values_from_all_runs(
ACCURACY_PER_FLOCK_2)
multiple_flocks_alpha = 0.05
# measurement_manager.single_run_measurements[0].get_custom_data('custom')
labels = topology_parameters
a1 = np.array(accuracy_1)
a2 = np.array(accuracy_2)
title = 'accuracy_1_2'
f = plot_multiple_runs(steps,
np.stack((a1, a2), axis=-1),
title=title,
ylabel='accuracy_1_2',
xlabel='steps',
labels=labels,
other_params=[{
'color': None
}])
add_fig_to_doc(f, os.path.join(docs_folder, title), document)
experiment_params = measurement_manager.single_run_measurements[
0].model_parameters['params']['experiment_params']
experiment_params.component.publish_results(document, docs_folder,
measurement_manager,
topology_parameters)
title = 'accuracy_per_flock_1'
f = plot_multiple_runs(steps,
accuracy_per_flock_1,
title=title,
ylabel='accuracy_per_flock_1',
xlabel='steps',
labels=labels,
hide_legend=True,
other_params=[{
'alpha': multiple_flocks_alpha
}])
add_fig_to_doc(f, os.path.join(docs_folder, title), document)
# title = 'accuracy_2'
# f = plot_multiple_runs(
# steps,
# accuracy_2,
# title=title,
# ylabel='accuracy_2',
# xlabel='steps',
# labels=labels
# )
# add_fig_to_doc(f, os.path.join(docs_folder, title), document)
title = 'accuracy_per_flock_2'
f = plot_multiple_runs(steps,
accuracy_per_flock_2,
title=title,
ylabel='accuracy_per_flock_2',
xlabel='steps',
labels=labels,
hide_legend=True,
other_params=[{
'alpha': multiple_flocks_alpha
}])
add_fig_to_doc(f, os.path.join(docs_folder, title), document)
# Add table with MSE single-step values
# prediction_mse_values = [f'{v:.5f}' for v in prediction_mse[0]]
# document.add_table(['step', 'prediction_mse'], list(zip(steps[0], prediction_mse_values)),
# attribs={'style': 'font-size:0.8em;'})
# for single_run_measurements in measurement_manager.single_run_measurements:
# self.publish_one_run(single_run_measurements, document, docs_folder, topology_parameters)
doc_path = os.path.join(docs_folder,
to_safe_name(self.experiment_name + ".html"))
if doc_path.startswith("\\\\?\\"):
doc_path = doc_path[len("\\\\?\\"):]
# Note not logging to UI now
logger.info(
f'Results published <a href="file:///{doc_path}">{doc_path}</a>')
logger.info(f'Results published {doc_path}')
def publish_results(self, document: Document, docs_folder: str,
measurement_manager: MeasurementManager,
topology_parameters: List[str]):
"""An alternative to the _publish_results method, this is called from _publish_results now
Draw and add your plots to the document here.
"""
steps = measurement_manager.get_values_from_all_runs('current_step')
plotted_training_phase_id = measurement_manager.get_values_from_all_runs(
'training_phase_id')
plotted_testing_phase_id = measurement_manager.get_values_from_all_runs(
'testing_phase_id')
labels = topology_parameters
title = 'training_phase_id'
f = plot_multiple_runs(steps,
plotted_training_phase_id,
title=title,
ylabel='training_phase_id',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(docs_folder, title), document)
title = 'testing_phase_id'
f = plot_multiple_runs(steps,
plotted_testing_phase_id,
title=title,
ylabel='testing_phase_id',
xlabel='steps',
labels=labels)
add_fig_to_doc(f, path.join(docs_folder, title), document)
max_x = max(steps[0])
average_delta_train_0 = measurement_manager.get_items_from_all_runs(
'average_delta0_train')
test_sp_steps = [key for key, value in average_delta_train_0[0]]
title = 'average_delta_train_layer0'
f = plot_multiple_runs(test_sp_steps,
[[value for key, value in sequence]
for sequence in average_delta_train_0],
title=title,
ylabel='average_deltas',
xlabel='steps',
labels=labels,
xlim=[0, max_x],
disable_ascii_labels=True,
hide_legend=True
# use_scatter=True
)
add_fig_to_doc(f, path.join(docs_folder, title), document)
average_delta_train_1 = measurement_manager.get_items_from_all_runs(
'average_delta1_train')
title = 'average_delta_train_layer1'
f = plot_multiple_runs(test_sp_steps,
[[value for key, value in sequence]
for sequence in average_delta_train_1],
title=title,
ylabel='average_deltas',
xlabel='steps',
labels=labels,
xlim=[0, max_x],
disable_ascii_labels=True,
hide_legend=True
# use_scatter=True
)
add_fig_to_doc(f, path.join(docs_folder, title), document)
predicted_labels_mse = measurement_manager.get_custom_data_from_all_runs(
'predicted_labels_mse')
testing_phases_x = list(range(0, len(predicted_labels_mse[0])))
model_accuracy = measurement_manager.get_custom_data_from_all_runs(
'model_accuracy')
baseline_accuracy = measurement_manager.get_custom_data_from_all_runs(
'baseline_accuracy')
# plot the classification accuracy
title = 'Label reconstruction accuracy (step-wise)'
f = plot_multiple_runs_with_baselines(testing_phases_x,
model_accuracy,
baseline_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True)
add_fig_to_doc(f, path.join(docs_folder, title), document)
model_se_accuracy = measurement_manager.get_custom_data_from_all_runs(
'model_se_accuracy')
baseline_se_accuracy = measurement_manager.get_custom_data_from_all_runs(
'baseline_se_accuracy')
# plot the classification accuracy
title = 'Label reconstruction SE accuracy (step-wise)'
f = plot_multiple_runs_with_baselines(testing_phases_x,
model_se_accuracy,
baseline_se_accuracy,
title=title,
ylabel='accuracy (1 ~ 100%)',
xlabel='testing_phase ID',
ylim=[-0.1, 1.1],
labels=labels,
hide_legend=True)
add_fig_to_doc(f, path.join(docs_folder, title), document)
baseline_labels_mse = measurement_manager.get_custom_data_from_all_runs(
'baseline_labels_mse')
# plot the MSE
title = 'Mean Square Error of label reconstruction'
f = plot_multiple_runs_with_baselines(
testing_phases_x,
predicted_labels_mse,
baseline_labels_mse,
title=title,
ylabel='MSE',
xlabel='testing phase ID',
labels=labels,
hide_legend=True,
ylim=[-0.1, 0.2] # just for better resolution
)
add_fig_to_doc(f, path.join(docs_folder, title), document)