def after_eval_exp(
self,
strategy: "SupervisedTemplate",
metric_values: List["MetricValue"],
**kwargs,
):
super().after_eval_exp(strategy, metric_values, **kwargs)
exp_id = strategy.experience.current_experience
task_id = phase_and_task(strategy)[1]
if task_id is None:
print(
f"> Eval on experience {exp_id} "
f"from {stream_type(strategy.experience)} stream ended.",
file=self.file,
flush=True,
)
else:
print(
f"> Eval on experience {exp_id} (Task "
f"{task_id}) "
f"from {stream_type(strategy.experience)} stream ended.",
file=self.file,
flush=True,
)
self.print_current_metrics()
self.metric_vals = {}
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_cm = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
if self._save_image:
class_order = self._get_display_class_order(exp_cm, strategy)
cm_image = self._image_creator(
exp_cm[class_order][:, class_order],
class_order
)
metric_representation = MetricValue(
self, metric_name, AlternativeValues(cm_image, exp_cm),
plot_x_position)
else:
metric_representation = MetricValue(
self, metric_name, exp_cm, plot_x_position)
return [metric_representation]
def _package_result(self, strategy: 'PluggableStrategy') -> 'MetricResult':
phase_name, _ = phase_and_task(strategy)
experience_gpu = self.result()
metric_name = 'GPU_usage/{}'.format(phase_name)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, experience_gpu, plot_x_position)]
def _package_result(self, strategy: "BaseStrategy") -> MetricResult:
metric_value = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = "{}/{}_phase/{}_stream".format(str(self), phase_name,
stream)
plot_x_position = strategy.clock.train_iterations
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def _package_result(self, strategy: 'PluggableStrategy') -> MetricResult:
ram_usage = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, ram_usage, plot_x_position)]
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_disk = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, exp_disk, plot_x_position)]
def _on_exp_start(self, strategy: 'BaseStrategy'):
action_name = 'training' if strategy.is_training else 'eval'
exp_id = strategy.experience.current_experience
task_id = phase_and_task(strategy)[1]
stream = stream_type(strategy.experience)
if task_id is None:
print('-- Starting {} on experience {} from {} stream --'.format(
action_name, exp_id, stream),
file=self.file,
flush=True)
else:
print(
'-- Starting {} on experience {} (Task {}) from {} stream --'.
format(action_name, exp_id, task_id, stream),
file=self.file,
flush=True)
def _on_exp_start(self, strategy: "SupervisedTemplate"):
action_name = "training" if strategy.is_training else "eval"
exp_id = strategy.experience.current_experience
task_id = phase_and_task(strategy)[1]
stream = stream_type(strategy.experience)
if task_id is None:
print(
"-- Starting {} on experience {} from {} stream --".format(
action_name, exp_id, stream),
file=self.file,
flush=True,
)
else:
print(
"-- Starting {} on experience {} (Task {}) from {}"
" stream --".format(action_name, exp_id, task_id, stream),
file=self.file,
flush=True,
)
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_cm = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self._next_x_position(metric_name)
if self._save_image:
cm_image = self._image_creator(exp_cm)
metric_representation = MetricValue(
self, metric_name, AlternativeValues(cm_image, exp_cm),
plot_x_position)
else:
metric_representation = MetricValue(
self, metric_name, exp_cm, plot_x_position)
return [metric_representation]
def after_eval_exp(self, strategy: 'BaseStrategy',
metric_values: List['MetricValue'], **kwargs):
super().after_eval_exp(strategy, metric_values, **kwargs)
exp_id = strategy.experience.current_experience
task_id = phase_and_task(strategy)[1]
if task_id is None:
print(
f'> Eval on experience {exp_id} '
f'from {stream_type(strategy.experience)} stream ended.',
file=self.file,
flush=True)
else:
print(
f'> Eval on experience {exp_id} (Task '
f'{task_id}) '
f'from {stream_type(strategy.experience)} stream ended.',
file=self.file,
flush=True)
self.print_current_metrics()
self.metric_vals = {}
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
outputs, targets = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
# compute predicted classes
preds = torch.argmax(outputs, dim=1).cpu().numpy()
result = wandb.plot.confusion_matrix(preds=preds,
y_true=targets.cpu().numpy(),
class_names=self.class_names)
metric_representation = MetricValue(
self, metric_name, AlternativeValues(result),
plot_x_position)
return [metric_representation]
def result(self, strategy=None) -> float:
if self._emit_at == "stream" or strategy is None:
return self._metric.result()
else:
return self._metric.result(phase_and_task(strategy)[1])
def reset(self, strategy=None) -> None:
if self._reset_at == "stream" or strategy is None:
self._metric.reset()
else:
self._metric.reset(phase_and_task(strategy)[1])
