def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_cm = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
if self._save_image:
class_order = self._get_display_class_order(exp_cm, strategy)
cm_image = self._image_creator(
exp_cm[class_order][:, class_order],
class_order
)
metric_representation = MetricValue(
self, metric_name, AlternativeValues(cm_image, exp_cm),
plot_x_position)
else:
metric_representation = MetricValue(
self, metric_name, exp_cm, plot_x_position)
return [metric_representation]
def _package_result(self, strategy: 'BaseStrategy'):
metric_value = self.result()
metric_name = get_metric_name(self, strategy)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_cpu = self.result()
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, exp_cpu, plot_x_position)]
def test_publish_metric(self):
ep = EvaluationPlugin()
mval = MetricValue(self, "metric", 1.0, 0)
ep.publish_metric_value(mval)
# check key exists
assert len(ep.get_all_metrics()["metric"][1]) == 1
def _package_result(self, strategy: 'PluggableStrategy') -> MetricResult:
exp_time = self.result()
metric_name = get_metric_name(self, strategy)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, exp_time, plot_x_position)]
def _package_result(self,
strategy: "SupervisedTemplate") -> "MetricResult":
self.steps.append(self.global_it_counter)
task2label2count = self.labels_repartition.result()
for task, label2count in task2label2count.items():
for label, count in label2count.items():
self.task2label2counts[task].setdefault(
label, [0] * (len(self.steps) - 2)).extend((count, count))
for task, label2counts in self.task2label2counts.items():
for label, counts in label2counts.items():
counts.extend([0] * (len(self.steps) - len(counts)))
return [
MetricValue(
self,
name=f"Repartition"
f"/{self._mode}_phase"
f"/{stream_type(strategy.experience)}_stream"
f"/Task_{task:03}",
value=AlternativeValues(
self.image_creator(label2counts, self.steps),
label2counts,
) if self.image_creator is not None else label2counts,
x_plot=strategy.clock.train_iterations,
) for task, label2counts in self.task2label2counts.items()
]
def _package_result(self, strategy: 'PluggableStrategy') -> MetricResult:
ram_usage = self.result()
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, ram_usage, plot_x_position)]
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_time = self.result()
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, exp_time, plot_x_position)]
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
metric_value = self.result()
metric_name = get_metric_name(self, strategy)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def _package_result(self, strategy: 'PluggableStrategy') -> 'MetricResult':
phase_name, _ = phase_and_task(strategy)
experience_gpu = self.result()
metric_name = 'GPU_usage/{}'.format(phase_name)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, experience_gpu, plot_x_position)]
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
average_epoch_time = self.result()
metric_name = get_metric_name(self, strategy)
plot_x_position = self.get_global_counter()
return [
MetricValue(self, metric_name, average_epoch_time, plot_x_position)
]
def _package_result(self, strategy: "BaseStrategy") -> MetricResult:
metric_value = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = "{}/{}_phase/{}_stream".format(str(self), phase_name,
stream)
plot_x_position = strategy.clock.train_iterations
return [MetricValue(self, metric_name, metric_value, plot_x_position)]
def _package_result(self, strategy: "BaseStrategy") -> "MetricResult":
label_cat2mean_score: Dict[LabelCat, float] = self.result()
for label_cat, m in label_cat2mean_score.items():
self.label_cat2step2mean[label_cat][self.global_it_counter] = m
base_metric_name = get_metric_name(
self, strategy, add_experience=False, add_task=False
)
rv = [
MetricValue(
self,
name=base_metric_name + f"/{label_cat}_classes",
value=m,
x_plot=self.global_it_counter,
)
for label_cat, m in label_cat2mean_score.items()
]
if "old" in label_cat2mean_score and "new" in label_cat2mean_score:
rv.append(
MetricValue(
self,
name=base_metric_name + f"/new_old_diff",
value=label_cat2mean_score["new"]
- label_cat2mean_score["old"],
x_plot=self.global_it_counter,
)
)
if self.image_creator is not None:
rv.append(
MetricValue(
self,
name=base_metric_name,
value=AlternativeValues(
self.image_creator(self.label_cat2step2mean),
self.label_cat2step2mean,
),
x_plot=self.global_it_counter,
)
)
return rv
def _package_result(self, strategy) -> "MetricResult":
weights = self.result()
metric_name = get_metric_name(
self, strategy, add_experience=True, add_task=False
)
return [
MetricValue(
self, metric_name, weights, strategy.clock.train_iterations
)
]
def _package_result(self, strategy: "BaseStrategy") -> MetricResult:
shifting = self.result(k=self.eval_exp_id)
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = self.get_global_counter()
metric_values = [
MetricValue(self, metric_name, shifting, plot_x_position)
]
return metric_values
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_cm = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self._next_x_position(metric_name)
if self._save_image:
cm_image = self._image_creator(exp_cm)
metric_representation = MetricValue(
self, metric_name, AlternativeValues(cm_image, exp_cm),
plot_x_position)
else:
metric_representation = MetricValue(
self, metric_name, exp_cm, plot_x_position)
return [metric_representation]
def _package_result(self, strategy: 'PluggableStrategy') \
-> MetricResult:
forgetting = self.result()
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = self._next_x_position(metric_name)
metric_values = [
MetricValue(self, metric_name, forgetting, plot_x_position)
]
return metric_values
def _package_result(self, strategy: 'PluggableStrategy') -> MetricResult:
ram_usage = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self._next_x_position(metric_name)
return [MetricValue(self, metric_name, ram_usage, plot_x_position)]
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
exp_disk = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
return [MetricValue(self, metric_name, exp_disk, plot_x_position)]
def _package_result(self, strategy: "BaseStrategy") -> MetricResult:
# Only after the previous experience was trained on can we return the
# forward transfer metric for this experience.
result = self.result(k=self.eval_exp_id)
if result is not None:
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = strategy.clock.train_iterations
metric_values = [
MetricValue(self, metric_name, result, plot_x_position)
]
return metric_values
def _package_result(self, strategy: 'BaseStrategy') -> \
MetricResult:
metric_value = self.result()
plot_x_position = self.get_global_counter()
results = []
for k, v in metric_value.items():
metric_name = get_metric_name(self,
strategy,
add_experience=False,
add_task=k)
results.append(MetricValue(self, metric_name, v, plot_x_position))
return results
def _package_result(self, strategy: "SupervisedTemplate") -> MetricResult:
# this checks if the evaluation experience has been
# already encountered at training time
# before the last training.
# If not, forgetting should not be returned.
forgetting = self.result(k=self.eval_exp_id)
if forgetting is not None:
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = strategy.clock.train_iterations
metric_values = [
MetricValue(self, metric_name, forgetting, plot_x_position)
]
return metric_values
def _package_result(self, strategy: 'BaseStrategy') \
-> MetricResult:
# this checks if the evaluation experience has been
# already encountered at training time
# before the last training.
# If not, forgetting should not be returned.
forgetting = self.result(k=self.eval_exp_id)
if forgetting is not None:
metric_name = get_metric_name(self, strategy, add_experience=True)
plot_x_position = self.get_global_counter()
metric_values = [
MetricValue(self, metric_name, forgetting, plot_x_position)
]
return metric_values
def _make_grid_sample(self, strategy: "BaseStrategy") -> "MetricResult":
self._load_sorted_images(strategy)
return [
MetricValue(
self,
name=get_metric_name(
self,
strategy,
add_experience=self.mode == "eval",
add_task=True,
),
value=TensorImage(
make_grid(list(self.images),
normalize=False,
nrow=self.n_cols)),
x_plot=strategy.clock.train_iterations,
)
]
def _package_result(self, strategy):
base_metric_name = get_metric_name(self,
strategy,
add_experience=True,
add_task=False)
plot_x_position = strategy.clock.train_iterations
result_dict = self.result()
if result_dict is None:
return
metric_values = []
for iou, iou_dict in result_dict.items():
for metric_key, metric_value in iou_dict.items():
metric_name = base_metric_name + f"/{iou}/{metric_key}"
metric_values.append(
MetricValue(self, metric_name, metric_value,
plot_x_position))
return metric_values
def _package_result(self, strategy: 'BaseStrategy') -> MetricResult:
outputs, targets = self.result()
phase_name, _ = phase_and_task(strategy)
stream = stream_type(strategy.experience)
metric_name = '{}/{}_phase/{}_stream' \
.format(str(self),
phase_name,
stream)
plot_x_position = self.get_global_counter()
# compute predicted classes
preds = torch.argmax(outputs, dim=1).cpu().numpy()
result = wandb.plot.confusion_matrix(preds=preds,
y_true=targets.cpu().numpy(),
class_names=self.class_names)
metric_representation = MetricValue(
self, metric_name, AlternativeValues(result),
plot_x_position)
return [metric_representation]
def _package_result(self, strategy) -> 'MetricResult':
weights = self.result()
metric_name = get_metric_name(self, strategy,
add_experience=True, add_task=False)
return [MetricValue(self, metric_name, weights,
self.get_global_counter())]
