def _log(self, state, prefix, predictions, targets):
score, num, denom = cohen_kappa_score(predictions, targets, weights='quadratic')
if self.class_names is None:
class_names = [str(i) for i in range(num.shape[1])]
else:
class_names = self.class_names
num_classes = len(class_names)
num_fig = plot_matrix(num / np.sum(num),
figsize=(6 + num_classes // 3, 6 + num_classes // 3),
class_names=class_names,
noshow=True)
denom_fig = plot_matrix(denom / np.sum(denom),
figsize=(6 + num_classes // 3, 6 + num_classes // 3),
class_names=class_names,
noshow=True)
num_fig = render_figure_to_tensor(num_fig)
denom_fig = render_figure_to_tensor(denom_fig)
logger = get_tensorboard_logger(state)
logger.add_image(f'{prefix}/epoch/num', num_fig, global_step=state.step)
logger.add_image(f'{prefix}/epoch/denom', denom_fig, global_step=state.step)
state.metrics.epoch_values[state.loader_name][prefix] = score
def on_loader_end(self, state: RunnerState):
eps = 1e-7
ious_per_image = []
for image_id, values in self.scores_per_image.items():
intersection = values["intersection"]
union = values["union"]
metric = intersection / (union - intersection + eps)
ious_per_image.append(metric)
iou = np.mean(ious_per_image, axis=0)
assert len(iou) == len(self.thresholds)
threshold_index = np.argmax(iou)
iou_at_threshold = iou[threshold_index]
threshold_value = self.thresholds[threshold_index]
state.metrics.epoch_values[state.loader_name][
self.prefix + "/" + "threshold"] = float(threshold_value)
state.metrics.epoch_values[state.loader_name][self.prefix] = float(
iou_at_threshold)
logger = get_tensorboard_logger(state)
logger.add_histogram(self.prefix, iou, global_step=state.epoch)
def on_loader_end(self, state: RunnerState):
logger = get_tensorboard_logger(state)
logger.add_embedding(mat=torch.cat(self.embeddings, dim=0),
metadata=self.targets,
label_img=torch.cat(self.images, dim=0),
global_step=state.epoch,
tag=self.prefix
)
def on_stage_end(self, state: IRunner):
logger = get_tensorboard_logger(state)
hparam_dict = self.hparam_dict.copy()
hparam_dict["stage"] = state.stage_name
logger.add_hparams(
hparam_dict=self.hparam_dict, metric_dict={"best_" + state.main_metric: state.best_valid_metrics},
)
def on_loader_end(self, state: RunnerState):
true_labels = np.array(self.true_labels)
pred_probas = np.array(self.pred_labels)
if len(np.unique(true_labels) > 2):
true_labels = true_labels > 0.5
else:
true_labels = true_labels.astype(np.bool)
logger = get_tensorboard_logger(state)
logger.add_histogram(self.prefix + "/neg", pred_probas[true_labels == False], state.epoch)
logger.add_histogram(self.prefix + "/pos", pred_probas[true_labels == True], state.epoch)
def on_loader_end(self, state: RunnerState):
true_labels = np.array(self.true_labels)
pred_labels = np.array(self.pred_labels)
quality_factors = np.array(self.quality_factors)
true_labels = all_gather(true_labels)
true_labels = np.concatenate(true_labels)
pred_labels = all_gather(pred_labels)
pred_labels = np.concatenate(pred_labels)
quality_factors = all_gather(quality_factors)
quality_factors = np.concatenate(quality_factors)
true_labels_b = (true_labels > 0).astype(int)
# Just ensure true_labels are 0,1
score = alaska_weighted_auc(true_labels_b, pred_labels)
state.metrics.epoch_values[state.loader_name][self.prefix] = float(score)
logger = get_tensorboard_logger(state)
logger.add_pr_curve(self.prefix, true_labels_b, pred_labels)
# Compute
if len(quality_factors) > 0:
score_75 = alaska_weighted_auc(true_labels_b[quality_factors == 0], pred_labels[quality_factors == 0])
score_90 = alaska_weighted_auc(true_labels_b[quality_factors == 1], pred_labels[quality_factors == 1])
score_95 = alaska_weighted_auc(true_labels_b[quality_factors == 2], pred_labels[quality_factors == 2])
state.metrics.epoch_values[state.loader_name][self.prefix + "/qf_75"] = float(score_75)
state.metrics.epoch_values[state.loader_name][self.prefix + "/qf_90"] = float(score_90)
state.metrics.epoch_values[state.loader_name][self.prefix + "/qf_95"] = float(score_95)
score_mask = np.zeros((3, 3))
for qf in [0, 1, 2]:
for target in range(len(self.class_names) - 1):
mask = (quality_factors == qf) & ((true_labels == 0) | (true_labels == target + 1))
score = alaska_weighted_auc(true_labels_b[mask], pred_labels[mask])
score_mask[qf, target] = score
fig = self.plot_matrix(
score_mask,
figsize=(8, 8),
x_names=self.class_names[1:],
y_names=["75", "90", "95"],
normalize=False,
noshow=True,
)
fig = render_figure_to_tensor(fig)
logger.add_image(f"{self.prefix}/matrix", fig, global_step=state.step)
def on_loader_end(self, state: RunnerState):
true_labels = np.array(self.true_labels)
pred_labels = np.array(self.pred_labels)
true_labels = all_gather(true_labels)
true_labels = np.concatenate(true_labels)
pred_labels = all_gather(pred_labels)
pred_labels = np.concatenate(pred_labels)
true_labels_b = (true_labels > 0).astype(int)
# Just ensure true_labels are 0,1
score = alaska_weighted_auc(true_labels_b, pred_labels)
state.metrics.epoch_values[state.loader_name][self.prefix] = float(score)
logger = get_tensorboard_logger(state)
logger.add_pr_curve(self.prefix, true_labels_b, pred_labels)
def on_loader_end(self, state):
if self.class_names is None:
class_names = [str(i) for i in range(self.num_classes)]
else:
class_names = self.class_names
num_classes = len(class_names)
cm = self.confusion_matrix.value()
fig = plot_confusion_matrix(
cm,
figsize=(6 + num_classes // 3, 6 + num_classes // 3),
class_names=class_names,
normalize=True,
noshow=True,
)
fig = render_figure_to_tensor(fig)
logger = get_tensorboard_logger(state)
logger.add_image(f"{self.prefix}/epoch", fig, global_step=state.step)
def on_loader_end(self, state):
targets = np.array(self.targets)
outputs = np.array(self.outputs)
if self.class_names is None:
class_names = [str(i) for i in range(targets.shape[1])]
else:
class_names = self.class_names
num_classes = len(class_names)
cm = confusion_matrix(targets, outputs, labels=range(num_classes))
fig = plot_confusion_matrix(cm,
figsize=(6 + num_classes // 3, 6 + num_classes // 3),
class_names=class_names,
normalize=True,
noshow=True)
fig = render_figure_to_tensor(fig)
logger = get_tensorboard_logger(state)
logger.add_image(f'{self.prefix}/epoch', fig, global_step=state.step)
def on_loader_end(self, runner: IRunner):
eps = 1e-7
ious_per_image = []
# Gather statistics from all nodes
all_gathered_scores_per_image = all_gather(self.scores_per_image)
n = len(self.thresholds)
all_scores_per_image = defaultdict(lambda: {
"intersection": np.zeros(n),
"union": np.zeros(n)
})
for scores_per_image in all_gathered_scores_per_image:
for image_id, values in scores_per_image.items():
all_scores_per_image[image_id]["intersection"] += values[
"intersection"]
all_scores_per_image[image_id]["union"] += values["union"]
for image_id, values in all_scores_per_image.items():
intersection = values["intersection"]
union = values["union"]
metric = intersection / (union + eps)
ious_per_image.append(metric)
thresholds = to_numpy(self.thresholds)
iou = np.mean(ious_per_image, axis=0)
assert len(iou) == len(thresholds)
threshold_index = np.argmax(iou)
iou_at_threshold = iou[threshold_index]
threshold_value = thresholds[threshold_index]
runner.loader_metrics[self.prefix + "/" +
"threshold"] = float(threshold_value)
runner.loader_metrics[self.prefix] = float(iou_at_threshold)
if get_rank() in {-1, 0}:
logger = get_tensorboard_logger(runner)
logger.add_histogram(self.prefix, iou, global_step=runner.epoch)