def on_loader_end(self, runner):
y_trues = np.concatenate(all_gather(self.y_trues))
y_preds = np.concatenate(all_gather(self.y_preds))
score = roc_auc_score(y_true=y_trues,
y_score=y_preds,
average=self.average)
runner.loader_metrics[self.prefix] = float(score)
def _compute_embedings(self):
features = np.array(self.features)
features = np.concatenate(all_gather(features))
targets = np.array(self.targets)
targets = np.concatenate(all_gather(targets))
outputs = np.array(self.outputs)
outputs = np.concatenate(all_gather(outputs))
import umap
return umap.UMAP(**self._fit_params).fit(features), targets, outputs
def on_loader_end(self, state: RunnerState):
true_labels = np.array(self.true_labels)
pred_labels = np.array(self.pred_labels)
quality_factors = np.array(self.quality_factors)
true_labels = all_gather(true_labels)
true_labels = np.concatenate(true_labels)
pred_labels = all_gather(pred_labels)
pred_labels = np.concatenate(pred_labels)
quality_factors = all_gather(quality_factors)
quality_factors = np.concatenate(quality_factors)
true_labels_b = (true_labels > 0).astype(int)
# Just ensure true_labels are 0,1
score = alaska_weighted_auc(true_labels_b, pred_labels)
state.metrics.epoch_values[state.loader_name][self.prefix] = float(score)
logger = get_tensorboard_logger(state)
logger.add_pr_curve(self.prefix, true_labels_b, pred_labels)
# Compute
if len(quality_factors) > 0:
score_75 = alaska_weighted_auc(true_labels_b[quality_factors == 0], pred_labels[quality_factors == 0])
score_90 = alaska_weighted_auc(true_labels_b[quality_factors == 1], pred_labels[quality_factors == 1])
score_95 = alaska_weighted_auc(true_labels_b[quality_factors == 2], pred_labels[quality_factors == 2])
state.metrics.epoch_values[state.loader_name][self.prefix + "/qf_75"] = float(score_75)
state.metrics.epoch_values[state.loader_name][self.prefix + "/qf_90"] = float(score_90)
state.metrics.epoch_values[state.loader_name][self.prefix + "/qf_95"] = float(score_95)
score_mask = np.zeros((3, 3))
for qf in [0, 1, 2]:
for target in range(len(self.class_names) - 1):
mask = (quality_factors == qf) & ((true_labels == 0) | (true_labels == target + 1))
score = alaska_weighted_auc(true_labels_b[mask], pred_labels[mask])
score_mask[qf, target] = score
fig = self.plot_matrix(
score_mask,
figsize=(8, 8),
x_names=self.class_names[1:],
y_names=["75", "90", "95"],
normalize=False,
noshow=True,
)
fig = render_figure_to_tensor(fig)
logger.add_image(f"{self.prefix}/matrix", fig, global_step=state.step)
def on_loader_end(self, runner: IRunner):
# Gather statistics from all nodes
gathered_scores_per_image = all_gather(self.scores_per_image)
all_scores_per_image = defaultdict(lambda: {
"intersection": 0.0,
"union": 0.0
})
for scores_per_image in gathered_scores_per_image:
for image_id, values in scores_per_image.items():
all_scores_per_image[image_id]["intersection"] += values[
"intersection"]
all_scores_per_image[image_id]["union"] += values["union"]
eps = 1e-7
ious_per_image = []
ious_per_location = defaultdict(list)
for image_id, values in all_scores_per_image.items():
intersection = values["intersection"]
union = values["union"]
metric = intersection / (union + eps)
ious_per_image.append(metric)
for location in self.locations:
if str.startswith(image_id, location):
ious_per_location[location].append(metric)
metric = float(np.mean(ious_per_image))
runner.loader_metrics[self.prefix] = metric
for location, ious in ious_per_location.items():
runner.loader_metrics[f"{self.prefix}/{location}"] = float(
np.mean(ious))
def on_loader_end(self, state: RunnerState):
true_labels = np.array(self.true_labels)
pred_labels = np.array(self.pred_labels)
true_labels = all_gather(true_labels)
true_labels = np.concatenate(true_labels)
pred_labels = all_gather(pred_labels)
pred_labels = np.concatenate(pred_labels)
true_labels_b = (true_labels > 0).astype(int)
# Just ensure true_labels are 0,1
score = alaska_weighted_auc(true_labels_b, pred_labels)
state.metrics.epoch_values[state.loader_name][self.prefix] = float(score)
logger = get_tensorboard_logger(state)
logger.add_pr_curve(self.prefix, true_labels_b, pred_labels)
def on_loader_end(self, runner: IRunner):
eps = 1e-7
ious_per_image = []
# Gather statistics from all nodes
all_gathered_scores_per_image = all_gather(self.scores_per_image)
n = len(self.thresholds)
all_scores_per_image = defaultdict(lambda: {
"intersection": np.zeros(n),
"union": np.zeros(n)
})
for scores_per_image in all_gathered_scores_per_image:
for image_id, values in scores_per_image.items():
all_scores_per_image[image_id]["intersection"] += values[
"intersection"]
all_scores_per_image[image_id]["union"] += values["union"]
for image_id, values in all_scores_per_image.items():
intersection = values["intersection"]
union = values["union"]
metric = intersection / (union + eps)
ious_per_image.append(metric)
thresholds = to_numpy(self.thresholds)
iou = np.mean(ious_per_image, axis=0)
assert len(iou) == len(thresholds)
threshold_index = np.argmax(iou)
iou_at_threshold = iou[threshold_index]
threshold_value = thresholds[threshold_index]
runner.loader_metrics[self.prefix + "/" +
"threshold"] = float(threshold_value)
runner.loader_metrics[self.prefix] = float(iou_at_threshold)
if get_rank() in {-1, 0}:
logger = get_tensorboard_logger(runner)
logger.add_histogram(self.prefix, iou, global_step=runner.epoch)