def _test(metric_device, y_test_1, y_test_2):
criterion = nn.NLLLoss().to(device)
loss = Loss(criterion, device=metric_device)
y_pred, y, _ = y_test_1
loss.update((y_pred, y))
n = loss._num_examples
assert n == len(y)
res = loss.compute()
assert n * idist.get_world_size() == loss._num_examples
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
true_loss_value = criterion(y_pred, y)
assert_almost_equal(res, true_loss_value.item())
loss.reset()
y_pred, y, _ = y_test_2
loss.update((y_pred, y))
n = loss._num_examples
res = loss.compute()
assert n * idist.get_world_size() == loss._num_examples
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
true_loss_value = criterion(y_pred, y)
if tol is None:
assert_almost_equal(res, true_loss_value.item())
else:
assert pytest.approx(res, rel=tol) == true_loss_value.item()
def _test(metric_device):
criterion = nn.NLLLoss().to(device)
loss = Loss(criterion, device=metric_device)
y_pred = torch.tensor([[0.1, 0.4, 0.5], [0.1, 0.7, 0.2]], device=device).log()
y = torch.tensor([2, 2], device=device).long()
loss.update((y_pred, y))
n = loss._num_examples
assert n == len(y)
res = loss.compute()
assert n * idist.get_world_size() == loss._num_examples
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
true_loss_value = criterion(y_pred, y)
assert_almost_equal(res, true_loss_value.item())
loss.reset()
y_pred = torch.tensor([[0.1, 0.3, 0.6], [0.6, 0.2, 0.2], [0.2, 0.7, 0.1]], device=device).log()
y = torch.tensor([2, 0, 2], device=device).long()
loss.update((y_pred, y))
n = loss._num_examples
res = loss.compute()
assert n * idist.get_world_size() == loss._num_examples
y_pred = idist.all_gather(y_pred)
y = idist.all_gather(y)
true_loss_value = criterion(y_pred, y)
if tol is None:
assert_almost_equal(res, true_loss_value.item())
else:
assert pytest.approx(res, rel=tol) == true_loss_value.item()
def _evaluate_one_epoch(self, loader, epoch):
""" Evaluate one epoch
Args:
loader (DataLoader): pytorch dataloader
epoch (int): the current epoch number
"""
logger.info(f"Epoch[{epoch}] evaluation started")
self.model.eval()
loss_metric = Loss(self._loss_fn)
# TODO: Support other metrics other than IoU and support multiple
# mettics
iou_metric = EvaluationMetric.create(self.config.metric,
num_classes=self.num_classes)
with torch.no_grad():
for image, target in loader:
image, target = image.to(self.device), target.to(self.device)
output = self.model(image)
loss_metric.update((output, target))
iou_metric.update((output["out"], target))
loss = loss_metric.compute()
iou = iou_metric.compute()
# some classes are not used in cityscapes evaluation.
# TODO: Move class masking logic to IoU metric class.
keep_mask = [
not c.ignore_in_eval
for c in torchvision.datasets.Cityscapes.classes
]
class_names = [c.name for c in torchvision.datasets.Cityscapes.classes]
iou_info = {
name: f"{iou[i].item():.3f}"
for i, name in enumerate(class_names) if keep_mask[i]
}
miou = iou[keep_mask].mean()
logger.info(f"Epoch[{epoch}] evaluation completed. "
f"Loss: {loss:.3f}, mIoU: {miou:.3f}\n"
f"IoU per class: {iou_info}")
self.writer.add_scalar("validation/loss", loss, epoch)
self.writer.add_scalar("validation/miou", miou, epoch)
inv_normalize = T.Normalize(mean=_INV_IMGNET_MEAN, std=_INV_IMGNET_STD)
# Visualize segmentation images from last mini-batch
n_images = list(image.shape)[0]
image_grid = []
for i in range(n_images):
img = inv_normalize(image[i, :]).permute(1, 2, 0).cpu().numpy()
out = decode_segmap(output["out"][i, :].max(0)[1].cpu().numpy())
tgt = decode_segmap(target[i, :].cpu().numpy())
image_grid.append([img, out, tgt])
fig = grid_plot(image_grid)
self.writer.add_figure("validation/visualize", fig, epoch)
loss_metric.reset()
iou_metric.reset()
def test_reset():
loss = Loss(nll_loss)
y_pred, y = y_test_3()
loss.update((y_pred, y))
loss.compute()
loss.reset()
with pytest.raises(NotComputableError):
loss.compute()
def test_reset():
loss = Loss(nll_loss)
y_pred = torch.tensor([[0.1, 0.3, 0.6], [0.6, 0.2, 0.2]]).log()
y = torch.tensor([2, 0]).long()
loss.update((y_pred, y))
loss.compute()
loss.reset()
with pytest.raises(NotComputableError):
loss.compute()
def _train_one_epoch(self, loader, epoch):
""" Train one epoch
Args:
loader (DataLoader): pytorch dataloader
epoch (int): the current epoch number
"""
logger.info(f"Epoch[{epoch}] training started.")
self.model.train()
n_batch = len(loader)
accumulation_steps = self.config.train.accumulation_steps
loss_metric = Loss(self._loss_fn)
self.optimizer.zero_grad()
for i, (image, target) in enumerate(loader):
image, target = image.to(self.device), target.to(self.device)
output = self.model(image)
loss = self._loss_fn(output, target)
loss.backward()
# Accumulated Gradients are only updated after X steps.
# This creates an effective batch size of
# batch_size * accumulation_steps
if (i + 1) % accumulation_steps == 0:
self.optimizer.step()
self.lr_scheduler.step()
self.optimizer.zero_grad()
loss_metric.update((output, target))
iter_num = (i + 1) % n_batch
logger.debug(f"Epoch[{epoch}] Iteration[{iter_num}/{n_batch}] "
f"Loss: {loss:.3f}")
epoch_loss = loss_metric.compute()
logger.info(
f"Epoch[{epoch}] training completed. Loss: {epoch_loss:.3f}")
self.writer.add_scalar("training/loss", epoch_loss, epoch)
loss_metric.reset()
def _test_distrib_compute_on_criterion(device):
import torch.distributed as dist
def _gather(y):
output = [torch.zeros_like(y) for i in range(dist.get_world_size())]
dist.all_gather(output, y)
y = torch.cat(output, dim=0)
return y
criterion = nn.NLLLoss().to(device)
loss = Loss(criterion, device=device)
y_pred = torch.tensor([[0.1, 0.4, 0.5], [0.1, 0.7, 0.2]],
device=device).log()
y = torch.tensor([2, 2], device=device).long()
loss.update((y_pred, y))
n = loss._num_examples
assert n == len(y)
res = loss.compute()
assert n * dist.get_world_size() == loss._num_examples
y_pred = _gather(y_pred)
y = _gather(y)
true_loss_value = criterion(y_pred, y)
assert_almost_equal(res, true_loss_value.item())
loss.reset()
y_pred = torch.tensor([[0.1, 0.3, 0.6], [0.6, 0.2, 0.2], [0.2, 0.7, 0.1]],
device=device).log()
y = torch.tensor([2, 0, 2], device=device).long()
loss.update((y_pred, y))
n = loss._num_examples
res = loss.compute()
assert n * dist.get_world_size() == loss._num_examples
y_pred = _gather(y_pred)
y = _gather(y)
true_loss_value = criterion(y_pred, y)
assert_almost_equal(res, true_loss_value.item())
def _evaluate_one_epoch(self, loader, epoch, n_epochs):
""" Evaluate one epoch
Args:
loader (DataLoader): pytorch dataloader
epoch (int): the current epoch number
n_epochs (int): total epoch number
"""
logger.info(f"Epoch[{epoch}/{n_epochs}] evaluation started")
self.model.eval()
loss_metric = Loss(self._loss_fn)
# metrics
# The RMSE is smaller than the normal scale. Because we didn't do the:
# depth = self.to_tensor(depth).float() * 10
are_metric = EvaluationMetric.create("AverageRelativeError")
log10_metric = EvaluationMetric.create("AverageLog10Error")
rmse_metric = EvaluationMetric.create("RootMeanSquareError")
a1_metric = EvaluationMetric.create("ThresholdAccuracy",
threshold=1.25)
a2_metric = EvaluationMetric.create("ThresholdAccuracy",
threshold=1.25**2)
a3_metric = EvaluationMetric.create("ThresholdAccuracy",
threshold=1.25**3)
with torch.no_grad():
for image, depth in loader:
image = image.to(self.device)
depth_n = depth.to(self.device)
output = self.model(image)
loss_metric.update((output, depth_n))
depth_image_pair = (output.cpu().numpy(),
depth_n.cpu().numpy())
are_metric.update(depth_image_pair)
log10_metric.update(depth_image_pair)
rmse_metric.update(depth_image_pair)
a1_metric.update(depth_image_pair)
a2_metric.update(depth_image_pair)
a3_metric.update(depth_image_pair)
# Compute the loss
loss = loss_metric.compute()
are_metric_val = are_metric.compute()
log10_metric_val = log10_metric.compute()
rmse_metric_val = rmse_metric.compute()
a1_metric_val = a1_metric.compute()
a2_metric_val = a2_metric.compute()
a3_metric_val = a3_metric.compute()
logger.info(f"Epoch[{epoch}/{n_epochs}] evaluation completed.\n"
f"Validation Loss: {loss:.3f}\n"
f"Average Relative Error: {are_metric_val:.3f}\n"
f"Average Log10 Error: {log10_metric_val:.3f}\n"
f"Root Mean Square Error: {rmse_metric_val:.3f}\n"
f"Threshold Accuracy (delta1): {a1_metric_val:.3f}\n"
f"Threshold Accuracy (delta2): {a2_metric_val:.3f}\n"
f"Threshold Accuracy (delta3): {a3_metric_val:.3f}\n")
self.writer.add_scalar("Validation/loss", loss, epoch)
self.writer.add_scalar("Validation/Average_Relative_Error",
are_metric_val, epoch)
self.writer.add_scalar("Validation/Average_Log10_Error",
log10_metric_val, epoch)
self.writer.add_scalar("Validation/Root_Mean_Square_Error",
rmse_metric_val, epoch)
self.writer.add_scalar("Validation/Threshold_Accuracy__delta1_",
a1_metric_val, epoch)
self.writer.add_scalar("Validation/Threshold_Accuracy__delta2_",
a2_metric_val, epoch)
self.writer.add_scalar("Validation/Threshold_Accuracy__delta3_",
a3_metric_val, epoch)
# Visualize depth images from last mini-batch
n_images = image.shape[0]
image_grid = []
gray_depth_grid = []
for i in range(n_images):
rgb_image = image[i].permute(1, 2, 0)
image_grid.append([rgb_image.cpu().numpy()])
gray_depth_grid.append([
output[i].permute(1, 2, 0)[:, :, -1].cpu().numpy(),
depth_n[i].permute(1, 2, 0)[:, :, -1].cpu().numpy(),
])
# Add figures
fig = grid_plot(image_grid)
gray_figs = grid_plot(gray_depth_grid, img_type="gray")
self.writer.add_figure("Validation/visualize_depths", gray_figs, epoch)
self.writer.add_figure("Validation/visualize_image", fig, epoch)
loss_metric.reset()
are_metric.reset()
log10_metric.reset()
rmse_metric.reset()
a1_metric.reset()
a2_metric.reset()
a3_metric.reset()
