def compute_metric(self, preds, minibatch):
if minibatch["target"].shape != preds["target"][-1].shape:
h, w = minibatch["target"].shape[-2:]
# minibatch = interpolate(minibatch, size=(h, w), mode='bilinear', align_corners=True)
preds = interpolate(preds,
size=(h, w),
mode='bilinear',
align_corners=True)
gt = minibatch["target"]
mask = (gt > 0.)
if len(gt[mask]) == 0:
return
imse, imae, mse, mae, rmse, absrel, lg10, silog, d1, d2, d3 \
= [], [], [], [], [], [], [], [], [], [], []
if self.n_stage == -1:
self.n_stage = len(preds["target"])
for scale_idx in range(self.n_stage):
pred = preds["target"][scale_idx]
metirc_dict = compute_metric(pred, gt)
metirc_dict = reduce_dict(metirc_dict)
# imse.append(metirc_dict["imse"])
# imae.append(metirc_dict["imae"])
mse.append(metirc_dict["mse"])
mae.append(metirc_dict["mae"])
rmse.append(metirc_dict["rmse"])
absrel.append(metirc_dict["absrel"])
# lg10.append(metirc_dict["lg10"])
silog.append(metirc_dict["silog"])
d1.append(metirc_dict["delta1"])
d2.append(metirc_dict["delta2"])
d3.append(metirc_dict["delta3"])
# self.imse.update(imse)
# self.imae.update(imae)
self.mse.update(mse)
self.mae.update(mae)
self.rmse.update(rmse)
self.absrel.update(absrel)
# self.lg10.update(lg10)
self.silog.update(silog)
self.d1.update(d1)
self.d2.update(d2)
self.d3.update(d3)
del imse
del imae
del mse
del rmse
del absrel
del lg10
del silog
del d1
del d2
del d3
def collate_fn(batch):
r"""Puts each data field into a tensor with outer dimension batch size"""
elem = batch[0]
elem_type = type(elem)
### dedicated collate function for CamInfo
if isinstance(elem, CamInfo):
return batch_cam_infos(batch)
elif isinstance(elem, torch.Tensor):
#### start
h, w = elem.shape[-2:]
if len(batch) > 0:
for i in range(1, len(batch)):
th, tw = batch[i].shape[-2:]
h, w = min(h, th), min(w, tw)
for i in range(len(batch)):
if batch[i].shape[-2:] != (h, w):
batch[i] = interpolate(batch[i], size=(h, w), mode="nearest")
#### end
out = None
# if torch.utils.data.get_worker_info() is not None:
# # If we're in a background process, concatenate directly into a
# # shared memory tensor to avoid an extra copy
# numel = sum([x.numel() for x in batch])
# storage = elem.storage()._new_shared(numel)
# out = elem.new(storage)
return torch.stack(batch, 0, out=out)
elif elem_type.__module__ == 'numpy' and elem_type.__name__ != 'str_' \
and elem_type.__name__ != 'string_':
elem = batch[0]
if elem_type.__name__ == 'ndarray':
# array of string classes and object
if np_str_obj_array_pattern.search(elem.dtype.str) is not None:
raise TypeError(
default_collate_err_msg_format.format(elem.dtype))
return collate_fn([torch.as_tensor(b) for b in batch])
elif elem.shape == (): # scalars
return torch.as_tensor(batch)
elif isinstance(elem, float):
return torch.tensor(batch, dtype=torch.float64)
elif isinstance(elem, int_classes):
return torch.tensor(batch)
elif isinstance(elem, string_classes):
return batch
elif isinstance(elem, container_abcs.Mapping):
return {key: collate_fn([d[key] for d in batch]) for key in elem}
elif isinstance(elem, tuple) and hasattr(elem, '_fields'): # namedtuple
return elem_type(*(collate_fn(samples) for samples in zip(*batch)))
elif isinstance(elem, container_abcs.Sequence):
transposed = zip(*batch)
return [collate_fn(samples) for samples in transposed]
raise TypeError(default_collate_err_msg_format.format(elem_type))
def visualize(self, batch, out, epoch=0):
"""
:param batch_in: minibatch
:param pred_out: model output for visualization, dic, {"target": [NxHxW]}
:param tensorboard: if tensorboard = True, the visualized image should be in [0, 1].
:return: vis_ims: image for visualization.
"""
fn = batch["fn"]
if batch["target"].shape != out["target"][-1].shape:
h, w = batch["target"].shape[-2:]
# batch = interpolate(batch, size=(h, w), mode='nearest')
out = interpolate(out,
size=(h, w),
mode='bilinear',
align_corners=True)
image = batch["image_n"].numpy()
has_gt = False
if batch.get("target") is not None:
depth_gts = tensor2numpy(batch["target"])
has_gt = True
for i in range(len(fn)):
image = image[i].astype(np.float)
depth = np.zeros(
(out['target'][0].shape[0], batch["H"], batch["W"]),
dtype=np.float32)
depth[:] = np.nan
out_depth = tensor2numpy(out['target'][0])
for j in range(depth.shape[0]):
x0 = batch["x0"][j]
x1 = batch["x1"][j]
y0 = batch["y0"][j]
y1 = batch["y1"][j]
target_patch = depth[j, y0:y1, x0:x1]
patch = cv2.resize(
out_depth[j],
(target_patch.shape[1], target_patch.shape[0]),
interpolation=cv2.INTER_LINEAR)
depth[j, y0:y1, x0:x1] = patch
depth = np.nanmean(depth, axis=0)
depth = cv2.resize(depth, (batch["W_img"], batch["H_img"]),
interpolation=cv2.INTER_LINEAR)
output_dir, sample_idx = os.path.split(batch["target_path"][i])
output_dir = os.path.dirname(output_dir)
output_file = os.path.join(output_dir, "depth_pred", sample_idx)
depth_image = (depth * 255.0).astype(np.uint16)
cv2.imwrite(output_file, depth_image)
def __call__(self, pred, gt):
assert pred.dim() == gt.dim(), \
"inconsistent dimensions, pred shape is {}, but gt shape is {}.".format(pred.shape, gt.shape)
if pred.shape != gt.shape:
pred = interpolate(pred,
size=gt.shape[-2:],
mode="bilinear",
align_corners=True)
valid_mask = (gt > 0).detach()
diff = gt - pred
diff = diff[valid_mask]
self.loss = diff.abs().mean()
return self.loss
def visualize(self, batch, out, epoch=0):
"""
:param batch_in: minibatch
:param pred_out: model output for visualization, dic, {"target": [NxHxW]}
:param tensorboard: if tensorboard = True, the visualized image should be in [0, 1].
:return: vis_ims: image for visualization.
"""
fn = batch["fn"]
if batch["target"].shape != out["target"][-1].shape:
h, w = batch["target"].shape[-2:]
# batch = interpolate(batch, size=(h, w), mode='nearest')
out = interpolate(out, size=(h, w), mode='bilinear', align_corners=True)
image = batch["image_n"].numpy()
has_gt = False
if batch.get("target") is not None:
depth_gts = tensor2numpy(batch["target"])
has_gt = True
for i in range(len(fn)):
image = image[i].astype(np.float)
depth = tensor2numpy(out['target'][0][i])
# print("!! depth shape:", depth.shape)
if has_gt:
depth_gt = depth_gts[i]
err = error_to_color(depth, depth_gt)
depth_gt = depth_to_color(depth_gt)
depth = depth_to_color(depth)
# print("pred:", depth.shape, " target:", depth_gt.shape)
group = np.concatenate((image, depth), axis=0)
if has_gt:
gt_group = np.concatenate((depth_gt, err), axis=0)
group = np.concatenate((group, gt_group), axis=1)
if self.writer is not None:
group = group.transpose((2, 0, 1)) / 255.0
group = group.astype(np.float32)
# print("group shape:", group.shape)
self.writer.add_image(fn[i] + "/image", group, epoch)