def single_inference(model, image_dict, return_offset=True):
with torch.no_grad():
image, trimap = image_dict['image'], image_dict['trimap']
alpha_shape = image_dict['alpha_shape']
# image = image.cuda()
# trimap = trimap.cuda()
alpha_pred, info_dict = model(image, trimap)
if CONFIG.model.trimap_channel == 3:
trimap_argmax = trimap.argmax(dim=1, keepdim=True)
alpha_pred[trimap_argmax == 2] = 1
alpha_pred[trimap_argmax == 0] = 0
h, w = alpha_shape
test_pred = alpha_pred[0, 0, ...].data.cpu().numpy() * 255
test_pred = test_pred.astype(np.uint8)
test_pred = test_pred[32:h + 32, 32:w + 32]
if return_offset:
short_side = h if h < w else w
ratio = 512 / short_side
offset_1 = utils.flow_to_image(
info_dict['offset_1'][0][0, ...].data.cpu().numpy()).astype(
np.uint8)
# write softmax_scale to offset image
scale = info_dict['offset_1'][1].cpu()
offset_1 = cv2.resize(offset_1, (int(w * ratio), int(h * ratio)),
interpolation=cv2.INTER_NEAREST)
text = 'unknown: {:.2f}, known: {:.2f}'.format(
scale[-1, 0].item(), scale[-1, 1].item())
offset_1 = cv2.putText(offset_1,
text, (10, 20),
cv2.FONT_HERSHEY_SIMPLEX,
0.8,
0,
thickness=2)
offset_2 = utils.flow_to_image(
info_dict['offset_2'][0][0, ...].data.cpu().numpy()).astype(
np.uint8)
# write softmax_scale to offset image
scale = info_dict['offset_2'][1].cpu()
offset_2 = cv2.resize(offset_2, (int(w * ratio), int(h * ratio)),
interpolation=cv2.INTER_NEAREST)
text = 'unknown: {:.2f}, known: {:.2f}'.format(
scale[-1, 0].item(), scale[-1, 1].item())
offset_2 = cv2.putText(offset_2,
text, (10, 20),
cv2.FONT_HERSHEY_SIMPLEX,
0.8,
0,
thickness=2)
return test_pred, (offset_1, offset_2)
else:
return test_pred, None
def unsupervised_loss(image1_2,
image2_2,
pred_depth,
pred_R,
pred_T,
summarize=True):
fy = tf.fill([image1_2.shape.as_list()[0]], 1.18821287 * 192)
fx = tf.fill([image1_2.shape.as_list()[0]], 0.89115971 * 256)
y0 = x0 = tf.fill([image1_2.shape.as_list()[0]], 0.5)
# Convert predicted *extrinsics* to pose
pose_R = tf.transpose(pred_R, perm=[0, 2, 1])
pose_t = tf.matmul(tf.matrix_inverse(-pred_R), tf.expand_dims(pred_T,
2))[:, :, 0]
flow, _ = zrt2flow(1.0 / pred_depth, pose_R, pose_t, fy, fx, y0, x0)
warped_image2, _ = warper(image2_2, flow)
loss = l1Loss(warped_image2, image1_2)
if summarize:
tf.summary.image('image 1', tf.expand_dims(image1_2[0, ...], 0))
tf.summary.image('image 2', tf.expand_dims(image2_2[0, ...], 0))
tf.summary.image('flow', tf.expand_dims(flow_to_image(flow[0, ...]),
0))
tf.summary.image('image 2 warped',
tf.expand_dims(warped_image2[0, ...], 0))
tf.summary.image('inv. depth', tf.expand_dims(pred_depth[0, ...], 0))
tf.summary.scalar('total_loss', loss)
return loss
def process_two_images(model, imgs, ctx=None):
"""
Process two images into one flow image
Args:
model: The model to use
imgs: a list of 2 images
ctx: the model ctx
Returns:
"""
if len(imgs) != 2:
return None
if isinstance(imgs[0], str):
if os.path.exists(imgs[0]):
imgs[0] = cv2.cvtColor(cv2.imread(files[i]), cv2.COLOR_BGR2RGB)
else:
return None
if isinstance(imgs[1], str):
if os.path.exists(imgs[1]):
imgs[1] = cv2.cvtColor(cv2.imread(files[i]), cv2.COLOR_BGR2RGB)
else:
return None
imgs = crop(imgs)
imgs = np.array(imgs)
imgs = np.moveaxis(imgs, -1, 1)
imgs = normalise(imgs)
imgs = mx.nd.array(imgs, ctx=ctx)
imgs = mx.nd.expand_dims(imgs, 0) # add batch axis
flow = model(imgs) # run the model
flow = flow.asnumpy()
flow = flow.squeeze()
flow = flow.transpose(1, 2, 0)
img = flow_to_image(flow)
img = imresize(
img, 4.0
) # doing the bilinear interpolation on the img, NOT flow cause was too hard :'(
return img, flow
def main(args):
model = build_optical_flow_model(args)
dataset, collate_fn, batch_frontend = train.build_dataset(args,
read_frames=True)
print(args.checkpoint, args.dataset_root_dir, args.dataset_root_dir_flow)
for idx, (frames_paths, frames, flow_paths, flow) in enumerate(dataset):
assert frames.shape[-1] % 8 == 0 and frames.shape[-1] % 8 == 0
frames = frames.to(args.device)
img_src, img_dst, flow_dst = frames[:1], frames[1:], flow_paths[1:]
#if all(map(os.path.exists, flow_dst)):
# print(idx, '/', len(dataset), '. Skipping existing', flow_dst)
# continue
flow_lo, flow_hi = model(img_src.expand(len(img_dst), -1, -1, -1),
img_dst,
iters=args.num_iter,
test_mode=True)
ufactor, vfactor = (args.resolution[-1] / flow_hi.shape[-1],
args.resolution[-2] / flow_hi.shape[-2])
flow = F.interpolate(flow_hi, args.resolution).movedim(
1, -1) * torch.tensor([ufactor, vfactor], device=args.device)
print('frames:', frames.shape, 'flow lowres:', flow_lo.shape,
'flow hires:', flow_hi.shape, 'flow:', flow.shape)
os.makedirs(os.path.dirname(flow_dst[0]), exist_ok=True)
for flo, flow_path in zip(flow.cpu(), flow_dst):
flo = torch.as_tensor(utils.flow_to_image(flo.numpy()))
cv2.imwrite(flow_path, flo.flip(-1).numpy())
print(idx, '/', len(dataset))
print(args.checkpoint, args.dataset_root_dir, args.dataset_root_dir_flow)