def video_completion_seamless(args):
print('Pytorch 1.6.0 for flow prediction')
try:
RAFT_model = initialize_RAFT(args)
except:
RAFT_model = None
# Loads frames.
filename_list = glob.glob(os.path.join(args.path, '*.png')) + \
glob.glob(os.path.join(args.path, '*.jpg'))
# Obtains imgH, imgW and nFrame.
imgH, imgW = np.array(Image.open(filename_list[0])).shape[:2]
nFrame = len(filename_list)
# Loads video.
video = []
for filename in sorted(filename_list):
video.append(
torch.from_numpy(np.array(Image.open(filename)).astype(
np.uint8)).permute(2, 0, 1).float())
video = torch.stack(video, dim=0)
video = video.to('cuda')
# Calcutes the corrupted flow.
corrFlowF = calculate_flow(args, RAFT_model, video, 'forward')
corrFlowB = calculate_flow(args, RAFT_model, video, 'backward')
print('\nFinish Calculating flow.')
# Makes sure video is in BGR (opencv) format.
video = video.permute(2, 3, 1, 0).cpu().numpy()[:, :, ::-1, :] / 255.
if args.mode == 'video_extrapolation':
# Creates video and flow where the extrapolated region are missing.
video, corrFlowF, corrFlowB, flow_mask, mask_dilated, start_point, end_point = extrapolation(
args, video, corrFlowF, corrFlowB)
imgH, imgW = video.shape[:2]
# mask indicating the missing region in the video.
mask = np.tile(flow_mask[..., None], (1, 1, nFrame))
flow_mask = np.tile(flow_mask[..., None], (1, 1, nFrame))
mask_dilated = np.tile(mask_dilated[..., None], (1, 1, nFrame))
else:
# Loads masks.
filename_list = glob.glob(os.path.join(args.path_mask, '*.png')) + \
glob.glob(os.path.join(args.path_mask, '*.jpg'))
mask = []
mask_dilated = []
flow_mask = []
for filename in sorted(filename_list):
mask_img = np.array(Image.open(filename).convert('L'))
# Dilate 15 pixel so that all known pixel is trustworthy
flow_mask_img = scipy.ndimage.binary_dilation(mask_img,
iterations=15)
# Close the small holes inside the foreground objects
flow_mask_img = cv2.morphologyEx(flow_mask_img.astype(np.uint8),
cv2.MORPH_CLOSE,
np.ones((21, 21),
np.uint8)).astype(np.bool)
flow_mask_img = scipy.ndimage.binary_fill_holes(
flow_mask_img).astype(np.bool)
flow_mask.append(flow_mask_img)
mask_img = scipy.ndimage.binary_dilation(mask_img, iterations=5)
mask_img = scipy.ndimage.binary_fill_holes(mask_img).astype(
np.bool)
mask.append(mask_img)
mask_dilated.append(gradient_mask(mask_img))
# mask indicating the missing region in the video.
mask = np.stack(mask, -1).astype(np.bool)
mask_dilated = np.stack(mask_dilated, -1).astype(np.bool)
flow_mask = np.stack(flow_mask, -1).astype(np.bool)
if args.edge_guide:
print('Pytorch 1.6.0 for edge completion')
try:
from edgeconnect.networks import EdgeGenerator_
EdgeGenerator = EdgeGenerator_()
EdgeComp_ckpt = torch.load(args.edge_completion_model)
EdgeGenerator.load_state_dict(EdgeComp_ckpt['generator'])
EdgeGenerator.to(torch.device('cuda:0'))
EdgeGenerator.eval()
except:
EdgeGenerator = None
# Edge completion.
FlowF_edge = edge_completion(args, EdgeGenerator, corrFlowF, flow_mask,
'forward')
FlowB_edge = edge_completion(args, EdgeGenerator, corrFlowB, flow_mask,
'backward')
else:
FlowF_edge, FlowB_edge = None, None
# Completes the flow.
videoFlowF = complete_flow(args, corrFlowF, flow_mask, 'forward',
FlowF_edge)
videoFlowB = complete_flow(args, corrFlowB, flow_mask, 'backward',
FlowB_edge)
# Prepare gradients
gradient_x = np.empty(((imgH, imgW, 3, 0)), dtype=np.float32)
gradient_y = np.empty(((imgH, imgW, 3, 0)), dtype=np.float32)
for indFrame in range(nFrame):
img = video[:, :, :, indFrame]
img[mask[:, :, indFrame], :] = 0
img = cv2.inpaint(
(img * 255).astype(np.uint8), mask[:, :, indFrame].astype(
np.uint8), 3, cv2.INPAINT_TELEA).astype(np.float32) / 255.
gradient_x_ = np.concatenate(
(np.diff(img, axis=1), np.zeros((imgH, 1, 3), dtype=np.float32)),
axis=1)
gradient_y_ = np.concatenate(
(np.diff(img, axis=0), np.zeros((1, imgW, 3), dtype=np.float32)),
axis=0)
gradient_x = np.concatenate(
(gradient_x, gradient_x_.reshape(imgH, imgW, 3, 1)), axis=-1)
gradient_y = np.concatenate(
(gradient_y, gradient_y_.reshape(imgH, imgW, 3, 1)), axis=-1)
gradient_x[mask_dilated[:, :, indFrame], :, indFrame] = 0
gradient_y[mask_dilated[:, :, indFrame], :, indFrame] = 0
iter = 0
mask_tofill = mask
gradient_x_filled = gradient_x # corrupted gradient_x, mask_gradient indicates the missing gradient region
gradient_y_filled = gradient_y # corrupted gradient_y, mask_gradient indicates the missing gradient region
mask_gradient = mask_dilated
video_comp = video
try:
from spatial_inpaint import spatial_inpaint
from frame_inpaint import DeepFillv1
deepfill = DeepFillv1(
pretrained_model='/home/chengao/Weight/imagenet_deepfill.pth',
image_shape=[imgH, imgW])
except:
print('Please switch to Pytorch 0.4.0')
return
while ((np.sum(mask) > 0) and iter <= 5):
create_dir(
os.path.join(args.outroot, 'frame_seamless_comp_' + str(iter)))
# Gradient propagation.
gradient_x_filled, gradient_y_filled, mask_gradient = \
get_flowNN_gradient(args,
gradient_x_filled,
gradient_y_filled,
mask,
mask_gradient,
videoFlowF,
videoFlowB,
None,
None)
# if there exist holes in mask, Poisson blending will fail. So I did this trick. I sacrifice some value. Another solution is to modify Poisson blending.
for indFrame in range(nFrame):
mask_gradient[:, :, indFrame] = scipy.ndimage.binary_fill_holes(
mask_gradient[:, :, indFrame]).astype(np.bool)
# After one gradient propagation iteration
# gradient --> RGB
for indFrame in range(nFrame):
print("Poisson blending frame {0:3d}".format(indFrame))
if mask[:, :, indFrame].sum() > 0:
try:
flowBlend, UnfilledMask = Poisson_blend_img(
video_comp[:, :, :, indFrame],
gradient_x_filled[:, 0:imgW - 1, :, indFrame],
gradient_y_filled[0:imgH - 1, :, :, indFrame],
mask[:, :, indFrame], mask_gradient[:, :, indFrame])
# UnfilledMask = scipy.ndimage.binary_fill_holes(UnfilledMask).astype(np.bool)
except:
flowBlend, UnfilledMask = video_comp[:, :, :,
indFrame], mask[:, :,
indFrame]
flowBlend = np.clip(flowBlend, 0, 1.0)
tmp = cv2.inpaint((flowBlend * 255).astype(np.uint8),
UnfilledMask.astype(np.uint8), 3,
cv2.INPAINT_TELEA).astype(np.float32) / 255.
flowBlend[UnfilledMask, :] = tmp[UnfilledMask, :]
video_comp[:, :, :, indFrame] = flowBlend
mask[:, :, indFrame] = UnfilledMask
flowBlend_ = copy.deepcopy(flowBlend)
flowBlend_[mask[:, :, indFrame], :] = [0, 1., 0]
else:
flowBlend_ = video_comp[:, :, :, indFrame]
cv2.imwrite(
os.path.join(args.outroot, 'frame_seamless_comp_' + str(iter),
'%05d.png' % indFrame), flowBlend_ * 255.)
video_comp_ = (video_comp * 255).astype(np.uint8).transpose(
3, 0, 1, 2)[:, :, :, ::-1]
imageio.mimwrite(os.path.join(args.outroot,
'frame_seamless_comp_' + str(iter),
'video_extrapolation.mp4'),
video_comp_,
fps=12,
quality=8,
macro_block_size=1)
imageio.mimsave(os.path.join(args.outroot,
'frame_seamless_comp_' + str(iter),
'video_extrapolation.gif'),
video_comp_,
format='gif',
fps=12)
mask, video_comp = spatial_inpaint(deepfill, mask, video_comp)
iter += 1
# Re-calculate gradient_x/y_filled and mask_gradient
for indFrame in range(nFrame):
mask_gradient[:, :, indFrame] = gradient_mask(mask[:, :, indFrame])
gradient_x_filled[:, :, :, indFrame] = np.concatenate(
(np.diff(video_comp[:, :, :, indFrame],
axis=1), np.zeros((imgH, 1, 3), dtype=np.float32)),
axis=1)
gradient_y_filled[:, :, :, indFrame] = np.concatenate(
(np.diff(video_comp[:, :, :, indFrame],
axis=0), np.zeros((1, imgW, 3), dtype=np.float32)),
axis=0)
gradient_x_filled[mask_gradient[:, :, indFrame], :, indFrame] = 0
gradient_y_filled[mask_gradient[:, :, indFrame], :, indFrame] = 0
def video_completion(args):
print('Pytorch 1.6.0 for flow prediction')
try:
RAFT_model = initialize_RAFT(args)
except:
RAFT_model = None
# Loads frames.
filename_list = glob.glob(os.path.join(args.path, '*.png')) + \
glob.glob(os.path.join(args.path, '*.jpg'))
# Obtains imgH, imgW and nFrame.
imgH, imgW = np.array(Image.open(filename_list[0])).shape[:2]
nFrame = len(filename_list)
# Loads video.
video = []
for filename in sorted(filename_list):
video.append(
torch.from_numpy(np.array(Image.open(filename)).astype(
np.uint8)).permute(2, 0, 1).float())
video = torch.stack(video, dim=0)
video = video.to('cuda')
# Calcutes the corrupted flow.
corrFlowF = calculate_flow(args, RAFT_model, video, 'forward')
corrFlowB = calculate_flow(args, RAFT_model, video, 'backward')
print('\nFinish Calculating flow.')
# Makes sure video is in BGR (opencv) format.
video = video.permute(2, 3, 1, 0).cpu().numpy()[:, :, ::-1, :] / 255.
if args.mode == 'video_extrapolation':
# Creates video and flow where the extrapolated region are missing.
video, corrFlowF, corrFlowB, flow_mask, mask_dilated, start_point, end_point = extrapolation(
args, video, corrFlowF, corrFlowB)
imgH, imgW = video.shape[:2]
# mask indicating the missing region in the video.
mask = np.tile(flow_mask[..., None], (1, 1, nFrame))
flow_mask = np.tile(flow_mask[..., None], (1, 1, nFrame))
else:
# Loads masks.
filename_list = glob.glob(os.path.join(args.path_mask, '*.png')) + \
glob.glob(os.path.join(args.path_mask, '*.jpg'))
mask = []
flow_mask = []
for filename in sorted(filename_list):
mask_img = np.array(Image.open(filename).convert('L'))
mask.append(mask_img)
# Dilate 15 pixel so that all known pixel is trustworthy
flow_mask_img = scipy.ndimage.binary_dilation(mask_img,
iterations=15)
# Close the small holes inside the foreground objects
flow_mask_img = cv2.morphologyEx(flow_mask_img.astype(np.uint8),
cv2.MORPH_CLOSE,
np.ones((21, 21),
np.uint8)).astype(np.bool)
flow_mask_img = scipy.ndimage.binary_fill_holes(
flow_mask_img).astype(np.bool)
flow_mask.append(flow_mask_img)
# mask indicating the missing region in the video.
mask = np.stack(mask, -1).astype(np.bool)
flow_mask = np.stack(flow_mask, -1).astype(np.bool)
if args.edge_guide:
print('Pytorch 1.6.0 for edge completion')
try:
from edgeconnect.networks import EdgeGenerator_
EdgeGenerator = EdgeGenerator_()
EdgeComp_ckpt = torch.load(args.edge_completion_model)
EdgeGenerator.load_state_dict(EdgeComp_ckpt['generator'])
EdgeGenerator.to(torch.device('cuda:0'))
EdgeGenerator.eval()
except:
EdgeGenerator = None
# Edge completion.
FlowF_edge = edge_completion(args, EdgeGenerator, corrFlowF, flow_mask,
'forward')
FlowB_edge = edge_completion(args, EdgeGenerator, corrFlowB, flow_mask,
'backward')
else:
FlowF_edge, FlowB_edge = None, None
# Completes the flow.
videoFlowF = complete_flow(args, corrFlowF, flow_mask, 'forward',
FlowF_edge)
videoFlowB = complete_flow(args, corrFlowB, flow_mask, 'backward',
FlowB_edge)
iter = 0
mask_tofill = mask
video_comp = video
try:
from spatial_inpaint import spatial_inpaint
from frame_inpaint import DeepFillv1
deepfill = DeepFillv1(
pretrained_model='/home/chengao/Weight/imagenet_deepfill.pth',
image_shape=[imgH, imgW])
except:
print('Please switch to Pytorch 0.4.0')
return
while ((np.sum(mask_tofill) > 0) and iter <= 5):
create_dir(os.path.join(args.outroot, 'frame_comp_' + str(iter)))
# Color propagation.
video_comp, mask_tofill, _ = get_flowNN(args, video_comp, mask_tofill,
videoFlowF, videoFlowB, None,
None)
for i in range(nFrame):
mask_tofill[:, :,
i] = scipy.ndimage.binary_dilation(mask_tofill[:, :,
i],
iterations=2)
img = video_comp[:, :, :, i] * 255
img[mask_tofill[:, :, i]] = [0, 255, 0]
cv2.imwrite(
os.path.join(args.outroot, 'frame_comp_' + str(iter),
'%05d.png' % i), img)
video_comp_ = (video_comp * 255).astype(np.uint8).transpose(
3, 0, 1, 2)[:, :, :, ::-1]
imageio.mimwrite(os.path.join(args.outroot, 'frame_comp_' + str(iter),
'video_extrapolation.mp4'),
video_comp_,
fps=12,
quality=8,
macro_block_size=1)
imageio.mimsave(os.path.join(args.outroot, 'frame_comp_' + str(iter),
'video_extrapolation.gif'),
video_comp_,
format='gif',
fps=12)
mask_tofill, video_comp = spatial_inpaint(deepfill, mask_tofill,
video_comp)
iter += 1
def video_completion(args):
# Flow model.
RAFT_model = initialize_RAFT(args)
# Loads frames.
filename_list = glob.glob(os.path.join(args.path, '*.png')) + \
glob.glob(os.path.join(args.path, '*.jpg'))
# Obtains imgH, imgW and nFrame.
imgH, imgW = np.array(Image.open(filename_list[0])).shape[:2]
nFrame = len(filename_list)
# Loads video.
video = []
for filename in sorted(filename_list):
video.append(
torch.from_numpy(
np.array(Image.open(filename)).astype(
np.uint8)[..., :3]).permute(2, 0, 1).float())
video = torch.stack(video, dim=0)
video = video.to('cuda')
# Calcutes the corrupted flow.
corrFlowF, corrFlowB, corrFlowNLF, corrFlowNLB = calculate_flow(
args, RAFT_model, video)
print('\nFinish flow prediction.')
# Makes sure video is in BGR (opencv) format.
video = video.permute(2, 3, 1, 0).cpu().numpy()[:, :, ::-1, :] / 255.
if args.mode == 'video_extrapolation':
# Creates video and flow where the extrapolated region are missing.
video, corrFlowF, corrFlowB, corrFlowNLF, corrFlowNLB, flow_mask, mask_dilated, start_point, end_point = extrapolation(
args, video, corrFlowF, corrFlowB, corrFlowNLF, corrFlowNLB)
imgH, imgW = video.shape[:2]
# mask indicating the missing region in the video.
mask = np.tile(flow_mask[..., None], (1, 1, nFrame))
flow_mask = np.tile(flow_mask[..., None], (1, 1, nFrame))
else:
# Loads masks.
filename_list = glob.glob(os.path.join(args.path_mask, '*.png')) + \
glob.glob(os.path.join(args.path_mask, '*.jpg'))
mask = []
flow_mask = []
for filename in sorted(filename_list):
mask_img = np.array(Image.open(filename).convert('L'))
mask.append(mask_img)
# Dilate 15 pixel so that all known pixel is trustworthy
flow_mask_img = scipy.ndimage.binary_dilation(mask_img,
iterations=15)
# Close the small holes inside the foreground objects
flow_mask_img = cv2.morphologyEx(flow_mask_img.astype(np.uint8),
cv2.MORPH_CLOSE,
np.ones((21, 21),
np.uint8)).astype(bool)
flow_mask_img = scipy.ndimage.binary_fill_holes(
flow_mask_img).astype(bool)
flow_mask.append(flow_mask_img)
# mask indicating the missing region in the video.
mask = np.stack(mask, -1).astype(bool)
flow_mask = np.stack(flow_mask, -1).astype(bool)
if args.edge_guide:
# Edge completion model.
EdgeGenerator = EdgeGenerator_()
EdgeComp_ckpt = torch.load(args.edge_completion_model)
EdgeGenerator.load_state_dict(EdgeComp_ckpt['generator'])
EdgeGenerator.to(torch.device('cuda:0'))
EdgeGenerator.eval()
# Edge completion.
FlowF_edge = edge_completion(args, EdgeGenerator, corrFlowF, flow_mask,
'forward')
FlowB_edge = edge_completion(args, EdgeGenerator, corrFlowB, flow_mask,
'backward')
print('\nFinish edge completion.')
else:
FlowF_edge, FlowB_edge = None, None
# Completes the flow.
videoFlowF = complete_flow(args, corrFlowF, flow_mask, 'forward',
FlowF_edge)
videoFlowB = complete_flow(args, corrFlowB, flow_mask, 'backward',
FlowB_edge)
if args.Nonlocal:
videoNonLocalFlowF = complete_flow(args, corrFlowNLF, flow_mask,
'nonlocal_forward', None)
videoNonLocalFlowB = complete_flow(args, corrFlowNLB, flow_mask,
'nonlocal_backward', None)
else:
videoNonLocalFlowF = None
videoNonLocalFlowB = None
print('\nFinish flow completion.')
iter = 0
mask_tofill = mask
video_comp = video
# Image inpainting model.
deepfill = DeepFillv1(pretrained_model=args.deepfill_model,
image_shape=[imgH, imgW])
# We iteratively complete the video.
while (np.sum(mask_tofill) > 0):
create_dir(os.path.join(args.outroot, 'frame_comp_' + str(iter)))
# Color propagation.
video_comp, mask_tofill, _ = get_flowNN(args, video_comp, mask_tofill,
videoFlowF, videoFlowB,
videoNonLocalFlowF,
videoNonLocalFlowB)
for i in range(nFrame):
mask_tofill[:, :,
i] = scipy.ndimage.binary_dilation(mask_tofill[:, :,
i],
iterations=2)
img = video_comp[:, :, :, i] * 255
# Green indicates the regions that are not filled yet.
img[mask_tofill[:, :, i]] = [0, 255, 0]
cv2.imwrite(
os.path.join(args.outroot, 'frame_comp_' + str(iter),
'%05d.png' % i), img)
# video_comp_ = (video_comp * 255).astype(np.uint8).transpose(3, 0, 1, 2)[:, :, :, ::-1]
# imageio.mimwrite(os.path.join(args.outroot, 'frame_comp_' + str(iter), 'intermediate_{0}.mp4'.format(str(iter))), video_comp_, fps=12, quality=8, macro_block_size=1)
# imageio.mimsave(os.path.join(args.outroot, 'frame_comp_' + str(iter), 'intermediate_{0}.gif'.format(str(iter))), video_comp_, format='gif', fps=12)
mask_tofill, video_comp = spatial_inpaint(deepfill, mask_tofill,
video_comp)
iter += 1
create_dir(os.path.join(args.outroot, 'frame_comp_' + 'final'))
video_comp_ = (video_comp * 255).astype(np.uint8).transpose(
3, 0, 1, 2)[:, :, :, ::-1]
for i in range(nFrame):
img = video_comp[:, :, :, i] * 255
cv2.imwrite(
os.path.join(args.outroot, 'frame_comp_' + 'final',
'%05d.png' % i), img)
imageio.mimwrite(os.path.join(args.outroot, 'frame_comp_' + 'final',
'final.mp4'),
video_comp_,
fps=12,
quality=8,
macro_block_size=1)
def video_completion_seamless(args):
os.environ["CUDA_VISIBLE_DEVICES"] = '0,1,2,3'
# Flow model.
RAFT_model = initialize_RAFT(args)
# Loads frames.
filename_list = glob.glob(os.path.join(args.path, '*.png')) + \
glob.glob(os.path.join(args.path, '*.jpg'))
# Obtains imgH, imgW and nFrame.
imgH, imgW = np.array(Image.open(filename_list[0])).shape[:2]
nFrame = len(filename_list)
# Loads video.
video = []
for filename in sorted(filename_list):
video.append(
torch.from_numpy(np.array(Image.open(filename)).astype(
np.uint8)).permute(2, 0, 1).float())
video = torch.stack(video, dim=0)
video = video.to('cuda')
# Loads masks.
filename_list = glob.glob(os.path.join(args.path_mask, '*.png')) + \
glob.glob(os.path.join(args.path_mask, '*.jpg'))
mask = []
mask_dilated = []
flow_mask = []
for filename in sorted(filename_list):
mask_img = np.array(Image.open(filename).convert('L'))
flow_mask_img = scipy.ndimage.binary_dilation(mask_img, iterations=3)
# Close the small holes inside the foreground objects
flow_mask_img = cv2.morphologyEx(flow_mask_img.astype(np.uint8),
cv2.MORPH_CLOSE,
np.ones((11, 11),
np.uint8)).astype(np.bool)
flow_mask_img = scipy.ndimage.binary_fill_holes(flow_mask_img).astype(
np.bool)
flow_mask.append(flow_mask_img)
# Dilate a little bit
mask_img = scipy.ndimage.binary_dilation(mask_img, iterations=3)
mask_img = scipy.ndimage.binary_fill_holes(mask_img).astype(np.bool)
mask.append(mask_img)
mask_dilated.append(gradient_mask(mask_img))
minbbox_tl, minbbox_br = find_minbbox(flow_mask)
# Image inpainting model.
deepfill = DeepFillv1(pretrained_model=args.deepfill_model,
image_shape=[imgH, imgW])
# timer
time_start = time.time()
# Calcutes the corrupted flow.
corrFlowF = calculate_flow(args, RAFT_model, video, 'forward')
corrFlowB = calculate_flow(args, RAFT_model, video, 'backward')
print('\nFinish flow prediction.')
# Makes sure video is in BGR (opencv) format.
video = video.permute(2, 3, 1, 0).cpu().numpy()[:, :, ::-1, :] / 255.
# mask indicating the missing region in the video.
mask = np.stack(mask, -1).astype(np.bool)
mask_dilated = np.stack(mask_dilated, -1).astype(np.bool)
flow_mask = np.stack(flow_mask, -1).astype(np.bool)
if args.edge_guide:
# Edge completion model.
EdgeGenerator = EdgeGenerator_()
EdgeComp_ckpt = torch.load(args.edge_completion_model)
EdgeGenerator.load_state_dict(EdgeComp_ckpt['generator'])
EdgeGenerator.to(torch.device('cuda:1'))
EdgeGenerator.eval()
# Edge completion.
FlowF_edge = edge_completion(args, EdgeGenerator, corrFlowF, flow_mask,
'forward')
FlowB_edge = edge_completion(args, EdgeGenerator, corrFlowB, flow_mask,
'backward')
print('\nFinish edge completion.')
else:
FlowF_edge, FlowB_edge = None, None
# Completes the flow.
videoFlowF = complete_flow(args, corrFlowF, flow_mask, 'forward',
minbbox_tl, minbbox_br, FlowF_edge)
videoFlowB = complete_flow(args, corrFlowB, flow_mask, 'backward',
minbbox_tl, minbbox_br, FlowB_edge)
print('\nFinish flow completion.')
# Prepare gradients
gradient_x = np.empty(((imgH, imgW, 3, 0)), dtype=np.float32)
gradient_y = np.empty(((imgH, imgW, 3, 0)), dtype=np.float32)
for indFrame in range(nFrame):
img = video[:, :, :, indFrame]
img[mask[:, :, indFrame], :] = 0
img = cv2.inpaint(
(img * 255).astype(np.uint8), mask[:, :, indFrame].astype(
np.uint8), 3, cv2.INPAINT_TELEA).astype(np.float32) / 255.
gradient_x_ = np.concatenate(
(np.diff(img, axis=1), np.zeros((imgH, 1, 3), dtype=np.float32)),
axis=1)
gradient_y_ = np.concatenate(
(np.diff(img, axis=0), np.zeros((1, imgW, 3), dtype=np.float32)),
axis=0)
gradient_x = np.concatenate(
(gradient_x, gradient_x_.reshape(imgH, imgW, 3, 1)), axis=-1)
gradient_y = np.concatenate(
(gradient_y, gradient_y_.reshape(imgH, imgW, 3, 1)), axis=-1)
gradient_x[mask_dilated[:, :, indFrame], :, indFrame] = 0
gradient_y[mask_dilated[:, :, indFrame], :, indFrame] = 0
iter = 0
mask_tofill = mask
gradient_x_filled = gradient_x # corrupted gradient_x, mask_gradient indicates the missing gradient region
gradient_y_filled = gradient_y # corrupted gradient_y, mask_gradient indicates the missing gradient region
mask_gradient = mask_dilated
video_comp = video
# We iteratively complete the video.
while (np.sum(mask) > 0):
# create_dir(os.path.join(args.outroot, 'frame_seamless_comp_' + str(iter)))
# Gradient propagation.
gradient_x_filled, gradient_y_filled, mask_gradient = \
get_flowNN_gradient(args,
gradient_x_filled,
gradient_y_filled,
mask,
mask_gradient,
videoFlowF,
videoFlowB,
None,
None)
# if there exist holes in mask, Poisson blending will fail. So I did this trick. I sacrifice some value. Another solution is to modify Poisson blending.
for indFrame in range(nFrame):
mask_gradient[:, :, indFrame] = scipy.ndimage.binary_fill_holes(
mask_gradient[:, :, indFrame]).astype(np.bool)
# After one gradient propagation iteration
# gradient --> RGB
for indFrame in range(nFrame):
print("Poisson blending frame {0:3d}".format(indFrame))
if mask[:, :, indFrame].sum() > 0:
try:
video_comp_crop = video_comp[
minbbox_tl[indFrame][1]:minbbox_br[indFrame][1],
minbbox_tl[indFrame][0]:minbbox_br[indFrame][0], :,
indFrame]
gradient_x_filled_crop = gradient_x_filled[
minbbox_tl[indFrame][1]:minbbox_br[indFrame][1],
minbbox_tl[indFrame][0]:minbbox_br[indFrame][0] - 1, :,
indFrame]
gradient_y_filled_crop = gradient_y_filled[
minbbox_tl[indFrame][1]:minbbox_br[indFrame][1] - 1,
minbbox_tl[indFrame][0]:minbbox_br[indFrame][0], :,
indFrame]
mask_crop = mask[
minbbox_tl[indFrame][1]:minbbox_br[indFrame][1],
minbbox_tl[indFrame][0]:minbbox_br[indFrame][0],
indFrame]
mask_gradient_crop = mask_gradient[
minbbox_tl[indFrame][1]:minbbox_br[indFrame][1],
minbbox_tl[indFrame][0]:minbbox_br[indFrame][0],
indFrame]
frameBlend_crop, UnfilledMask_crop = Poisson_blend_img(
video_comp_crop, gradient_x_filled_crop,
gradient_y_filled_crop, mask_crop, mask_gradient_crop)
frameBlend, UnfilledMask = video_comp[:, :, :,
indFrame], mask[:, :,
indFrame]
frameBlend[minbbox_tl[indFrame][1]:minbbox_br[indFrame][1],
minbbox_tl[indFrame][0]:minbbox_br[indFrame]
[0], :] = frameBlend_crop
UnfilledMask[
minbbox_tl[indFrame][1]:minbbox_br[indFrame][1],
minbbox_tl[indFrame][0]:minbbox_br[indFrame]
[0]] = UnfilledMask_crop
# UnfilledMask = scipy.ndimage.binary_fill_holes(UnfilledMask).astype(np.bool)
# frameBlend, UnfilledMask = Poisson_blend_img(video_comp[:, :, :, indFrame], gradient_x_filled[:, 0 : imgW - 1, :, indFrame], gradient_y_filled[0 : imgH - 1, :, :, indFrame], mask[:, :, indFrame], mask_gradient[:, :, indFrame])
# UnfilledMask = scipy.ndimage.binary_fill_holes(UnfilledMask).astype(np.bool)
except:
frameBlend, UnfilledMask = video_comp[:, :, :,
indFrame], mask[:, :,
indFrame]
frameBlend = np.clip(frameBlend, 0, 1.0)
tmp = cv2.inpaint((frameBlend * 255).astype(np.uint8),
UnfilledMask.astype(np.uint8), 3,
cv2.INPAINT_TELEA).astype(np.float32) / 255.
frameBlend[UnfilledMask, :] = tmp[UnfilledMask, :]
video_comp[:, :, :, indFrame] = frameBlend
mask[:, :, indFrame] = UnfilledMask
frameBlend_ = copy.deepcopy(frameBlend)
# Green indicates the regions that are not filled yet.
frameBlend_[mask[:, :, indFrame], :] = [0, 1., 0]
else:
frameBlend_ = video_comp[:, :, :, indFrame]
# cv2.imwrite(os.path.join(args.outroot, 'frame_seamless_comp_' + str(iter), '%05d.png'%indFrame), frameBlend_ * 255.)
# video_comp_ = (video_comp * 255).astype(np.uint8).transpose(3, 0, 1, 2)[:, :, :, ::-1]
# imageio.mimwrite(os.path.join(args.outroot, 'frame_seamless_comp_' + str(iter), 'intermediate_{0}.mp4'.format(str(iter))), video_comp_, fps=12, quality=8, macro_block_size=1)
# imageio.mimsave(os.path.join(args.outroot, 'frame_seamless_comp_' + str(iter), 'intermediate_{0}.gif'.format(str(iter))), video_comp_, format='gif', fps=12)
mask, video_comp = spatial_inpaint(deepfill, mask, video_comp)
iter += 1
# Re-calculate gradient_x/y_filled and mask_gradient
for indFrame in range(nFrame):
mask_gradient[:, :, indFrame] = gradient_mask(mask[:, :, indFrame])
gradient_x_filled[:, :, :, indFrame] = np.concatenate(
(np.diff(video_comp[:, :, :, indFrame],
axis=1), np.zeros((imgH, 1, 3), dtype=np.float32)),
axis=1)
gradient_y_filled[:, :, :, indFrame] = np.concatenate(
(np.diff(video_comp[:, :, :, indFrame],
axis=0), np.zeros((1, imgW, 3), dtype=np.float32)),
axis=0)
gradient_x_filled[mask_gradient[:, :, indFrame], :, indFrame] = 0
gradient_y_filled[mask_gradient[:, :, indFrame], :, indFrame] = 0
video_comp_ = (video_comp * 255).astype(np.uint8).transpose(
3, 0, 1, 2)[:, :, :, ::-1]
time_end = time.time()
print('time cost', time_end - time_start, 's')
# write out
create_dir(os.path.join(args.outroot, 'frame_seamless_comp_' + 'final'))
for i in range(nFrame):
img = video_comp[:, :, :, i] * 255
cv2.imwrite(
os.path.join(args.outroot, 'frame_seamless_comp_' + 'final',
'%06d.png' % i), img)
def video_completion_seamless(args):
if args.gpu_profile == 'high':
num_gpus = 0.25
elif args.gpu_profile == 'medium':
num_gpus = 0.5
else:
num_gpus = 1
create_dirs(args)
zdata = zarr.open_group(os.path.join(args.outroot, 'datasets.zarr'),
mode='a')
rargs = ray.put(args)
# Flow model.
RAFT_model = initialize_RAFT(args)
rRAFT_model = ray.put(RAFT_model)
# Loads frames.
filename_list = glob.glob(os.path.join(args.path, '*.png')) + \
glob.glob(os.path.join(args.path, '*.jpg'))
filename_list = sorted(filename_list)
# Obtains imgH, imgW and nFrame.
imgH, imgW = cv2.imread(filename_list[0]).shape[:2]
nFrame = len(filename_list)
if args.mode == 'object_removal' and not divisible_by((
imgH,
imgW,
), 8):
raise Exception('Height and width must be divisible by 8')
if args.mode == 'video_extrapolation':
# Defines new FOV.
imgH_orig, imgW_orig = imgH, imgW
imgH = int(8 * round((args.H_scale * imgH) / 8))
imgW = int(8 * round((args.W_scale * imgW) / 8))
H_start = int((imgH - imgH_orig) / 2)
W_start = int((imgW - imgW_orig) / 2)
shape = (
slice(H_start, H_start + imgH_orig),
slice(W_start, W_start + imgW_orig),
)
else:
shape = (
slice(None, None),
slice(None, None),
)
#Create datasets
zchecksum = zdata.require_dataset('checksum',
shape=(10, nFrame),
chunks=(10, 1),
fill_value=0,
dtype=np.int)
zvideo = zdata.require_dataset('video',
shape=(imgH, imgW, 3, nFrame),
chunks=(imgH, imgW, 3, 1),
fill_value=0,
dtype=np.uint8) #0
zflow = zdata.require_dataset('flow',
shape=(imgH, imgW, 2, 2, nFrame - 1),
chunks=(imgH, imgW, 2, 2, 1),
fill_value=0,
dtype=np.float32) #1
zmasks = zdata.require_dataset('masks',
shape=(imgH, imgW, nFrame),
chunks=(imgH, imgW, 1),
fill_value=0,
dtype=np.bool) #3
zflow_masks = zdata.require_dataset('flow_masks',
shape=(imgH, imgW, nFrame),
chunks=(imgH, imgW, 1),
fill_value=0,
dtype=np.bool) #4
zmasks_dilated = zdata.require_dataset('masks_dilated',
shape=(imgH, imgW, nFrame),
chunks=(imgH, imgW, 1),
fill_value=0,
dtype=np.bool) #5
zflow_comp = zdata.require_dataset('flow_comp',
shape=(imgH, imgW, 2, 2, nFrame - 1),
chunks=(imgH, imgW, 2, 2, 1),
fill_value=0,
dtype=np.float32) #2
zgradient_x = zdata.require_dataset('gradient_x',
shape=(imgH, imgW, 3, nFrame),
chunks=(imgH, imgW, 3, 1),
fill_value=0,
dtype=np.float32) #6
zgradient_y = zdata.require_dataset('gradient_y',
shape=(imgH, imgW, 3, nFrame),
chunks=(imgH, imgW, 3, 1),
fill_value=0,
dtype=np.float32) #7
zedges = zdata.require_dataset(
'edges',
shape=(imgH, imgW, 2, nFrame - 1),
chunks=(imgH, imgW, 2, 1),
fill_value=0,
dtype=np.float32) if args.edge_guide else None #8
if args.force:
zchecksum[:] = 0
rzdataset = ray.put(zdata)
# Import frames.
for frame in trange(nFrame, desc="Importing frames"):
if not valid_checksum(zdata, zvideo[..., frame], 0, frame):
zvideo[shape[0], shape[1], :,
frame] = cv2.imread(filename_list[frame])
save_checksum(zdata, zvideo[..., frame], 0, frame)
rayProgressBar([
calculate_flow.options(num_gpus=num_gpus,
name='calculate_flow({})'.format(frame)).remote(
rargs, rRAFT_model, rzdataset, shape, frame)
for frame in range(nFrame - 1)
], 'Calculating flows')
if args.mode == 'video_extrapolation':
# Creates video and flow where the extrapolated region are missing.
rayProgressBar([
extrapolation.options(
name='prepare_masks({})'.format(frame)).remote(
rargs, rzdataset, shape, frame) for frame in range(nFrame)
], 'Processing masks')
else:
rayProgressBar([
prepare_masks.options(
name='prepare_masks({})'.format(frame)).remote(
rargs, rzdataset, frame) for frame in range(nFrame)
], 'Processing masks')
if args.edge_guide:
# Edge completion model.
EdgeGenerator = EdgeGenerator_()
EdgeComp_ckpt = torch.load(args.edge_completion_model)
EdgeGenerator.load_state_dict(EdgeComp_ckpt['generator'])
EdgeGenerator.to(torch.device('cuda:0'))
EdgeGenerator.eval()
rEdgeGenerator = ray.put(EdgeGenerator)
#zedges = zarr.open(os.path.join(args.outroot, 'edges_comp.zarr'), mode='w', shape=(imgH, imgW, 2, nFrame),
#chunks=(imgH, imgW, 2, 1), dtype=np.float32)
# Edge completion.
rayProgressBar([
edge_completion.options(
num_gpus=num_gpus,
name='edge_completion({})'.format(frame)).remote(
rEdgeGenerator, rzdataset, frame)
for frame in range(nFrame - 1)
], 'Completing edges')
rayProgressBar([
complete_flow.options(name='complete_flow({})'.format(frame)).remote(
rargs, rzdataset, frame) for frame in range(nFrame - 1)
], 'Completing flows')
rayProgressBar([
get_gradients.options(name='get_gradients({})'.format(frame)).remote(
rzdataset, frame) for frame in range(nFrame)
], 'Calculating gradients')
print(
'\nFlow completion finished.\nYou may run python 2-video-completion.py --outroot {}'
.format(args.outroot))