def vid_img(opt):
output_dir = (opt.output_dir + "/" + name(opt.input.content) + "_" +
"_".join([name(s) for s in opt.input.style.split(",")]))
flow_model = flow.get_flow_model(opt)
frames = load.process_content_video(flow_model, opt)
content_size = np.array(load.preprocess(frames[0]).size()[-2:])
style_images_big = load.process_style_images(opt)
for size_n, (current_size,
num_iters) in enumerate(zip(*determine_scaling(opt.param))):
print("\nCurrent size {}px".format(current_size))
os.makedirs(output_dir + "/" + str(current_size), exist_ok=True)
content_scale = current_size / max(*content_size)
if current_size <= 1024:
opt.model.gpu = 0
opt.model.multidevice = False
else:
opt.model.gpu = "0,1"
opt.model.multidevice = True
opt.param.tv_weight = 0
net = models.load_model(opt.model, opt.param)
net.set_style_targets(style_images_big, content_scale * content_size,
opt)
for pass_n in range(opt.param.passes_per_scale):
init_image = None
for (prev_frame, this_frame) in zip(frames,
frames[1:] + frames[:1]):
# TODO add update_style() function to support changing styles per frame
opt.output = "%s/%s/%s_%s.png" % (output_dir, current_size,
pass_n + 1, name(this_frame))
if os.path.isfile(opt.output):
print(
"Skipping pass: %s, frame: %s. File already exists." %
(pass_n + 1, name(this_frame)))
continue
print("Optimizing... size: %s, pass: %s, frame: %s" %
(current_size, pass_n + 1, name(this_frame)))
content_frames = [
F.interpolate(load.preprocess(prev_frame),
scale_factor=content_scale,
mode="bilinear",
align_corners=False),
F.interpolate(load.preprocess(this_frame),
scale_factor=content_scale,
mode="bilinear",
align_corners=False),
]
content_frames = [
match_histogram(frame, style_images_big[0])
for frame in content_frames
]
net.set_content_targets(content_frames[1], opt)
# Initialize the image
# TODO make sure initialization correct even when continuing half way through video stylization
if size_n == 0 and pass_n == 0:
if opt.input.init == "random":
init_image = th.randn(
content_frames[1].size()).mul(0.001)
elif opt.input.init == "prev_warp":
flo_file = "%s/flow/forward_%s_%s.flo" % (
output_dir, name(prev_frame), name(this_frame))
flow_map = load.flow_warp_map(flo_file)
if init_image is None:
init_image = content_frames[0]
init_image = F.grid_sample(init_image,
flow_map,
padding_mode="border")
else:
init_image = content_frames[1].clone()
else:
if pass_n == 0:
# load images from last pass of previous size
if init_image is None:
ifile = "%s/%s/%s_%s.png" % (
output_dir,
prev_size,
opt.param.passes_per_scale,
name(prev_frame),
)
init_image = load.preprocess(ifile)
init_image = F.interpolate(
init_image,
size=content_frames[0].size()[2:],
mode="bilinear",
align_corners=False)
bfile = "%s/%s/%s_%s.png" % (
output_dir,
prev_size,
opt.param.passes_per_scale,
name(this_frame),
)
blend_image = load.preprocess(bfile)
blend_image = F.interpolate(
blend_image,
size=content_frames[0].size()[2:],
mode="bilinear",
align_corners=False)
else:
# load images from previous pass of current size
if init_image is None:
ifile = "%s/%s/%s_%s.png" % (output_dir,
current_size, pass_n,
name(prev_frame))
init_image = load.preprocess(ifile)
bfile = "%s/%s/%s_%s.png" % (output_dir, current_size,
pass_n, name(this_frame))
blend_image = load.preprocess(bfile)
direction = "forward" if pass_n % 2 == 0 else "backward"
flo_file = f"{output_dir}/flow/{direction}_{name(prev_frame)}_{name(this_frame)}.flo"
flow_map = load.flow_warp_map(flo_file)
flow_map = F.interpolate(flow_map.permute(0, 3, 1, 2),
size=init_image.size()[2:],
mode="bilinear").permute(
0, 2, 3, 1)
warp_image = F.grid_sample(init_image,
flow_map,
padding_mode="border")
flow_weight_file = f"{output_dir}/flow/{direction}_{name(prev_frame)}_{name(this_frame)}.png"
reliable_flow = load.reliable_flow_weighting(
flow_weight_file)
reliable_flow = F.interpolate(reliable_flow,
size=init_image.size()[2:],
mode="bilinear",
align_corners=False)
net.set_temporal_targets(warp_image,
warp_weights=reliable_flow,
opt=opt)
blend_init_image = (
1 - opt.param.blend_weight
) * blend_image + opt.param.blend_weight * init_image
warp_blend_init_image = F.grid_sample(
blend_init_image, flow_map, padding_mode="border")
init_image = warp_blend_init_image
output_image = style.optimize(
content_frames, style_images_big, init_image,
num_iters // opt.param.passes_per_scale, opt)
init_image = match_histogram(output_image.detach().cpu(),
style_images_big[0])
disp = load.deprocess(init_image.clone())
if opt.param.original_colors == 1:
disp = load.original_colors(
load.deprocess(content_frames[1].clone()), disp)
disp.save(str(opt.output))
# clean up / prepare for next pass
frames = frames[7:] + frames[:7] # rotate frames
frames = list(reversed(frames))
ffmpeg.input(output_dir + "/" + str(current_size) + "/" + str(pass_n) +
"_%05d.png").output(
"%s/%s_%s.mp4" %
(output_dir, name(output_dir), current_size),
**opt.ffmpeg).overwrite_output().run()
prev_size = current_size
del net
th.cuda.empty_cache()
ffmpeg.input("{output_dir}/{current_size}/{pass_n}_%05d.png").output(
opt.output, **opt.ffmpeg).overwrite_output().run()
def img_img(opt):
style_images_big = load.process_style_images(opt)
content_image_big = match_histogram(load.preprocess(opt.input.content),
style_images_big)
content_size = np.array(content_image_big.size()[-2:])
if opt.input.init != "content" and opt.input.init != "random":
pastiche = load.preprocess(opt.input.init)
else:
pastiche = None
for (current_size, num_iters) in zip(*determine_scaling(opt.param)):
# print("\nCurrent size {}px".format(current_size))
opt.pbar.set_description(
f"{opt.output.split('/')[-1]} @ {current_size}px")
# scale content image
content_scale = current_size / max(*content_size)
if not content_scale == 1:
content_image = F.interpolate(content_image_big,
scale_factor=content_scale,
mode="bilinear",
align_corners=False)
# scale style images
style_images = []
content_area = content_image.shape[2] * content_image.shape[3]
for img in style_images_big:
style_scale = math.sqrt(
content_area /
(img.size(3) * img.size(2))) * opt.param.style_scale
style_images.append(
F.interpolate(th.clone(img),
scale_factor=style_scale,
mode="bilinear",
align_corners=False))
# Initialize the pastiche image
if opt.input.init == "random" and pastiche is None:
H, W = content_image.shape[2:]
pastiche = th.randn(1, 3, H, W).mul(0.001)
elif opt.input.init == "content" and pastiche is None:
pastiche = F.interpolate(
content_image_big.clone(),
tuple(np.int64(content_image.shape[2:])),
mode="bilinear",
align_corners=False,
)
else:
pastiche = F.interpolate(pastiche.clone(),
tuple(np.int64(content_image.shape[2:])),
mode="bilinear",
align_corners=False)
pastiche = match_histogram(pastiche, style_images_big)
output_image = style.optimize(content_image, style_images, pastiche,
num_iters, opt)
pastiche = match_histogram(output_image.detach().cpu(),
style_images_big)
load.save_tensor_to_file(pastiche.detach().cpu(), opt)
def img_img(args):
style_images_big = load.process_style_images(args)
content_image_big = match_histogram(load.preprocess(args.content),
style_images_big,
mode=args.match_histograms)
content_size = np.array(content_image_big.size()[-2:])
if args.init != "content" and args.init != "random":
pastiche = load.preprocess(args.init)
else:
pastiche = None
for (current_size, num_iters) in zip(args.image_sizes, args.num_iters):
print("\nCurrent size {}px".format(current_size))
if os.path.exists(f"{args.output}_{current_size}.png"):
pastiche = load.preprocess(f"{args.output}_{current_size}.png")
continue
# scale content image
content_scale = current_size / max(*content_size)
content_image = F.interpolate(content_image_big,
scale_factor=content_scale,
mode="bilinear",
align_corners=False)
# scale style images
style_images = []
content_area = content_image.shape[2] * content_image.shape[3]
for img in style_images_big:
style_scale = math.sqrt(
content_area / (img.size(3) * img.size(2))) * args.style_scale
style_images.append(
F.interpolate(th.clone(img),
scale_factor=style_scale,
mode="bilinear",
align_corners=False))
# Initialize the pastiche image
if args.init == "random" and pastiche is None:
H, W = content_image.shape[2:]
pastiche = th.randn(1, 3, H, W).mul(0.001)
elif args.init == "content" and pastiche is None:
pastiche = F.interpolate(
content_image_big.clone(),
tuple(np.int64(content_image.shape[2:])),
mode="bilinear",
align_corners=False,
)
else:
pastiche = F.interpolate(pastiche.clone(),
tuple(np.int64(content_image.shape[2:])),
mode="bilinear",
align_corners=False)
pastiche = match_histogram(pastiche,
style_images_big,
mode=args.match_histograms)
output_image = optim.optimize(content_image, style_images, pastiche,
num_iters, args)
pastiche = match_histogram(output_image.detach().cpu(),
style_images_big,
mode=args.match_histograms)
load.save_tensor_to_file(pastiche.detach().cpu(),
args,
size=current_size)
args = config.get_args()
if args.seed >= 0:
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
if args.backend == "cudnn":
torch.backends.cudnn.deterministic = True
output_dir = args.output_dir + "/" + name(args.content) + "_" + "_".join(
[name(s) for s in args.style])
flow_model = flow.get_flow_model(args)
frames = load.process_content_video(flow_model, args)
content_size = np.array(load.preprocess(frames[0]).size()[-2:])
style_images_big = load.process_style_images(args)
for size_n, (current_size,
num_iters) in enumerate(zip(args.image_sizes, args.num_iters)):
if len(
glob("%s/%s/*.png" % (output_dir, args.image_sizes[min(
len(args.image_sizes) - 1, size_n + 1)]))) == len(frames):
print("Skipping size: %s, already done." % current_size)
prev_size = current_size
continue
print("\nCurrent size {}px".format(current_size))
os.makedirs(output_dir + "/" + str(current_size), exist_ok=True)
content_scale = current_size / max(*content_size)
def vid_img(args):
output_dir = args.output_dir + "/" + name(args.content) + "_" + "_".join(
[name(s) for s in args.style])
flow_model = flow.get_flow_model(args)
frames = load.process_content_video(flow_model, args)
content_size = np.array(load.preprocess(frames[0]).size()[-2:])
style_images_big = load.process_style_images(args)
for size_n, (current_size,
num_iters) in enumerate(zip(args.image_sizes,
args.num_iters)):
if len(
glob.glob("%s/%s/*.png" % (output_dir, args.image_sizes[min(
len(args.image_sizes) - 1, size_n + 1)]))) == len(frames):
print("Skipping size: %s, already done." % current_size)
prev_size = current_size
continue
print("\nCurrent size {}px".format(current_size))
os.makedirs(output_dir + "/" + str(current_size), exist_ok=True)
content_scale = current_size / max(*content_size)
# scale style images
style_images = []
content_area = content_scale**2 * content_size[0] * content_size[1]
for img in style_images_big:
style_scale = math.sqrt(
content_area / (img.size(3) * img.size(2))) * args.style_scale
style_images.append(
F.interpolate(th.clone(img),
scale_factor=style_scale,
mode="bilinear",
align_corners=False))
optim.set_model_args(args, current_size)
net, losses = models.load_model(args)
# optim.set_style_targets(net, style_images, args)
for pass_n in range(args.passes_per_scale):
pastiche = None
if args.loop:
start_idx = random.randrange(0, len(frames) - 1)
frames = frames[
start_idx:] + frames[:start_idx] # rotate frames
if len(
glob.glob("%s/%s/%s_*.png" % (output_dir, current_size,
pass_n + 2))) == len(frames):
print(f"Skipping pass: {pass_n + 1}, already done.")
frames = list(reversed(frames))
continue
for n, (prev_frame, this_frame) in enumerate(
zip(frames + frames[:11 if args.loop else 1],
frames[1:] + frames[:10 if args.loop else 1])):
# TODO add update_style() function to support changing styles per frame
args.output = "%s/%s/%s_%s.png" % (
output_dir, current_size, pass_n + 1, name(this_frame))
if os.path.isfile(args.output) and not n >= len(frames):
print(
"Skipping pass: %s, frame: %s. File already exists." %
(pass_n + 1, name(this_frame)))
continue
print("Optimizing... size: %s, pass: %s, frame: %s" %
(current_size, pass_n + 1, name(this_frame)))
content_frames = [
F.interpolate(load.preprocess(prev_frame),
scale_factor=content_scale,
mode="bilinear",
align_corners=False),
F.interpolate(load.preprocess(this_frame),
scale_factor=content_scale,
mode="bilinear",
align_corners=False),
]
content_frames = [
match_histogram(frame,
style_images_big[0],
mode=args.match_histograms)
for frame in content_frames
]
flow_direction = "forward" if pass_n % 2 == 0 else "backward"
# optim.set_content_targets(net, content_frames[1], args)
# Initialize the image
# TODO make sure initialization correct even when continuing half way through video stylization
if size_n == 0 and pass_n == 0:
if args.init == "random":
pastiche = th.randn(
content_frames[1].size()).mul(0.001)
elif args.init == "prev_warp":
if pastiche is None:
pastiche = content_frames[0]
flo_file = f"{output_dir}/flow/{flow_direction}_{name(prev_frame)}_{name(this_frame)}.flo"
flow_map = load.flow_warp_map(flo_file,
pastiche.shape[2:])
pastiche = F.grid_sample(pastiche,
flow_map,
padding_mode="border")
else:
pastiche = content_frames[1].clone()
else:
if pass_n == 0:
# load images from last pass of previous size
if pastiche is None:
ifile = "%s/%s/%s_%s.png" % (
output_dir,
prev_size
if n <= len(frames) else current_size,
args.passes_per_scale
if n <= len(frames) else pass_n + 1,
name(prev_frame),
)
pastiche = load.preprocess(ifile)
pastiche = F.interpolate(
pastiche,
size=content_frames[0].size()[2:],
mode="bilinear",
align_corners=False)
bfile = "%s/%s/%s_%s.png" % (
output_dir,
prev_size if n <= len(frames) else current_size,
args.passes_per_scale
if n <= len(frames) else pass_n + 1,
name(this_frame),
)
blend_image = load.preprocess(bfile)
blend_image = F.interpolate(
blend_image,
size=content_frames[0].size()[2:],
mode="bilinear",
align_corners=False)
else:
# load images from previous pass of current size
if pastiche is None:
ifile = "%s/%s/%s_%s.png" % (
output_dir,
current_size,
pass_n if n <= len(frames) else pass_n + 1,
name(prev_frame),
)
pastiche = load.preprocess(ifile)
bfile = "%s/%s/%s_%s.png" % (
output_dir,
current_size,
pass_n if n <= len(frames) else pass_n + 1,
name(this_frame),
)
blend_image = load.preprocess(bfile)
flo_file = f"{output_dir}/flow/{flow_direction}_{name(prev_frame)}_{name(this_frame)}.flo"
flow_map = load.flow_warp_map(flo_file, pastiche.shape[2:])
warp_image = F.grid_sample(pastiche,
flow_map,
padding_mode="border")
flow_weight_file = f"{output_dir}/flow/{flow_direction}_{name(prev_frame)}_{name(this_frame)}.png"
reliable_flow = load.reliable_flow_weighting(
flow_weight_file)
reliable_flow = F.interpolate(reliable_flow,
size=pastiche.size()[2:],
mode="bilinear",
align_corners=False)
optim.set_temporal_targets(net,
warp_image,
warp_weights=reliable_flow,
args=args)
pastiche = (
1 - args.temporal_blend
) * blend_image + args.temporal_blend * pastiche
output_image = optim.optimize(
content_frames[1], style_images, pastiche,
num_iters // args.passes_per_scale, args, net, losses)
pastiche = match_histogram(output_image.detach().cpu(),
style_images_big[0],
mode=args.match_histograms)
disp = load.deprocess(pastiche.clone())
if args.original_colors == 1:
disp = load.original_colors(
load.deprocess(content_frames[1].clone()), disp)
disp.save(str(args.output))
# reverse frames for next pass
frames = list(reversed(frames))
ffmpeg.input(output_dir + "/" + str(current_size) + "/" + str(pass_n) +
"_%05d.png").output(
"%s/%s_%s.mp4" %
(output_dir, name(output_dir), current_size),
**args.ffmpeg).overwrite_output().run()
prev_size = current_size
del net
th.cuda.empty_cache()