def image_loader(image_name):
loader = transforms.Compose([
transforms.ToTensor()])
image = Image.open(image_name).resize((224,224), Image.ANTIALIAS)
# fake batch dimension required to fit network's input dimensions
image = loader(image).unsqueeze(0)
img = image.to(pydiffvg.get_device(), torch.float)
return img
def gen_image_from_curves(self, t, shapes, shape_groups, gamma, background_image):
render = pydiffvg.RenderFunction.apply
scene_args = pydiffvg.RenderFunction.serialize_scene( \
self.canvas_width, self.canvas_height, shapes, shape_groups)
img = render(self.canvas_width, self.canvas_height, 2, 2, t, background_image, *scene_args)
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3,
device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
img = img[:, :, :3]
dir_ = "./gens/"
if not os.path.exists(dir_):
os.mkdir(dir_)
if t % 200 == 1:
pydiffvg.imwrite(img.cpu(), os.path.join(dir_, 'iter_{}.png'.format(int(t / 5))), gamma=gamma)
img = img[:, :, :3]
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2)
return img
def main(target_path, svg_path, output_dir, num_iter=1000, use_lpips_loss=False):
perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
target = torch.from_numpy(skimage.io.imread(target_path, as_gray=False, pilmode="RGB")).to(torch.float32) / 255.0
print("target", target.size())
target = target.pow(gamma)
target = target.to(pydiffvg.get_device())
target = target.unsqueeze(0)
target = target.permute(0, 3, 1, 2) # NHWC -> NCHW
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(svg_path)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), f'{output_dir}/init.png', gamma=gamma)
points_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
color_vars = {}
for group in shape_groups:
group.fill_color.requires_grad = True
color_vars[group.fill_color.data_ptr()] = group.fill_color
# color_vars = list(color_vars.values())
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=0.1)
# color_optim = torch.optim.Adam(color_vars, lr=0.01)
# Adam iterations.
for t in range(num_iter):
print('iteration:', t)
points_optim.zero_grad()
# color_optim.zero_grad()
# Forward pass: render the image.
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
img = render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# Compose img with white background
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(), f'{output_dir}/iter_{t}.png', gamma=gamma)
img = img[:, :, :3]
# Convert img from HWC to NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
# print(img.size())
# print(target.size())
if use_lpips_loss:
loss = perception_loss(img, target)
else:
loss = (img - target).pow(2).mean()
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
points_optim.step()
# color_optim.step()
for group in shape_groups:
group.fill_color.data.clamp_(0.0, 1.0)
if t % 10 == 0 or t == num_iter - 1:
pydiffvg.save_svg_paths_only(f'{output_dir}/iter_{t}.svg',
canvas_width, canvas_height, shapes, shape_groups)
# Render the final result.
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
img = render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(), f'{output_dir}/final.png', gamma=gamma)
# Convert the intermediate renderings to a video.
from subprocess import call
call(["ffmpeg", "-framerate", "24", "-i",
f"{output_dir}/iter_%d.png", "-vb", "20M",
f"{output_dir}/out.mp4"])
def image_loader(image_name):
image = Image.open(image_name)
# fake batch dimension required to fit network's input dimensions
image = loader(image).unsqueeze(0)
return image.to(pydiffvg.get_device(), torch.float)
def main(args):
pydiffvg.set_use_gpu(torch.cuda.is_available())
canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(
args.content_file)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Transform to gamma space
pydiffvg.imwrite(img.cpu(), 'results/style_transfer/init.png', gamma=1.0)
# HWC -> NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
loader = transforms.Compose([transforms.ToTensor()
]) # transform it into a torch tensor
def image_loader(image_name):
image = Image.open(image_name)
# fake batch dimension required to fit network's input dimensions
image = loader(image).unsqueeze(0)
return image.to(pydiffvg.get_device(), torch.float)
style_img = image_loader(args.style_img)
# alpha blend content with a gray background
content_img = img[:, :3, :, :] * img[:, 3, :, :] + \
0.5 * torch.ones([1, 3, img.shape[2], img.shape[3]]) * \
(1 - img[:, 3, :, :])
assert style_img.size() == content_img.size(), \
"we need to import style and content images of the same size"
# unloader = transforms.ToPILImage() # reconvert into PIL image
class ContentLoss(nn.Module):
def __init__(
self,
target,
):
super(ContentLoss, self).__init__()
# we 'detach' the target content from the tree used
# to dynamically compute the gradient: this is a stated value,
# not a variable. Otherwise the forward method of the criterion
# will throw an error.
self.target = target.detach()
def forward(self, input):
self.loss = F.mse_loss(input, self.target)
return input
def gram_matrix(input):
a, b, c, d = input.size() # a=batch size(=1)
# b=number of feature maps
# (c,d)=dimensions of a f. map (N=c*d)
features = input.view(a * b, c * d) # resise F_XL into \hat F_XL
G = torch.mm(features, features.t()) # compute the gram product
# we 'normalize' the values of the gram matrix
# by dividing by the number of element in each feature maps.
return G.div(a * b * c * d)
class StyleLoss(nn.Module):
def __init__(self, target_feature):
super(StyleLoss, self).__init__()
self.target = gram_matrix(target_feature).detach()
def forward(self, input):
G = gram_matrix(input)
self.loss = F.mse_loss(G, self.target)
return input
device = pydiffvg.get_device()
cnn = models.vgg19(pretrained=True).features.to(device).eval()
cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)
# create a module to normalize input image so we can easily put it in a
# nn.Sequential
class Normalization(nn.Module):
def __init__(self, mean, std):
super(Normalization, self).__init__()
# .view the mean and std to make them [C x 1 x 1] so that they can
# directly work with image Tensor of shape [B x C x H x W].
# B is batch size. C is number of channels. H is height and W is width.
self.mean = mean.clone().view(-1, 1, 1)
self.std = std.clone().view(-1, 1, 1)
def forward(self, img):
# normalize img
return (img - self.mean) / self.std
# desired depth layers to compute style/content losses :
content_layers_default = ['conv_4']
style_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5']
def get_style_model_and_losses(cnn,
normalization_mean,
normalization_std,
style_img,
content_img,
content_layers=content_layers_default,
style_layers=style_layers_default):
cnn = copy.deepcopy(cnn)
# normalization module
normalization = Normalization(normalization_mean,
normalization_std).to(device)
# just in order to have an iterable access to or list of content/syle
# losses
content_losses = []
style_losses = []
# assuming that cnn is a nn.Sequential, so we make a new nn.Sequential
# to put in modules that are supposed to be activated sequentially
model = nn.Sequential(normalization)
i = 0 # increment every time we see a conv
for layer in cnn.children():
if isinstance(layer, nn.Conv2d):
i += 1
name = 'conv_{}'.format(i)
elif isinstance(layer, nn.ReLU):
name = 'relu_{}'.format(i)
# The in-place version doesn't play very nicely with the ContentLoss
# and StyleLoss we insert below. So we replace with out-of-place
# ones here.
layer = nn.ReLU(inplace=False)
elif isinstance(layer, nn.MaxPool2d):
name = 'pool_{}'.format(i)
elif isinstance(layer, nn.BatchNorm2d):
name = 'bn_{}'.format(i)
else:
raise RuntimeError('Unrecognized layer: {}'.format(
layer.__class__.__name__))
model.add_module(name, layer)
if name in content_layers:
# add content loss:
target = model(content_img).detach()
content_loss = ContentLoss(target)
model.add_module("content_loss_{}".format(i), content_loss)
content_losses.append(content_loss)
if name in style_layers:
# add style loss:
target_feature = model(style_img).detach()
style_loss = StyleLoss(target_feature)
model.add_module("style_loss_{}".format(i), style_loss)
style_losses.append(style_loss)
# now we trim off the layers after the last content and style losses
for i in range(len(model) - 1, -1, -1):
if isinstance(model[i], ContentLoss) or isinstance(
model[i], StyleLoss):
break
model = model[:(i + 1)]
return model, style_losses, content_losses
def run_style_transfer(cnn,
normalization_mean,
normalization_std,
content_img,
style_img,
canvas_width,
canvas_height,
shapes,
shape_groups,
num_steps=500,
style_weight=5000,
content_weight=1):
"""Run the style transfer."""
print('Building the style transfer model..')
model, style_losses, content_losses = get_style_model_and_losses(
cnn, normalization_mean, normalization_std, style_img, content_img)
point_params = []
color_params = []
stroke_width_params = []
for shape in shapes:
if isinstance(shape, pydiffvg.Path):
point_params.append(shape.points.requires_grad_())
stroke_width_params.append(shape.stroke_width.requires_grad_())
for shape_group in shape_groups:
if isinstance(shape_group.fill_color, torch.Tensor):
color_params.append(shape_group.fill_color.requires_grad_())
elif isinstance(shape_group.fill_color, pydiffvg.LinearGradient):
point_params.append(
shape_group.fill_color.begin.requires_grad_())
point_params.append(
shape_group.fill_color.end.requires_grad_())
color_params.append(
shape_group.fill_color.stop_colors.requires_grad_())
if isinstance(shape_group.stroke_color, torch.Tensor):
color_params.append(shape_group.stroke_color.requires_grad_())
elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient):
point_params.append(
shape_group.stroke_color.begin.requires_grad_())
point_params.append(
shape_group.stroke_color.end.requires_grad_())
color_params.append(
shape_group.stroke_color.stop_colors.requires_grad_())
point_optimizer = optim.Adam(point_params, lr=1.0)
color_optimizer = optim.Adam(color_params, lr=0.01)
stroke_width_optimizers = optim.Adam(stroke_width_params, lr=0.1)
print('Optimizing..')
run = [0]
while run[0] <= num_steps:
point_optimizer.zero_grad()
color_optimizer.zero_grad()
stroke_width_optimizers.zero_grad()
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# alpha blend img with a gray background
img = img[:, :, :3] * img[:, :, 3:4] + \
0.5 * torch.ones([img.shape[0], img.shape[1], 3]) * \
(1 - img[:, :, 3:4])
pydiffvg.imwrite(img.cpu(),
'results/style_transfer/step_{}.png'.format(
run[0]),
gamma=1.0)
# HWC to NCHW
img = img.permute([2, 0, 1]).unsqueeze(0)
model(img)
style_score = 0
content_score = 0
for sl in style_losses:
style_score += sl.loss
for cl in content_losses:
content_score += cl.loss
style_score *= style_weight
content_score *= content_weight
loss = style_score + content_score
loss.backward()
run[0] += 1
if run[0] % 1 == 0:
print("run {}:".format(run))
print('Style Loss : {:4f} Content Loss: {:4f}'.format(
style_score.item(), content_score.item()))
print()
point_optimizer.step()
color_optimizer.step()
stroke_width_optimizers.step()
for color in color_params:
color.data.clamp_(0, 1)
for w in stroke_width_params:
w.data.clamp_(0.5, 4.0)
return shapes, shape_groups
shapes, shape_groups = run_style_transfer(cnn, cnn_normalization_mean,
cnn_normalization_std,
content_img, style_img,
canvas_width, canvas_height,
shapes, shape_groups)
scene_args = pydiffvg.RenderFunction.serialize_scene(shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Transform to gamma space
pydiffvg.imwrite(img.cpu(), 'results/style_transfer/output.png', gamma=1.0)
def main(args):
case_name = args.svg_file.split('/')[-1].split('.')[0]
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(args.svg_file)
w = int(canvas_width * args.size_scale)
h = int(canvas_height * args.size_scale)
print(w, h)
curve_counts = 0
for s in shapes:
if isinstance(s, pydiffvg.Circle):
curve_counts += 1
elif isinstance(s, pydiffvg.Ellipse):
curve_counts += 1
elif isinstance(s, pydiffvg.Path):
curve_counts += len(s.num_control_points)
elif isinstance(s, pydiffvg.Polygon):
curve_counts += len(s.points) - 1
if s.is_closed:
curve_counts += 1
elif isinstance(s, pydiffvg.Rect):
curve_counts += 1
print('curve_counts:', curve_counts)
pfilter = pydiffvg.PixelFilter(type=diffvg.FilterType.box,
radius=torch.tensor(0.5))
use_prefiltering = args.use_prefiltering
print('use_prefiltering:', use_prefiltering)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
num_samples_x = args.num_spp
num_samples_y = args.num_spp
if (use_prefiltering):
num_samples_x = 1
num_samples_y = 1
render = pydiffvg.RenderFunction.apply
img = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
pydiffvg.imwrite(
img.cpu(),
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/img.png',
gamma=1.0)
epsilon = 0.1
def perturb_scene(axis, epsilon):
for s in shapes:
if isinstance(s, pydiffvg.Circle):
s.center[axis] += epsilon
elif isinstance(s, pydiffvg.Ellipse):
s.center[axis] += epsilon
elif isinstance(s, pydiffvg.Path):
s.points[:, axis] += epsilon
elif isinstance(s, pydiffvg.Polygon):
s.points[:, axis] += epsilon
elif isinstance(s, pydiffvg.Rect):
s.p_min[axis] += epsilon
s.p_max[axis] += epsilon
for s in shape_groups:
if isinstance(s.fill_color, pydiffvg.LinearGradient):
s.fill_color.begin[axis] += epsilon
s.fill_color.end[axis] += epsilon
perturb_scene(0, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render = pydiffvg.RenderFunction.apply
img0 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
perturb_scene(0, -2 * epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
img1 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
x_diff = (img0 - img1) / (2 * epsilon)
x_diff = x_diff.sum(axis=2)
x_diff_max = x_diff.max() * args.clamping_factor
x_diff_min = x_diff.min() * args.clamping_factor
print(x_diff.max())
print(x_diff.min())
x_diff = cm.viridis(
normalize(x_diff, x_diff_min, x_diff_max).cpu().numpy())
pydiffvg.imwrite(
x_diff,
f'results/finite_difference_comp//{case_name}_{use_prefiltering}/finite_x_diff.png',
gamma=1.0)
perturb_scene(0, epsilon)
perturb_scene(1, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render = pydiffvg.RenderFunction.apply
img0 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
perturb_scene(1, -2 * epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
img1 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
y_diff = (img0 - img1) / (2 * epsilon)
y_diff = y_diff.sum(axis=2)
y_diff_max = y_diff.max() * args.clamping_factor
y_diff_min = y_diff.min() * args.clamping_factor
y_diff = cm.viridis(
normalize(y_diff, y_diff_min, y_diff_max).cpu().numpy())
pydiffvg.imwrite(
y_diff,
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/finite_y_diff.png',
gamma=1.0)
perturb_scene(1, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render_grad = pydiffvg.RenderFunction.render_grad
img_grad = render_grad(
torch.ones(h, w, 4, device=pydiffvg.get_device()),
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
print(img_grad[:, :, 0].max())
print(img_grad[:, :, 0].min())
x_diff = cm.viridis(
normalize(img_grad[:, :, 0], x_diff_min, x_diff_max).cpu().numpy())
y_diff = cm.viridis(
normalize(img_grad[:, :, 1], y_diff_min, y_diff_max).cpu().numpy())
pydiffvg.imwrite(
x_diff,
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/ours_x_diff.png',
gamma=1.0)
pydiffvg.imwrite(
y_diff,
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/ours_y_diff.png',
gamma=1.0)
def render(canvas_width, canvas_height, shapes, shape_groups):
_render = pydiffvg.RenderFunction.apply
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
img = _render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# print(img.size())
img = img[:, :, 3:4] * img[:, :, :3] + th.ones(img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
return img
def main(args):
# Use GPU if available
pydiffvg.set_use_gpu(torch.cuda.is_available())
perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
#target = torch.from_numpy(skimage.io.imread('imgs/lena.png')).to(torch.float32) / 255.0
target = torch.from_numpy(skimage.io.imread(args.target)).to(
torch.float32) / 255.0
target = target.pow(gamma)
target = target.to(pydiffvg.get_device())
target = target.unsqueeze(0)
target = target.permute(0, 3, 1, 2) # NHWC -> NCHW
#target = torch.nn.functional.interpolate(target, size = [256, 256], mode = 'area')
canvas_width, canvas_height = target.shape[3], target.shape[2]
num_paths = args.num_paths
max_width = args.max_width
random.seed(1234)
torch.manual_seed(1234)
shapes = []
shape_groups = []
if args.use_blob:
for i in range(num_paths):
num_segments = random.randint(3, 5)
num_control_points = torch.zeros(num_segments,
dtype=torch.int32) + 2
points = []
p0 = (random.random(), random.random())
points.append(p0)
for j in range(num_segments):
radius = 0.05
p1 = (p0[0] + radius * (random.random() - 0.5),
p0[1] + radius * (random.random() - 0.5))
p2 = (p1[0] + radius * (random.random() - 0.5),
p1[1] + radius * (random.random() - 0.5))
p3 = (p2[0] + radius * (random.random() - 0.5),
p2[1] + radius * (random.random() - 0.5))
points.append(p1)
points.append(p2)
if j < num_segments - 1:
points.append(p3)
p0 = p3
points = torch.tensor(points)
points[:, 0] *= canvas_width
points[:, 1] *= canvas_height
path = pydiffvg.Path(num_control_points=num_control_points,
points=points,
stroke_width=torch.tensor(1.0),
is_closed=True)
shapes.append(path)
path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor(
[len(shapes) - 1]),
fill_color=torch.tensor([
random.random(),
random.random(),
random.random(),
random.random()
]))
shape_groups.append(path_group)
else:
for i in range(num_paths):
num_segments = random.randint(1, 3)
num_control_points = torch.zeros(num_segments,
dtype=torch.int32) + 2
points = []
p0 = (random.random(), random.random())
points.append(p0)
for j in range(num_segments):
radius = 0.05
p1 = (p0[0] + radius * (random.random() - 0.5),
p0[1] + radius * (random.random() - 0.5))
p2 = (p1[0] + radius * (random.random() - 0.5),
p1[1] + radius * (random.random() - 0.5))
p3 = (p2[0] + radius * (random.random() - 0.5),
p2[1] + radius * (random.random() - 0.5))
points.append(p1)
points.append(p2)
points.append(p3)
p0 = p3
points = torch.tensor(points)
points[:, 0] *= canvas_width
points[:, 1] *= canvas_height
#points = torch.rand(3 * num_segments + 1, 2) * min(canvas_width, canvas_height)
path = pydiffvg.Path(num_control_points=num_control_points,
points=points,
stroke_width=torch.tensor(1.0),
is_closed=False)
shapes.append(path)
path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor(
[len(shapes) - 1]),
fill_color=None,
stroke_color=torch.tensor([
random.random(),
random.random(),
random.random(),
random.random()
]))
shape_groups.append(path_group)
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
pydiffvg.imwrite(img.cpu(),
'results/painterly_rendering/init.png',
gamma=gamma)
points_vars = []
stroke_width_vars = []
color_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
if not args.use_blob:
for path in shapes:
path.stroke_width.requires_grad = True
stroke_width_vars.append(path.stroke_width)
if args.use_blob:
for group in shape_groups:
group.fill_color.requires_grad = True
color_vars.append(group.fill_color)
else:
for group in shape_groups:
group.stroke_color.requires_grad = True
color_vars.append(group.stroke_color)
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=1.0)
if len(stroke_width_vars) > 0:
width_optim = torch.optim.Adam(stroke_width_vars, lr=0.1)
color_optim = torch.optim.Adam(color_vars, lr=0.01)
# Adam iterations.
for t in range(args.num_iter):
print('iteration:', t)
points_optim.zero_grad()
if len(stroke_width_vars) > 0:
width_optim.zero_grad()
color_optim.zero_grad()
# Forward pass: render the image.
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
t, # seed
None,
*scene_args)
# Compose img with white background
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(
img.shape[0], img.shape[1], 3,
device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'results/painterly_rendering/iter_{}.png'.format(t),
gamma=gamma)
img = img[:, :, :3]
# Convert img from HWC to NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
if args.use_lpips_loss:
loss = perception_loss(
img, target) + (img.mean() - target.mean()).pow(2)
else:
loss = (img - target).pow(2).mean()
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
points_optim.step()
if len(stroke_width_vars) > 0:
width_optim.step()
color_optim.step()
if len(stroke_width_vars) > 0:
for path in shapes:
path.stroke_width.data.clamp_(1.0, max_width)
if args.use_blob:
for group in shape_groups:
group.fill_color.data.clamp_(0.0, 1.0)
else:
for group in shape_groups:
group.stroke_color.data.clamp_(0.0, 1.0)
if t % 10 == 0 or t == args.num_iter - 1:
pydiffvg.save_svg(
'results/painterly_rendering/iter_{}.svg'.format(t),
canvas_width, canvas_height, shapes, shape_groups)
# Render the final result.
img = render(
target.shape[1], # width
target.shape[0], # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'results/painterly_rendering/final.png'.format(t),
gamma=gamma)
# Convert the intermediate renderings to a video.
from subprocess import call
call([
"ffmpeg", "-framerate", "24", "-i",
"results/painterly_rendering/iter_%d.png", "-vb", "20M",
"results/painterly_rendering/out.mp4"
])
def render_grad(grad_img,
width,
height,
num_samples_x,
num_samples_y,
seed,
background_image,
*args):
if not grad_img.is_contiguous():
grad_img = grad_img.contiguous()
assert(torch.isfinite(grad_img).all())
# Unpack arguments
current_index = 0
canvas_width = args[current_index]
current_index += 1
canvas_height = args[current_index]
current_index += 1
num_shapes = args[current_index]
current_index += 1
num_shape_groups = args[current_index]
current_index += 1
output_type = args[current_index]
current_index += 1
use_prefiltering = args[current_index]
current_index += 1
eval_positions = args[current_index]
current_index += 1
shapes = []
shape_groups = []
shape_contents = [] # Important to avoid GC deleting the shapes
color_contents = [] # Same as above
for shape_id in range(num_shapes):
shape_type = args[current_index]
current_index += 1
if shape_type == diffvg.ShapeType.circle:
radius = args[current_index]
current_index += 1
center = args[current_index]
current_index += 1
shape = diffvg.Circle(radius, diffvg.Vector2f(center[0], center[1]))
elif shape_type == diffvg.ShapeType.ellipse:
radius = args[current_index]
current_index += 1
center = args[current_index]
current_index += 1
shape = diffvg.Ellipse(diffvg.Vector2f(radius[0], radius[1]),
diffvg.Vector2f(center[0], center[1]))
elif shape_type == diffvg.ShapeType.path:
num_control_points = args[current_index]
current_index += 1
points = args[current_index]
current_index += 1
thickness = args[current_index]
current_index += 1
is_closed = args[current_index]
current_index += 1
use_distance_approx = args[current_index]
current_index += 1
shape = diffvg.Path(diffvg.int_ptr(num_control_points.data_ptr()),
diffvg.float_ptr(points.data_ptr()),
diffvg.float_ptr(thickness.data_ptr() if thickness is not None else 0),
num_control_points.shape[0],
points.shape[0],
is_closed,
use_distance_approx)
elif shape_type == diffvg.ShapeType.rect:
p_min = args[current_index]
current_index += 1
p_max = args[current_index]
current_index += 1
shape = diffvg.Rect(diffvg.Vector2f(p_min[0], p_min[1]),
diffvg.Vector2f(p_max[0], p_max[1]))
else:
assert(False)
stroke_width = args[current_index]
current_index += 1
shapes.append(diffvg.Shape(
shape_type, shape.get_ptr(), stroke_width.item()))
shape_contents.append(shape)
for shape_group_id in range(num_shape_groups):
shape_ids = args[current_index]
current_index += 1
fill_color_type = args[current_index]
current_index += 1
if fill_color_type == diffvg.ColorType.constant:
color = args[current_index]
current_index += 1
fill_color = diffvg.Constant(
diffvg.Vector4f(color[0], color[1], color[2], color[3]))
elif fill_color_type == diffvg.ColorType.linear_gradient:
beg = args[current_index]
current_index += 1
end = args[current_index]
current_index += 1
offsets = args[current_index]
current_index += 1
stop_colors = args[current_index]
current_index += 1
assert(offsets.shape[0] == stop_colors.shape[0])
fill_color = diffvg.LinearGradient(diffvg.Vector2f(beg[0], beg[1]),
diffvg.Vector2f(end[0], end[1]),
offsets.shape[0],
diffvg.float_ptr(offsets.data_ptr()),
diffvg.float_ptr(stop_colors.data_ptr()))
elif fill_color_type == diffvg.ColorType.radial_gradient:
center = args[current_index]
current_index += 1
radius = args[current_index]
current_index += 1
offsets = args[current_index]
current_index += 1
stop_colors = args[current_index]
current_index += 1
assert(offsets.shape[0] == stop_colors.shape[0])
fill_color = diffvg.RadialGradient(diffvg.Vector2f(center[0], center[1]),
diffvg.Vector2f(radius[0], radius[1]),
offsets.shape[0],
diffvg.float_ptr(offsets.data_ptr()),
diffvg.float_ptr(stop_colors.data_ptr()))
elif fill_color_type is None:
fill_color = None
else:
assert(False)
stroke_color_type = args[current_index]
current_index += 1
if stroke_color_type == diffvg.ColorType.constant:
color = args[current_index]
current_index += 1
stroke_color = diffvg.Constant(
diffvg.Vector4f(color[0], color[1], color[2], color[3]))
elif stroke_color_type == diffvg.ColorType.linear_gradient:
beg = args[current_index]
current_index += 1
end = args[current_index]
current_index += 1
offsets = args[current_index]
current_index += 1
stop_colors = args[current_index]
current_index += 1
assert(offsets.shape[0] == stop_colors.shape[0])
stroke_color = diffvg.LinearGradient(diffvg.Vector2f(beg[0], beg[1]),
diffvg.Vector2f(end[0], end[1]),
offsets.shape[0],
diffvg.float_ptr(offsets.data_ptr()),
diffvg.float_ptr(stop_colors.data_ptr()))
elif stroke_color_type == diffvg.ColorType.radial_gradient:
center = args[current_index]
current_index += 1
radius = args[current_index]
current_index += 1
offsets = args[current_index]
current_index += 1
stop_colors = args[current_index]
current_index += 1
assert(offsets.shape[0] == stop_colors.shape[0])
stroke_color = diffvg.RadialGradient(diffvg.Vector2f(center[0], center[1]),
diffvg.Vector2f(radius[0], radius[1]),
offsets.shape[0],
diffvg.float_ptr(offsets.data_ptr()),
diffvg.float_ptr(stop_colors.data_ptr()))
elif stroke_color_type is None:
stroke_color = None
else:
assert(False)
use_even_odd_rule = args[current_index]
current_index += 1
shape_to_canvas = args[current_index]
current_index += 1
if fill_color is not None:
color_contents.append(fill_color)
if stroke_color is not None:
color_contents.append(stroke_color)
shape_groups.append(diffvg.ShapeGroup(
diffvg.int_ptr(shape_ids.data_ptr()),
shape_ids.shape[0],
diffvg.ColorType.constant if fill_color_type is None else fill_color_type,
diffvg.void_ptr(0) if fill_color is None else fill_color.get_ptr(),
diffvg.ColorType.constant if stroke_color_type is None else stroke_color_type,
diffvg.void_ptr(0) if stroke_color is None else stroke_color.get_ptr(),
use_even_odd_rule,
diffvg.float_ptr(shape_to_canvas.data_ptr())))
filter_type = args[current_index]
current_index += 1
filter_radius = args[current_index]
current_index += 1
filt = diffvg.Filter(filter_type, filter_radius)
scene = diffvg.Scene(canvas_width, canvas_height,
shapes, shape_groups, filt, pydiffvg.get_use_gpu(),
pydiffvg.get_device().index if pydiffvg.get_device().index is not None else -1)
if output_type == OutputType.color:
assert(grad_img.shape[2] == 4)
else:
assert(grad_img.shape[2] == 1)
if background_image is not None:
background_image = background_image.to(pydiffvg.get_device())
if background_image.shape[2] == 3:
background_image = torch.cat((
background_image, torch.ones(background_image.shape[0], background_image.shape[1], 1,
device=background_image.device)), dim=2)
background_image = background_image.contiguous()
# assert(background_image.shape[0] == rendered_image.shape[0])
# assert(background_image.shape[1] == rendered_image.shape[1])
assert(background_image.shape[2] == 4)
translation_grad_image = \
torch.zeros(height, width, 2, device=pydiffvg.get_device())
start = time.time()
diffvg.render(scene,
diffvg.float_ptr(background_image.data_ptr() if background_image is not None else 0),
diffvg.float_ptr(0), # render_image
diffvg.float_ptr(0), # render_sdf
width,
height,
num_samples_x,
num_samples_y,
seed,
diffvg.float_ptr(0), # d_background_image
diffvg.float_ptr(grad_img.data_ptr() if output_type == OutputType.color else 0),
diffvg.float_ptr(grad_img.data_ptr() if output_type == OutputType.sdf else 0),
diffvg.float_ptr(translation_grad_image.data_ptr()),
use_prefiltering,
diffvg.float_ptr(eval_positions.data_ptr()),
eval_positions.shape[0])
time_elapsed = time.time() - start
if print_timing:
print('Gradient pass, time: %.5f s' % time_elapsed)
assert(torch.isfinite(translation_grad_image).all())
return translation_grad_image
def serialize_scene(canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pydiffvg.PixelFilter(type=diffvg.FilterType.box,
radius=torch.tensor(0.5)),
output_type=OutputType.color,
use_prefiltering=False,
eval_positions=torch.tensor([])):
"""
Given a list of shapes, convert them to a linear list of argument,
so that we can use it in PyTorch.
"""
num_shapes = len(shapes)
num_shape_groups = len(shape_groups)
args = []
args.append(canvas_width)
args.append(canvas_height)
args.append(num_shapes)
args.append(num_shape_groups)
args.append(output_type)
args.append(use_prefiltering)
args.append(eval_positions.to(pydiffvg.get_device()))
for shape in shapes:
use_thickness = False
if isinstance(shape, pydiffvg.Circle):
assert(shape.center.is_contiguous())
args.append(diffvg.ShapeType.circle)
args.append(shape.radius.cpu())
args.append(shape.center.cpu())
elif isinstance(shape, pydiffvg.Ellipse):
assert(shape.radius.is_contiguous())
assert(shape.center.is_contiguous())
args.append(diffvg.ShapeType.ellipse)
args.append(shape.radius.cpu())
args.append(shape.center.cpu())
elif isinstance(shape, pydiffvg.Path):
assert(shape.num_control_points.is_contiguous())
assert(shape.points.is_contiguous())
assert(shape.points.shape[1] == 2)
assert(torch.isfinite(shape.points).all())
args.append(diffvg.ShapeType.path)
args.append(shape.num_control_points.to(torch.int32).cpu())
args.append(shape.points.cpu())
if len(shape.stroke_width.shape) > 0 and shape.stroke_width.shape[0] > 1:
assert(torch.isfinite(shape.stroke_width).all())
use_thickness = True
args.append(shape.stroke_width.cpu())
else:
args.append(None)
args.append(shape.is_closed)
args.append(shape.use_distance_approx)
elif isinstance(shape, pydiffvg.Polygon):
assert(shape.points.is_contiguous())
assert(shape.points.shape[1] == 2)
args.append(diffvg.ShapeType.path)
if shape.is_closed:
args.append(torch.zeros(shape.points.shape[0], dtype=torch.int32))
else:
args.append(torch.zeros(shape.points.shape[0] - 1, dtype=torch.int32))
args.append(shape.points.cpu())
args.append(None)
args.append(shape.is_closed)
args.append(False) # use_distance_approx
elif isinstance(shape, pydiffvg.Rect):
assert(shape.p_min.is_contiguous())
assert(shape.p_max.is_contiguous())
args.append(diffvg.ShapeType.rect)
args.append(shape.p_min.cpu())
args.append(shape.p_max.cpu())
else:
assert(False)
if use_thickness:
args.append(torch.tensor(0.0))
else:
args.append(shape.stroke_width.cpu())
for shape_group in shape_groups:
assert(shape_group.shape_ids.is_contiguous())
args.append(shape_group.shape_ids.to(torch.int32).cpu())
# Fill color
if shape_group.fill_color is None:
args.append(None)
elif isinstance(shape_group.fill_color, torch.Tensor):
assert(shape_group.fill_color.is_contiguous())
args.append(diffvg.ColorType.constant)
args.append(shape_group.fill_color.cpu())
elif isinstance(shape_group.fill_color, pydiffvg.LinearGradient):
assert(shape_group.fill_color.begin.is_contiguous())
assert(shape_group.fill_color.end.is_contiguous())
assert(shape_group.fill_color.offsets.is_contiguous())
assert(shape_group.fill_color.stop_colors.is_contiguous())
args.append(diffvg.ColorType.linear_gradient)
args.append(shape_group.fill_color.begin.cpu())
args.append(shape_group.fill_color.end.cpu())
args.append(shape_group.fill_color.offsets.cpu())
args.append(shape_group.fill_color.stop_colors.cpu())
elif isinstance(shape_group.fill_color, pydiffvg.RadialGradient):
assert(shape_group.fill_color.center.is_contiguous())
assert(shape_group.fill_color.radius.is_contiguous())
assert(shape_group.fill_color.offsets.is_contiguous())
assert(shape_group.fill_color.stop_colors.is_contiguous())
args.append(diffvg.ColorType.radial_gradient)
args.append(shape_group.fill_color.center.cpu())
args.append(shape_group.fill_color.radius.cpu())
args.append(shape_group.fill_color.offsets.cpu())
args.append(shape_group.fill_color.stop_colors.cpu())
if shape_group.fill_color is not None:
# go through the underlying shapes and check if they are all closed
for shape_id in shape_group.shape_ids:
if isinstance(shapes[shape_id], pydiffvg.Path):
if not shapes[shape_id].is_closed:
warnings.warn("Detected non-closed paths with fill color. This might causes unexpected results.", Warning)
# Stroke color
if shape_group.stroke_color is None:
args.append(None)
elif isinstance(shape_group.stroke_color, torch.Tensor):
assert(shape_group.stroke_color.is_contiguous())
args.append(diffvg.ColorType.constant)
args.append(shape_group.stroke_color.cpu())
elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient):
assert(shape_group.stroke_color.begin.is_contiguous())
assert(shape_group.stroke_color.end.is_contiguous())
assert(shape_group.stroke_color.offsets.is_contiguous())
assert(shape_group.stroke_color.stop_colors.is_contiguous())
assert(torch.isfinite(shape_group.stroke_color.stop_colors).all())
args.append(diffvg.ColorType.linear_gradient)
args.append(shape_group.stroke_color.begin.cpu())
args.append(shape_group.stroke_color.end.cpu())
args.append(shape_group.stroke_color.offsets.cpu())
args.append(shape_group.stroke_color.stop_colors.cpu())
elif isinstance(shape_group.stroke_color, pydiffvg.RadialGradient):
assert(shape_group.stroke_color.center.is_contiguous())
assert(shape_group.stroke_color.radius.is_contiguous())
assert(shape_group.stroke_color.offsets.is_contiguous())
assert(shape_group.stroke_color.stop_colors.is_contiguous())
assert(torch.isfinite(shape_group.stroke_color.stop_colors).all())
args.append(diffvg.ColorType.radial_gradient)
args.append(shape_group.stroke_color.center.cpu())
args.append(shape_group.stroke_color.radius.cpu())
args.append(shape_group.stroke_color.offsets.cpu())
args.append(shape_group.stroke_color.stop_colors.cpu())
args.append(shape_group.use_even_odd_rule)
# Transformation
args.append(shape_group.shape_to_canvas.contiguous().cpu())
args.append(filter.type)
args.append(filter.radius.cpu())
return args
def fit_to_image(self, img_path, n_iterations=100, use_perc_loss=False):
"""
This method updates internal pathes to visually match provided image.
It is an updated version of "painterly rendering" https://github.com/BachiLi/diffvg/blob/master/apps/painterly_rendering.py
:param img_path:
:param n_iterations:
:return:
"""
image = skimage.io.imread(img_path)
image = skimage.transform.resize(
image, (self.canvas_width, self.canvas_height))
target = torch.from_numpy(image).float().cuda()
target = target.permute(2, 0, 1).unsqueeze(0)
plt.imshow(image)
plt.title('Target image')
plt.show()
optim_points = torch.optim.Adam(self.point_variables, lr=1.0)
optim_widths = torch.optim.Adam(self.widths_variables, lr=0.1)
optim_color = torch.optim.Adam(self.color_variables, lr=0.05)
perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
for iteration in range(n_iterations):
optim_color.zero_grad()
optim_widths.zero_grad()
optim_points.zero_grad()
scene_args = pydiffvg.RenderFunction.serialize_scene(
self.canvas_width, self.canvas_height, self.shapes,
self.shape_groups)
img = self.render(self.canvas_width, self.canvas_height, 2, 2, 0,
None, *scene_args)
# Compose img with white background
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(
img.shape[0], img.shape[1], 3,
device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
img = img[:, :, :3]
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2)
if use_perc_loss:
loss = perception_loss(
img, target) + (img.mean() - target.mean()).pow(2)
else:
loss = (img - target).pow(2).mean()
print(f'{iteration}: visual loss = {loss.item():.3f}')
loss.backward()
optim_points.step()
optim_widths.step()
optim_color.step()
for path in self.shapes:
path.stroke_width.data.clamp_(self.min_width, self.max_width)
for group in self.shape_groups:
group.stroke_color.data.clamp_(0.0, 1.0)
if iteration % self.frequency_plot == self.frequency_plot - 1:
self.plot()
self.save_svg(
name=f'images/paint_iteration/iter_{iteration}.svg')
def main(args):
# set device -> use cpu now since I haven't solved the nvcc issue
pydiffvg.set_use_gpu(False)
# pydiffvg.set_device(torch.device('cuda:1'))
# use L2 for now
# perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
# generate a texture synthesized
target_img = texture_syn(args.target)
tar_h, tar_w = target_img.shape[1], target_img.shape[0]
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(args.svg_path)
# svgpathtools for checking the bounding box
# paths, _, _ = svg2paths2(args.svg_path)
# print(len(paths))
# xmin, xmax, ymin, ymax = big_bounding_box(paths)
# print(xmin, xmax, ymin, ymax)
# input("check")
print('tar h : %d tar w : %d' % (tar_h, tar_w))
print('canvas h : %d canvas w : %d' % (canvas_height, canvas_width))
scale_ratio = tar_h / canvas_height
print("scale ratio : ", scale_ratio)
# input("check")
for path in shapes:
path.points[..., 0] = path.points[..., 0] * scale_ratio
path.points[..., 1] = path.points[..., 1] * scale_ratio
init_img = render(tar_w, tar_h, shapes, shape_groups)
pydiffvg.imwrite(init_img.cpu(),
'results/texture_synthesis/%d/init.png' % (args.case),
gamma=2.2)
# input("check")
random.seed(1234)
torch.manual_seed(1234)
points_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
color_vars = []
for group in shape_groups:
group.fill_color.requires_grad = True
color_vars.append(group.fill_color)
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=1.0)
color_optim = torch.optim.Adam(color_vars, lr=0.01)
target = torch.from_numpy(target_img).to(torch.float32) / 255.0
target = target.pow(2.2)
target = target.to(pydiffvg.get_device())
target = target.unsqueeze(0)
target = target.permute(0, 3, 1, 2) # NHWC -> NCHW
canvas_width, canvas_height = target.shape[3], target.shape[2]
# print('canvas h : %d canvas w : %d' % (canvas_height, canvas_width))
# input("check")
for t in range(args.max_iter):
print('iteration:', t)
points_optim.zero_grad()
color_optim.zero_grad()
cur_img = render(canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(cur_img.cpu(),
'results/texture_synthesis/%d/iter_%d.png' %
(args.case, t),
gamma=2.2)
cur_img = cur_img[:, :, :3]
cur_img = cur_img.unsqueeze(0)
cur_img = cur_img.permute(0, 3, 1, 2) # NHWC -> NCHW
# perceptual loss
# loss = perception_loss(cur_img, target)
# l2 loss
loss = (cur_img - target).pow(2).mean()
print('render loss:', loss.item())
loss.backward()
points_optim.step()
color_optim.step()
for group in shape_groups:
group.fill_color.data.clamp_(0.0, 1.0)
# write svg
if t % 10 == 0 or t == args.max_iter - 1:
pydiffvg.save_svg(
'results/texture_synthesis/%d/iter_%d.svg' % (args.case, t),
canvas_width, canvas_height, shapes, shape_groups)
# render final result
final_img = render(tar_h, tar_w, shapes, shape_groups)
pydiffvg.imwrite(final_img.cpu(),
'results/texture_synthesis/%d/final.png' % (args.case),
gamma=2.2)
from subprocess import call
call([
"ffmpeg", "-framerate", "24", "-i",
"results/texture_synthesis/%d/iter_%d.png" % (args.case), "-vb", "20M",
"results/texture_synthesis/%d/out.mp4" % (args.case)
])
# make gif
make_gif("results/texture_synthesis/%d" % (args.case),
"results/texture_synthesis/%d/out.gif" % (args.case),
frame_every_X_steps=1,
repeat_ending=3,
total_iter=args.max_iter)
def main(args):
inceptionv3 = models.inception_v3(pretrained=True,
transform_input=False).cuda()
inceptionv3.eval()
perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(args.svg)
scene_args = pydiffvg.RenderFunction.serialize_scene( \
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), 'logs/refine_svg/init.png', gamma=gamma)
pydiffvg.imwrite(img.cpu(), 'logs/refine_svg/init_.png')
points_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
# color_vars = {}
# for group in shape_groups:
# group.fill_color.requires_grad = True
# color_vars[group.fill_color.data_ptr()] = group.fill_color
# color_vars = list(color_vars.values())
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=1.0)
# color_optim = torch.optim.Adam(color_vars, lr=0.01)
# Adam iterations.
for t in range(args.num_iter):
print('iteration:', t)
points_optim.zero_grad()
# color_optim.zero_grad()
# Forward pass: render the image.
scene_args = pydiffvg.RenderFunction.serialize_scene( \
canvas_width, canvas_height, shapes, shape_groups)
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# Compose img with white background
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(
img.shape[0], img.shape[1], 3,
device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'logs/refine_svg/iter_{}.png'.format(t),
gamma=gamma)
img = img[:, :, :3]
# Convert img from HWC to NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
output = inceptionv3.forward(img.cuda())
get_class(output)
target = torch.autograd.Variable(torch.LongTensor([291]),
requires_grad=False).cuda()
loss = torch.nn.CrossEntropyLoss()(output, target)
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
points_optim.step()
# color_optim.step()
# for group in shape_groups:
# group.fill_color.data.clamp_(0.0, 1.0)
if t % 10 == 0 or t == args.num_iter - 1:
pydiffvg.save_svg('logs/refine_svg/iter_{}.svg'.format(t),
canvas_width, canvas_height, shapes,
shape_groups)
# Render the final result.
scene_args = pydiffvg.RenderFunction.serialize_scene( \
canvas_width, canvas_height, shapes, shape_groups)
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'logs/refine_svg/final.png'.format(t),
gamma=gamma)