def loss(self, x, y):
self.forward(x)
criterion = nn.MSELoss()#nn.BCELoss()
x_recon = self.cs[-1] # torch.sigmoid(self.cs[-1])
pydiffvg.imwrite(y[0].reshape([3,32, 32]).cpu().permute(1,2,0), './in.png')
pydiffvg.imwrite(x_recon[0].reshape([3,32, 32]).cpu().permute(1,2,0), './out.png')
# Reconstruction loss.
# Only want to average across the mini-batch, hence, multiply by the image dimensions.
Lx = criterion(x_recon, y) * self.A * self.B * self.channel
# Latent loss.
Lz = 0
for t in range(self.T):
mu_2 = self.mus[t] * self.mus[t]
sigma_2 = self.sigmas[t] * self.sigmas[t]
logsigma = self.logsigmas[t]
kl_loss = 0.5*torch.sum(mu_2 + sigma_2 - 2*logsigma, 1) - 0.5*self.T
Lz += kl_loss
Lz = torch.mean(Lz)
net_loss = Lx + Lz
return net_loss
def comp_loss_and_grad(img, tgt, it, sz):
dif = img-tgt
loss = dif.pow(2).mean()
dif = dif.detach()
cdif = dif.clone().abs()
cdif[:, :, 3] = 1.
resdif = torch.nn.functional.interpolate(cdif.permute(2, 0, 1).unsqueeze(0), sz, mode='bilinear').squeeze().permute(1, 2, 0).abs()
pydiffvg.imwrite(resdif[:, :, 0:4], 'results/simple_transform_svg/dif_{:04}.png'.format(it))
dif = dif.numpy()
padded = np.pad(dif, [(1, 1), (1, 1), (0, 0)], mode='edge')
# print(padded[:-2,:,:].shape)
grad_x = (padded[:-2, :, :]-padded[2:, :, :])[:, 1:-1, :]
grad_y = (padded[:, :-2, :]-padded[:, 2:, :])[1:-1, :, :]
resshape = dif.shape
resshape = (resshape[0], resshape[1], 2)
res = np.zeros(resshape)
for x in range(resshape[0]):
for y in range(resshape[1]):
A = np.concatenate((grad_x[x, y, :][:, np.newaxis], grad_y[x, y, :][:, np.newaxis]), axis=1)
b = -dif[x, y, :]
v = np.linalg.lstsq(np.dot(A.T, A), np.dot(A.T, b))
res[x, y, :] = v[0]
return loss, res
def interpolate(args):
chkpt = VAE_OUTPUT
if args.conditional:
chkpt += "_conditional"
if args.fc:
chkpt += "_fc"
meta = ttools.Checkpointer.load_meta(chkpt, prefix="g_")
if meta is None:
LOG.info("No metadata in checkpoint (or no checkpoint), aborting.")
return
model = VectorMNISTVAE(imsize=128, **meta)
checkpointer = ttools.Checkpointer(chkpt, model, prefix="g_")
checkpointer.load_latest()
model.eval()
# Sample some latent vectors
bs = 10
z = th.randn(bs, model.zdim)
label = None
label = th.arange(0, 10)
animation = []
nframes = 60
with th.no_grad():
for idx, _z in enumerate(z):
if idx == 0: # skip first
continue
_z0 = z[idx-1].unsqueeze(0).repeat(nframes, 1)
_z = _z.unsqueeze(0).repeat(nframes, 1)
if args.conditional:
_label = label[idx].unsqueeze(0).repeat(nframes)
else:
_label = None
# interp weights
alpha = th.linspace(0, 1, nframes).view(nframes, 1)
batch = alpha*_z + (1.0 - alpha)*_z0
images, aux = model.decode(batch, label=_label)
images += 1.0
images /= 2.0
animation.append(images)
anim_dir = os.path.join(chkpt, "interpolation")
os.makedirs(anim_dir, exist_ok=True)
animation = th.cat(animation, 0)
for idx, frame in enumerate(animation):
frame = frame.squeeze()
frame = th.clamp(frame, 0, 1).cpu().numpy()
path = os.path.join(anim_dir, "frame%03d.png" % idx)
pydiffvg.imwrite(frame, path, gamma=2.2)
LOG.info("Results saved to %s", anim_dir)
def main(args):
pydiffvg.set_device(th.device('cuda:1'))
# Load SVG
svg = os.path.join(args.svg)
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(svg)
# Save initial state
ref = render(canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(ref.cpu(), args.out, gamma=2.2)
def print_gradimg(gradimg, it, shape=None):
out = torch.zeros((gradimg.shape[0], gradimg.shape[1], 3), requires_grad=False, dtype=torch.float32)
for x in range(gradimg.shape[0]):
for y in range(gradimg.shape[1]):
# h = math.atan2(gradimg[x, y, 1], gradimg[x, y, 0])
# s = math.tanh(np.linalg.norm(gradimg[x, y, :]))
# v = 1.
vec = (gradimg[x, y, :].clip(min=-1, max=1)/2)+.5
# out[x,y,:]=torch.tensor(colorsys.hsv_to_rgb(h,s,v),dtype=torch.float32)
out[x, y, :] = torch.tensor([vec[0], vec[1], 0])
if shape is not None:
out = torch.nn.functional.interpolate(out.permute(2, 0, 1).unsqueeze(0), shape, mode='bilinear').squeeze().permute(1, 2, 0)
pydiffvg.imwrite(out.cpu(), 'results/simple_transform_svg/grad_{:04}.png'.format(it))
def main(svg_dirs):
pydiffvg.set_device(th.device('cuda:1'))
assert os.path.exists(svg_dirs)
svg_files = os.listdir(svg_dirs)
for svg_file in svg_files:
if '.svg' not in svg_file:
continue
svg_file_path = os.path.join(svg_dirs, svg_file)
out_file_path = svg_file_path.replace('.svg', '.png')
# Load SVG
canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(svg_file_path)
# Save initial state
ref = render(canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(ref.cpu(), out_file_path, gamma=2.2)
def gen_image_from_curves(self, t, shapes, shape_groups, gamma, background_image):
render = pydiffvg.RenderFunction.apply
scene_args = pydiffvg.RenderFunction.serialize_scene( \
self.canvas_width, self.canvas_height, shapes, shape_groups)
img = render(self.canvas_width, self.canvas_height, 2, 2, t, background_image, *scene_args)
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3,
device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
img = img[:, :, :3]
dir_ = "./gens/"
if not os.path.exists(dir_):
os.mkdir(dir_)
if t % 200 == 1:
pydiffvg.imwrite(img.cpu(), os.path.join(dir_, 'iter_{}.png'.format(int(t / 5))), gamma=gamma)
img = img[:, :, :3]
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2)
return img
def main(args):
pydiffvg.set_use_gpu(torch.cuda.is_available())
canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(
args.content_file)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Transform to gamma space
pydiffvg.imwrite(img.cpu(), 'results/style_transfer/init.png', gamma=1.0)
# HWC -> NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
loader = transforms.Compose([transforms.ToTensor()
]) # transform it into a torch tensor
def image_loader(image_name):
image = Image.open(image_name)
# fake batch dimension required to fit network's input dimensions
image = loader(image).unsqueeze(0)
return image.to(pydiffvg.get_device(), torch.float)
style_img = image_loader(args.style_img)
# alpha blend content with a gray background
content_img = img[:, :3, :, :] * img[:, 3, :, :] + \
0.5 * torch.ones([1, 3, img.shape[2], img.shape[3]]) * \
(1 - img[:, 3, :, :])
assert style_img.size() == content_img.size(), \
"we need to import style and content images of the same size"
# unloader = transforms.ToPILImage() # reconvert into PIL image
class ContentLoss(nn.Module):
def __init__(
self,
target,
):
super(ContentLoss, self).__init__()
# we 'detach' the target content from the tree used
# to dynamically compute the gradient: this is a stated value,
# not a variable. Otherwise the forward method of the criterion
# will throw an error.
self.target = target.detach()
def forward(self, input):
self.loss = F.mse_loss(input, self.target)
return input
def gram_matrix(input):
a, b, c, d = input.size() # a=batch size(=1)
# b=number of feature maps
# (c,d)=dimensions of a f. map (N=c*d)
features = input.view(a * b, c * d) # resise F_XL into \hat F_XL
G = torch.mm(features, features.t()) # compute the gram product
# we 'normalize' the values of the gram matrix
# by dividing by the number of element in each feature maps.
return G.div(a * b * c * d)
class StyleLoss(nn.Module):
def __init__(self, target_feature):
super(StyleLoss, self).__init__()
self.target = gram_matrix(target_feature).detach()
def forward(self, input):
G = gram_matrix(input)
self.loss = F.mse_loss(G, self.target)
return input
device = pydiffvg.get_device()
cnn = models.vgg19(pretrained=True).features.to(device).eval()
cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device)
cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device)
# create a module to normalize input image so we can easily put it in a
# nn.Sequential
class Normalization(nn.Module):
def __init__(self, mean, std):
super(Normalization, self).__init__()
# .view the mean and std to make them [C x 1 x 1] so that they can
# directly work with image Tensor of shape [B x C x H x W].
# B is batch size. C is number of channels. H is height and W is width.
self.mean = mean.clone().view(-1, 1, 1)
self.std = std.clone().view(-1, 1, 1)
def forward(self, img):
# normalize img
return (img - self.mean) / self.std
# desired depth layers to compute style/content losses :
content_layers_default = ['conv_4']
style_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5']
def get_style_model_and_losses(cnn,
normalization_mean,
normalization_std,
style_img,
content_img,
content_layers=content_layers_default,
style_layers=style_layers_default):
cnn = copy.deepcopy(cnn)
# normalization module
normalization = Normalization(normalization_mean,
normalization_std).to(device)
# just in order to have an iterable access to or list of content/syle
# losses
content_losses = []
style_losses = []
# assuming that cnn is a nn.Sequential, so we make a new nn.Sequential
# to put in modules that are supposed to be activated sequentially
model = nn.Sequential(normalization)
i = 0 # increment every time we see a conv
for layer in cnn.children():
if isinstance(layer, nn.Conv2d):
i += 1
name = 'conv_{}'.format(i)
elif isinstance(layer, nn.ReLU):
name = 'relu_{}'.format(i)
# The in-place version doesn't play very nicely with the ContentLoss
# and StyleLoss we insert below. So we replace with out-of-place
# ones here.
layer = nn.ReLU(inplace=False)
elif isinstance(layer, nn.MaxPool2d):
name = 'pool_{}'.format(i)
elif isinstance(layer, nn.BatchNorm2d):
name = 'bn_{}'.format(i)
else:
raise RuntimeError('Unrecognized layer: {}'.format(
layer.__class__.__name__))
model.add_module(name, layer)
if name in content_layers:
# add content loss:
target = model(content_img).detach()
content_loss = ContentLoss(target)
model.add_module("content_loss_{}".format(i), content_loss)
content_losses.append(content_loss)
if name in style_layers:
# add style loss:
target_feature = model(style_img).detach()
style_loss = StyleLoss(target_feature)
model.add_module("style_loss_{}".format(i), style_loss)
style_losses.append(style_loss)
# now we trim off the layers after the last content and style losses
for i in range(len(model) - 1, -1, -1):
if isinstance(model[i], ContentLoss) or isinstance(
model[i], StyleLoss):
break
model = model[:(i + 1)]
return model, style_losses, content_losses
def run_style_transfer(cnn,
normalization_mean,
normalization_std,
content_img,
style_img,
canvas_width,
canvas_height,
shapes,
shape_groups,
num_steps=500,
style_weight=5000,
content_weight=1):
"""Run the style transfer."""
print('Building the style transfer model..')
model, style_losses, content_losses = get_style_model_and_losses(
cnn, normalization_mean, normalization_std, style_img, content_img)
point_params = []
color_params = []
stroke_width_params = []
for shape in shapes:
if isinstance(shape, pydiffvg.Path):
point_params.append(shape.points.requires_grad_())
stroke_width_params.append(shape.stroke_width.requires_grad_())
for shape_group in shape_groups:
if isinstance(shape_group.fill_color, torch.Tensor):
color_params.append(shape_group.fill_color.requires_grad_())
elif isinstance(shape_group.fill_color, pydiffvg.LinearGradient):
point_params.append(
shape_group.fill_color.begin.requires_grad_())
point_params.append(
shape_group.fill_color.end.requires_grad_())
color_params.append(
shape_group.fill_color.stop_colors.requires_grad_())
if isinstance(shape_group.stroke_color, torch.Tensor):
color_params.append(shape_group.stroke_color.requires_grad_())
elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient):
point_params.append(
shape_group.stroke_color.begin.requires_grad_())
point_params.append(
shape_group.stroke_color.end.requires_grad_())
color_params.append(
shape_group.stroke_color.stop_colors.requires_grad_())
point_optimizer = optim.Adam(point_params, lr=1.0)
color_optimizer = optim.Adam(color_params, lr=0.01)
stroke_width_optimizers = optim.Adam(stroke_width_params, lr=0.1)
print('Optimizing..')
run = [0]
while run[0] <= num_steps:
point_optimizer.zero_grad()
color_optimizer.zero_grad()
stroke_width_optimizers.zero_grad()
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# alpha blend img with a gray background
img = img[:, :, :3] * img[:, :, 3:4] + \
0.5 * torch.ones([img.shape[0], img.shape[1], 3]) * \
(1 - img[:, :, 3:4])
pydiffvg.imwrite(img.cpu(),
'results/style_transfer/step_{}.png'.format(
run[0]),
gamma=1.0)
# HWC to NCHW
img = img.permute([2, 0, 1]).unsqueeze(0)
model(img)
style_score = 0
content_score = 0
for sl in style_losses:
style_score += sl.loss
for cl in content_losses:
content_score += cl.loss
style_score *= style_weight
content_score *= content_weight
loss = style_score + content_score
loss.backward()
run[0] += 1
if run[0] % 1 == 0:
print("run {}:".format(run))
print('Style Loss : {:4f} Content Loss: {:4f}'.format(
style_score.item(), content_score.item()))
print()
point_optimizer.step()
color_optimizer.step()
stroke_width_optimizers.step()
for color in color_params:
color.data.clamp_(0, 1)
for w in stroke_width_params:
w.data.clamp_(0.5, 4.0)
return shapes, shape_groups
shapes, shape_groups = run_style_transfer(cnn, cnn_normalization_mean,
cnn_normalization_std,
content_img, style_img,
canvas_width, canvas_height,
shapes, shape_groups)
scene_args = pydiffvg.RenderFunction.serialize_scene(shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Transform to gamma space
pydiffvg.imwrite(img.cpu(), 'results/style_transfer/output.png', gamma=1.0)
def main(args):
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(args.svg_file)
w = int(canvas_width * args.size_scale)
h = int(canvas_height * args.size_scale)
pfilter = pydiffvg.PixelFilter(type=diffvg.FilterType.box,
radius=torch.tensor(0.5))
use_prefiltering = False
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
num_samples_x = 16
num_samples_y = 16
render = pydiffvg.RenderFunction.apply
img = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None,
*scene_args)
pydiffvg.imwrite(img.cpu(),
'results/finite_difference_comp/img.png',
gamma=1.0)
epsilon = 0.1
def perturb_scene(axis, epsilon):
shapes[2].points[:, axis] += epsilon
# for s in shapes:
# if isinstance(s, pydiffvg.Circle):
# s.center[axis] += epsilon
# elif isinstance(s, pydiffvg.Ellipse):
# s.center[axis] += epsilon
# elif isinstance(s, pydiffvg.Path):
# s.points[:, axis] += epsilon
# elif isinstance(s, pydiffvg.Polygon):
# s.points[:, axis] += epsilon
# elif isinstance(s, pydiffvg.Rect):
# s.p_min[axis] += epsilon
# s.p_max[axis] += epsilon
# for s in shape_groups:
# if isinstance(s.fill_color, pydiffvg.LinearGradient):
# s.fill_color.begin[axis] += epsilon
# s.fill_color.end[axis] += epsilon
perturb_scene(0, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render = pydiffvg.RenderFunction.apply
img0 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None,
*scene_args)
forward_diff = (img0 - img) / (epsilon)
forward_diff = forward_diff.sum(axis=2)
x_diff_max = 1.5
x_diff_min = -1.5
print(forward_diff.max())
print(forward_diff.min())
forward_diff = cm.viridis(
normalize(forward_diff, x_diff_min, x_diff_max).cpu().numpy())
pydiffvg.imwrite(
forward_diff,
'results/finite_difference_comp/shared_edge_forward_diff.png',
gamma=1.0)
perturb_scene(0, -2 * epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
img1 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None,
*scene_args)
backward_diff = (img - img1) / (epsilon)
backward_diff = backward_diff.sum(axis=2)
print(backward_diff.max())
print(backward_diff.min())
backward_diff = cm.viridis(
normalize(backward_diff, x_diff_min, x_diff_max).cpu().numpy())
pydiffvg.imwrite(
backward_diff,
'results/finite_difference_comp/shared_edge_backward_diff.png',
gamma=1.0)
perturb_scene(0, epsilon)
num_samples_x = 4
num_samples_y = 4
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render_grad = pydiffvg.RenderFunction.render_grad
img_grad = render_grad(
torch.ones(h, w, 4),
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
*scene_args)
print(img_grad[:, :, 0].max())
print(img_grad[:, :, 0].min())
x_diff = cm.viridis(
normalize(img_grad[:, :, 0], x_diff_min, x_diff_max).cpu().numpy())
pydiffvg.imwrite(x_diff,
'results/finite_difference_comp/ours_x_diff.png',
gamma=1.0)
def main(args):
case_name = args.svg_file.split('/')[-1].split('.')[0]
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(args.svg_file)
w = int(canvas_width * args.size_scale)
h = int(canvas_height * args.size_scale)
print(w, h)
curve_counts = 0
for s in shapes:
if isinstance(s, pydiffvg.Circle):
curve_counts += 1
elif isinstance(s, pydiffvg.Ellipse):
curve_counts += 1
elif isinstance(s, pydiffvg.Path):
curve_counts += len(s.num_control_points)
elif isinstance(s, pydiffvg.Polygon):
curve_counts += len(s.points) - 1
if s.is_closed:
curve_counts += 1
elif isinstance(s, pydiffvg.Rect):
curve_counts += 1
print('curve_counts:', curve_counts)
pfilter = pydiffvg.PixelFilter(type=diffvg.FilterType.box,
radius=torch.tensor(0.5))
use_prefiltering = args.use_prefiltering
print('use_prefiltering:', use_prefiltering)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
num_samples_x = args.num_spp
num_samples_y = args.num_spp
if (use_prefiltering):
num_samples_x = 1
num_samples_y = 1
render = pydiffvg.RenderFunction.apply
img = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
pydiffvg.imwrite(
img.cpu(),
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/img.png',
gamma=1.0)
epsilon = 0.1
def perturb_scene(axis, epsilon):
for s in shapes:
if isinstance(s, pydiffvg.Circle):
s.center[axis] += epsilon
elif isinstance(s, pydiffvg.Ellipse):
s.center[axis] += epsilon
elif isinstance(s, pydiffvg.Path):
s.points[:, axis] += epsilon
elif isinstance(s, pydiffvg.Polygon):
s.points[:, axis] += epsilon
elif isinstance(s, pydiffvg.Rect):
s.p_min[axis] += epsilon
s.p_max[axis] += epsilon
for s in shape_groups:
if isinstance(s.fill_color, pydiffvg.LinearGradient):
s.fill_color.begin[axis] += epsilon
s.fill_color.end[axis] += epsilon
perturb_scene(0, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render = pydiffvg.RenderFunction.apply
img0 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
perturb_scene(0, -2 * epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
img1 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
x_diff = (img0 - img1) / (2 * epsilon)
x_diff = x_diff.sum(axis=2)
x_diff_max = x_diff.max() * args.clamping_factor
x_diff_min = x_diff.min() * args.clamping_factor
print(x_diff.max())
print(x_diff.min())
x_diff = cm.viridis(
normalize(x_diff, x_diff_min, x_diff_max).cpu().numpy())
pydiffvg.imwrite(
x_diff,
f'results/finite_difference_comp//{case_name}_{use_prefiltering}/finite_x_diff.png',
gamma=1.0)
perturb_scene(0, epsilon)
perturb_scene(1, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render = pydiffvg.RenderFunction.apply
img0 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
perturb_scene(1, -2 * epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
img1 = render(
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
y_diff = (img0 - img1) / (2 * epsilon)
y_diff = y_diff.sum(axis=2)
y_diff_max = y_diff.max() * args.clamping_factor
y_diff_min = y_diff.min() * args.clamping_factor
y_diff = cm.viridis(
normalize(y_diff, y_diff_min, y_diff_max).cpu().numpy())
pydiffvg.imwrite(
y_diff,
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/finite_y_diff.png',
gamma=1.0)
perturb_scene(1, epsilon)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
filter=pfilter,
use_prefiltering=use_prefiltering)
render_grad = pydiffvg.RenderFunction.render_grad
img_grad = render_grad(
torch.ones(h, w, 4, device=pydiffvg.get_device()),
w, # width
h, # height
num_samples_x, # num_samples_x
num_samples_y, # num_samples_y
0, # seed
None, # background_image
*scene_args)
print(img_grad[:, :, 0].max())
print(img_grad[:, :, 0].min())
x_diff = cm.viridis(
normalize(img_grad[:, :, 0], x_diff_min, x_diff_max).cpu().numpy())
y_diff = cm.viridis(
normalize(img_grad[:, :, 1], y_diff_min, y_diff_max).cpu().numpy())
pydiffvg.imwrite(
x_diff,
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/ours_x_diff.png',
gamma=1.0)
pydiffvg.imwrite(
y_diff,
f'results/finite_difference_comp/{case_name}_{use_prefiltering}/ours_y_diff.png',
gamma=1.0)
def main():
pydiffvg.set_device(th.device('cuda:1'))
# Load SVG
svg = os.path.join("imgs", "peppers.svg")
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(svg)
# Save initial state
ref = render(canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(ref.cpu(), 'results/gaussian_blur/init.png', gamma=2.2)
target = F.gaussian_filter(ref.cpu().numpy(), [10, 10, 0])
target = th.from_numpy(target).to(ref.device)
pydiffvg.imwrite(target.cpu(),
'results/gaussian_blur/target.png',
gamma=2.2)
# Collect variables to optimize
points_vars = []
width_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
path.stroke_width.requires_grad = True
width_vars.append(path.stroke_width)
color_vars = []
for group in shape_groups:
# do not optimize alpha
group.fill_color[..., :3].requires_grad = True
color_vars.append(group.fill_color)
# Optimize
points_optim = th.optim.Adam(points_vars, lr=1.0)
width_optim = th.optim.Adam(width_vars, lr=1.0)
color_optim = th.optim.Adam(color_vars, lr=0.01)
for t in range(20):
print('\niteration:', t)
points_optim.zero_grad()
width_optim.zero_grad()
color_optim.zero_grad()
# Forward pass: render the image.
img = render(canvas_width, canvas_height, shapes, shape_groups)
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'results/gaussian_blur/iter_{}.png'.format(t),
gamma=2.2)
loss = (img - target)[..., :3].pow(2).mean()
print('alpha:', img[..., 3].mean().item())
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
points_optim.step()
width_optim.step()
color_optim.step()
for group in shape_groups:
group.fill_color.data.clamp_(0.0, 1.0)
# Final render
img = render(canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(img.cpu(), 'results/gaussian_blur/final.png', gamma=2.2)
# Convert the intermediate renderings to a video.
from subprocess import call
call([
"ffmpeg", "-framerate", "24", "-i",
"results/gaussian_blur/iter_%d.png", "-vb", "20M",
"results/gaussian_blur/out.mp4"
])
[0.6, 0.3, 0.6, 0.8]))
shape_groups = [path_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(),
'results/single_open_curve_thickness/target.png',
gamma=2.2)
target = img.clone()
# Move the path to produce initial guess
# normalize points for easier learning rate
points_n = torch.tensor(
[
[100.0 / 256.0, 40.0 / 256.0], # base
[155.0 / 256.0, 65.0 / 256.0], # control point
[100.0 / 256.0, 180.0 / 256.0], # control point
[65.0 / 256.0, 238.0 / 256.0]
], # base
requires_grad=True)
thickness_n = torch.tensor(
[10.0 / 100.0, 10.0 / 100.0, 10.0 / 100.0, 10.0 / 100.0],
fill_color=torch.tensor([0.3, 0.6, 0.3, 1.0]))
shape_groups = [path_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
510, # width
510, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # background_image
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), 'results/single_path/target.png', gamma=2.2)
target = img.clone()
# Move the path to produce initial guess
# normalize points for easier learning rate
noise = torch.FloatTensor(shapes[0].points.shape).uniform_(0.0, 1.0)
points_n = (shapes[0].points.clone() + (noise * 60 - 30)) / 510.0
points_n.requires_grad = True
color = torch.tensor([0.3, 0.2, 0.5, 1.0], requires_grad=True)
shapes[0].points = points_n * 510
path_group.fill_color = color
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
img = render(
510, # width
510, # height
shapes = [circle]
circle_group = pydiffvg.ShapeGroup(shape_ids = torch.tensor([0]), fill_color = color)
shape_groups = [circle_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # background_image
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), 'results/single_gradient/target.png', gamma=2.2)
target = img.clone()
# Move the circle to produce initial guess
# normalize radius & center for easier learning rate
radius_n = torch.tensor(20.0 / 256.0, requires_grad=True)
center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True)
begin_n = torch.tensor([100.0 / 256.0, 100.0 / 256.0], requires_grad=True)
end_n = torch.tensor([150.0 / 256.0, 150.0 / 256.0], requires_grad=True)
stop_colors = torch.tensor([[0.1, 0.9, 0.2, 1.0],
[0.5, 0.3, 0.6, 1.0]], requires_grad=True)
color.begin = begin_n * 256
color.end = end_n * 256
color.stop_colors = stop_colors
circle.radius = radius_n * 256
circle.center = center_n * 256
canvas_height,
shapes,
shape_groups,
output_type=pydiffvg.OutputType.sdf)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # background_image
*scene_args)
img = img / 256 # Normalize SDF to [0, 1]
pydiffvg.imwrite(img.cpu(), 'results/test_eval_positions/target.png')
target = img.clone()
# Move the circle to produce initial guess
# normalize radius & center for easier learning rate
radius_n = torch.tensor(20.0 / 256.0, requires_grad=True)
center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True)
color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True)
circle.radius = radius_n * 256
circle.center = center_n * 256
circle_group.fill_color = color
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
shape_to_canvas = torch.eye(3, 3))
shape_groups = [ellipse_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(),
'results/single_ellipse_transform/target.png',
gamma=2.2)
target = img.clone()
# Affine transform the ellipse to produce initial guess
color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True)
affine = torch.zeros(2, 3)
affine[0, 0] = 1.3
affine[0, 1] = 0.2
affine[0, 2] = 0.1
affine[1, 0] = 0.2
affine[1, 1] = 0.6
affine[1, 2] = 0.3
affine.requires_grad = True
shape_to_canvas = torch.cat((affine, torch.tensor([[0.0, 0.0, 1.0]])), axis=0)
ellipse_group.fill_color = color
shapes,
shape_groups,
output_type=pydiffvg.OutputType.sdf)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
1, # num_samples_x
1, # num_samples_y
0, # seed
None, # background_image
*scene_args)
img /= 256.0
cm = plt.get_cmap('viridis')
img = cm(img.cpu().squeeze())
pydiffvg.imwrite(img, 'results/quadratic_distance_approx/ref_sdf.png')
scene_args = pydiffvg.RenderFunction.serialize_scene(canvas_width,
canvas_height, shapes,
shape_groups)
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # background_image
*scene_args)
pydiffvg.imwrite(img.cpu(), 'results/quadratic_distance_approx/ref_color.png')
shapes[0].use_distance_approx = True
settings.global_override(["paths", "optimize_points"], False)
settings.global_override(["transforms", "transform_lr"], 1e-2)
settings.undefault("linearGradient3152")
settings.retrieve(
"linearGradient3152")[0]["transforms"]["optimize_transforms"] = False
#optim=pydiffvg.OptimizableSvg("note_small.svg",settings,verbose=True)
optim = pydiffvg.OptimizableSvg("heart_green.svg", settings, verbose=True)
#img=torchvision.transforms.ToTensor()(Image.open("note_transformed.png")).permute(1,2,0)
img = torchvision.transforms.ToTensor()(
Image.open("heart_green_90.png")).permute(1, 2, 0)
name = "heart_green_90"
pydiffvg.imwrite(img.cpu(), 'results/simple_transform_svg/target.png')
target = img.clone().detach().requires_grad_(False)
img = optim.render()
pydiffvg.imwrite(img.cpu(), 'results/simple_transform_svg/init.png')
def smooth(input, kernel):
input = torch.nn.functional.pad(input.permute(2, 0, 1).unsqueeze(0),
(2, 2, 2, 2),
mode='reflect')
output = kernel(input)
return output
def printimg(optim):
canvas_height,
shapes,
shape_groups,
output_type=pydiffvg.OutputType.sdf)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
img = img / 256 # Normalize SDF to [0, 1]
pydiffvg.imwrite(img.cpu(), 'results/single_circle_sdf/target.png')
target = img.clone()
# Move the circle to produce initial guess
# normalize radius & center for easier learning rate
radius_n = torch.tensor(20.0 / 256.0, requires_grad=True)
center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True)
color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True)
circle.radius = radius_n * 256
circle.center = center_n * 256
circle_group.fill_color = color
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width,
canvas_height,
shapes,
shape_groups,
def main(args):
if args.seed:
np.random.seed(args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
pydiffvg.set_print_timing(False)
outdir = os.path.join(args.results_dir, args.prompt, args.subdir)
# Use GPU if available
pydiffvg.set_use_gpu(torch.cuda.is_available())
canvas_width, canvas_height = 224, 224
margin = args.initial_margin
total_paths = args.open_paths + args.closed_paths
step = min(args.step, total_paths)
if step == 0:
step = total_paths
fill_color = None
stroke_color = None
shapes = []
shape_groups = []
losses = []
tt = 0
for num_paths in range(step, total_paths + 1, step):
for i in range(num_paths - step, num_paths):
num_segments = random.randint(1, args.extra_segments + 1)
p0 = (margin + random.random() * (1 - 2 * margin),
margin + random.random() * (1 - 2 * margin))
points = [p0]
is_closed = i >= args.open_paths
if is_closed:
num_segments += 2
for j in range(num_segments):
p1 = (p0[0] + radius * (random.random() - 0.5),
p0[1] + radius * (random.random() - 0.5))
p2 = (p1[0] + radius * (random.random() - 0.5),
p1[1] + radius * (random.random() - 0.5))
p3 = (p2[0] + radius * (random.random() - 0.5),
p2[1] + radius * (random.random() - 0.5))
points.append(p1)
points.append(p2)
if is_closed and j < num_segments - 1 or not is_closed:
points.append(p3)
p0 = p3
points = torch.tensor(points)
points[:, 0] *= canvas_width
points[:, 1] *= canvas_height
stroke_width = torch.tensor(1.0)
color = torch.tensor([
random.random(),
random.random(),
random.random(),
random.random()
])
num_control_points = torch.zeros(num_segments,
dtype=torch.int32) + 2
path = pydiffvg.Path(num_control_points=num_control_points,
points=points,
stroke_width=stroke_width,
is_closed=is_closed)
shapes.append(path)
path_group = pydiffvg.ShapeGroup(
shape_ids=torch.tensor([len(shapes) - 1]),
fill_color=color if is_closed else None,
stroke_color=None if is_closed else color)
shape_groups.append(path_group)
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
pydiffvg.imwrite(img.cpu(),
os.path.join(outdir, 'init.png'),
gamma=gamma)
points_vars = []
stroke_width_vars = []
color_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
if not path.is_closed and args.max_width > 1:
path.stroke_width.requires_grad = True
stroke_width_vars.append(path.stroke_width)
for group in shape_groups:
if group.fill_color is not None:
group.fill_color.requires_grad = True
color_vars.append(group.fill_color)
else:
group.stroke_color.requires_grad = True
color_vars.append(group.stroke_color)
# Embed prompt
text_features = clip_utils.embed_text(args.prompt)
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=args.points_lr)
if len(stroke_width_vars) > 0:
width_optim = torch.optim.Adam(stroke_width_vars, lr=args.width_lr)
color_optim = torch.optim.Adam(color_vars, lr=args.color_lr)
# Adam iterations.
final = False
this_step_iters = max(1, round(args.num_iter * step / total_paths))
if num_paths + step > total_paths:
final = True
this_step_iters += args.extra_iter
for t in range(this_step_iters):
points_optim.zero_grad()
if len(stroke_width_vars) > 0:
width_optim.zero_grad()
color_optim.zero_grad()
# Forward pass: render the image.
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
tt, # seed
None,
*scene_args)
# Save the intermediate render.
with warnings.catch_warnings():
warnings.simplefilter("ignore")
pydiffvg.imwrite(img.cpu(),
os.path.join(outdir,
'iter_{}.png'.format(tt)),
gamma=gamma)
image_features = clip_utils.embed_image(img)
loss = -torch.cosine_similarity(
text_features, image_features, dim=-1).mean()
# Backpropagate the gradients.
loss.backward()
losses.append(loss.item())
# Take a gradient descent step.
points_optim.step()
if len(stroke_width_vars) > 0:
width_optim.step()
color_optim.step()
for path in shapes:
path.points.data[:, 0].clamp_(0.0, canvas_width)
path.points.data[:, 1].clamp_(0.0, canvas_height)
if not path.is_closed:
path.stroke_width.data.clamp_(1.0, args.max_width)
for group in shape_groups:
if group.fill_color is not None:
group.fill_color.data[:3].clamp_(0.0, 1.0)
group.fill_color.data[3].clamp_(args.min_alpha, 1.0)
else:
group.stroke_color.data[:3].clamp_(0.0, 1.0)
group.stroke_color.data[3].clamp_(args.min_alpha, 1.0)
if tt % 10 == 0 or final and t == this_step_iters - 1:
print('%d loss=%.3f' % (tt, 1 + losses[-1]))
pydiffvg.save_svg(
os.path.join(outdir, 'iter_{}.svg'.format(tt)),
canvas_width, canvas_height, shapes, shape_groups)
clip_utils.plot_losses(losses, outdir)
tt += 1
# Render the final result.
img = render(
args.final_px, # width
args.final_px, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Save the intermediate render
with warnings.catch_warnings():
warnings.simplefilter("ignore")
pydiffvg.imwrite(img.cpu(),
os.path.join(outdir, 'final.png'),
gamma=gamma)
# Convert the intermediate renderings to a video with a white background.
from subprocess import call
call([
"ffmpeg", "-framerate", "24", "-i",
os.path.join(outdir, "iter_%d.png"), "-vb", "20M", "-filter_complex",
"color=white,format=rgb24[c];[c][0]scale2ref[c][i];[c][i]overlay=format=auto:shortest=1,setsar=1",
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-profile:v", "baseline",
"-movflags", "+faststart",
os.path.join(outdir, "out.mp4")
])
def run_style_transfer(cnn,
normalization_mean,
normalization_std,
content_img,
style_img,
canvas_width,
canvas_height,
shapes,
shape_groups,
num_steps=500,
style_weight=5000,
content_weight=1):
"""Run the style transfer."""
print('Building the style transfer model..')
model, style_losses, content_losses = get_style_model_and_losses(
cnn, normalization_mean, normalization_std, style_img, content_img)
point_params = []
color_params = []
stroke_width_params = []
for shape in shapes:
if isinstance(shape, pydiffvg.Path):
point_params.append(shape.points.requires_grad_())
stroke_width_params.append(shape.stroke_width.requires_grad_())
for shape_group in shape_groups:
if isinstance(shape_group.fill_color, torch.Tensor):
color_params.append(shape_group.fill_color.requires_grad_())
elif isinstance(shape_group.fill_color, pydiffvg.LinearGradient):
point_params.append(
shape_group.fill_color.begin.requires_grad_())
point_params.append(
shape_group.fill_color.end.requires_grad_())
color_params.append(
shape_group.fill_color.stop_colors.requires_grad_())
if isinstance(shape_group.stroke_color, torch.Tensor):
color_params.append(shape_group.stroke_color.requires_grad_())
elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient):
point_params.append(
shape_group.stroke_color.begin.requires_grad_())
point_params.append(
shape_group.stroke_color.end.requires_grad_())
color_params.append(
shape_group.stroke_color.stop_colors.requires_grad_())
point_optimizer = optim.Adam(point_params, lr=1.0)
color_optimizer = optim.Adam(color_params, lr=0.01)
stroke_width_optimizers = optim.Adam(stroke_width_params, lr=0.1)
print('Optimizing..')
run = [0]
while run[0] <= num_steps:
point_optimizer.zero_grad()
color_optimizer.zero_grad()
stroke_width_optimizers.zero_grad()
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# alpha blend img with a gray background
img = img[:, :, :3] * img[:, :, 3:4] + \
0.5 * torch.ones([img.shape[0], img.shape[1], 3]) * \
(1 - img[:, :, 3:4])
pydiffvg.imwrite(img.cpu(),
'results/style_transfer/step_{}.png'.format(
run[0]),
gamma=1.0)
# HWC to NCHW
img = img.permute([2, 0, 1]).unsqueeze(0)
model(img)
style_score = 0
content_score = 0
for sl in style_losses:
style_score += sl.loss
for cl in content_losses:
content_score += cl.loss
style_score *= style_weight
content_score *= content_weight
loss = style_score + content_score
loss.backward()
run[0] += 1
if run[0] % 1 == 0:
print("run {}:".format(run))
print('Style Loss : {:4f} Content Loss: {:4f}'.format(
style_score.item(), content_score.item()))
print()
point_optimizer.step()
color_optimizer.step()
stroke_width_optimizers.step()
for color in color_params:
color.data.clamp_(0, 1)
for w in stroke_width_params:
w.data.clamp_(0.5, 4.0)
return shapes, shape_groups
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--svg",
default=os.path.join("imgs", "seamcarving",
"hokusai.svg"))
parser.add_argument("--optim_steps", default=10, type=int)
parser.add_argument("--lr", default=1e-1, type=int)
args = parser.parse_args()
name = os.path.splitext(os.path.basename(args.svg))[0]
root = os.path.join("results", "seam_carving", name)
svg_root = os.path.join(root, "svg")
os.makedirs(root, exist_ok=True)
os.makedirs(os.path.join(root, "svg"), exist_ok=True)
pydiffvg.set_use_gpu(False)
# pydiffvg.set_device(th.device('cuda'))
# Load SVG
print("loading svg %s" % args.svg)
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(args.svg)
print("done loading")
max_size = 512
scale_factor = max_size / max(canvas_width, canvas_height)
print("rescaling from %dx%d with scale %f" %
(canvas_width, canvas_height, scale_factor))
canvas_width = int(canvas_width * scale_factor)
canvas_height = int(canvas_height * scale_factor)
print("new shape %dx%d" % (canvas_width, canvas_height))
vector_rescale(shapes, scale_x=scale_factor, scale_y=scale_factor)
# Shrink image by 33 %
# num_seams_to_remove = 2
num_seams_to_remove = canvas_width // 3
new_canvas_width = canvas_width - num_seams_to_remove
scaling = new_canvas_width * 1.0 / canvas_width
# Naive scaling baseline
print("rendering naive rescaling...")
vector_rescale(shapes, scale_x=scaling)
resized = render(new_canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(resized.cpu(),
os.path.join(root, 'uniform_scaling.png'),
gamma=2.2)
pydiffvg.save_svg(os.path.join(svg_root, 'uniform_scaling.svg'),
canvas_width,
canvas_height,
shapes,
shape_groups,
use_gamma=False)
vector_rescale(shapes,
scale_x=1.0 / scaling) # bring back original coordinates
print("saved naiving scaling")
# Save initial state
print("rendering initial state...")
im = render(canvas_width, canvas_height, shapes, shape_groups)
pydiffvg.imwrite(im.cpu(), os.path.join(root, 'init.png'), gamma=2.2)
pydiffvg.save_svg(os.path.join(svg_root, 'init.svg'),
canvas_width,
canvas_height,
shapes,
shape_groups,
use_gamma=False)
print("saved initial state")
# Optimize
# color_optim = th.optim.Adam(color_vars, lr=0.01)
retargeted = im[..., :3].cpu().numpy()
previous_width = canvas_width
print("carving seams")
for seam_idx in range(num_seams_to_remove):
print('\nseam', seam_idx + 1, 'of', num_seams_to_remove)
# Remove a seam
retargeted = carve_seam(retargeted)
current_width = canvas_width - seam_idx - 1
scale_factor = current_width * 1.0 / previous_width
previous_width = current_width
padded = np.zeros((canvas_height, canvas_width, 4))
padded[:, :-seam_idx - 1, :3] = retargeted
padded[:, :-seam_idx - 1, -1] = 1.0 # alpha
padded = th.from_numpy(padded).to(im.device)
# Remap points to the smaller canvas and
# collect variables to optimize
points_vars = []
# width_vars = []
mini, maxi = canvas_width, 0
for path in shapes:
path.points.requires_grad = False
x = path.points[..., 0]
y = path.points[..., 1]
# rescale
x = x * scale_factor
# clip to canvas
path.points[..., 0] = th.clamp(x, 0, current_width)
path.points[..., 1] = th.clamp(y, 0, canvas_height)
path.points.requires_grad = True
points_vars.append(path.points)
path.stroke_width.requires_grad = True
# width_vars.append(path.stroke_width)
mini = min(mini, path.points.min().item())
maxi = max(maxi, path.points.max().item())
print("points", mini, maxi, "scale", scale_factor)
# recreate an optimizer so we don't carry over the previous update
# (momentum)?
geom_optim = th.optim.Adam(points_vars, lr=args.lr)
for step in range(args.optim_steps):
geom_optim.zero_grad()
img = render(canvas_width,
canvas_height,
shapes,
shape_groups,
samples=2)
pydiffvg.imwrite(img.cpu(),
os.path.join(
root,
"seam_%03d_iter_%02d.png" % (seam_idx, step)),
gamma=2.2)
# NO alpha
loss = (img - padded)[..., :3].pow(2).mean()
# loss = (img - padded).pow(2).mean()
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
geom_optim.step()
pydiffvg.save_svg(os.path.join(svg_root, "seam%03d.svg" % seam_idx),
canvas_width - seam_idx,
canvas_height,
shapes,
shape_groups,
use_gamma=False)
for path in shapes:
mini = min(mini, path.points.min().item())
maxi = max(maxi, path.points.max().item())
print("points", mini, maxi)
img = render(canvas_width, canvas_height, shapes, shape_groups)
img = img[:, :-num_seams_to_remove]
pydiffvg.imwrite(img.cpu(), os.path.join(root, 'final.png'), gamma=2.2)
pydiffvg.imwrite(retargeted, os.path.join(root, 'ref.png'), gamma=2.2)
pydiffvg.save_svg(os.path.join(svg_root, 'final.svg'),
canvas_width - num_seams_to_remove + 1,
canvas_height,
shapes,
shape_groups,
use_gamma=False)
# Convert the intermediate renderings to a video.
from subprocess import call
call([
"ffmpeg", "-framerate", "24", "-i",
os.path.join(root, "seam_%03d_iter_00.png"), "-vb", "20M",
os.path.join(root, "out.mp4")
])
shape_groups = [path_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # background_image
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(),
'results/single_curve_outline/target.png',
gamma=2.2)
target = img.clone()
# Move the path to produce initial guess
# normalize points for easier learning rate
points_n = torch.tensor(
[
[100.0 / 256.0, 40.0 / 256.0], # base
[155.0 / 256.0, 65.0 / 256.0], # control point
[100.0 / 256.0, 180.0 / 256.0], # control point
[65.0 / 256.0, 238.0 / 256.0], # base
[100.0 / 256.0, 200.0 / 256.0], # control point
[170.0 / 256.0, 55.0 / 256.0], # control point
[220.0 / 256.0, 100.0 / 256.0], # base
[210.0 / 256.0, 80.0 / 256.0], # control point
shape_groups = [circle_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(canvas_width,
canvas_height, shapes,
shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), 'logs/single_circle/target.png', gamma=2.2)
target = img.clone()
# Move the circle to produce initial guess
# normalize radius & center for easier learning rate
radius_n = torch.tensor(20.0 / 256.0, requires_grad=True)
center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True)
color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True)
circle.radius = radius_n * 256
circle.center = center_n * 256
circle_group.fill_color = color
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
img = render(
256, # width
256, # height
def main(args):
# Use GPU if available
pydiffvg.set_use_gpu(torch.cuda.is_available())
perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
#target = torch.from_numpy(skimage.io.imread('imgs/lena.png')).to(torch.float32) / 255.0
target = torch.from_numpy(skimage.io.imread(args.target)).to(
torch.float32) / 255.0
target = target.pow(gamma)
target = target.to(pydiffvg.get_device())
target = target.unsqueeze(0)
target = target.permute(0, 3, 1, 2) # NHWC -> NCHW
#target = torch.nn.functional.interpolate(target, size = [256, 256], mode = 'area')
canvas_width, canvas_height = target.shape[3], target.shape[2]
num_paths = args.num_paths
max_width = args.max_width
random.seed(1234)
torch.manual_seed(1234)
shapes = []
shape_groups = []
if args.use_blob:
for i in range(num_paths):
num_segments = random.randint(3, 5)
num_control_points = torch.zeros(num_segments,
dtype=torch.int32) + 2
points = []
p0 = (random.random(), random.random())
points.append(p0)
for j in range(num_segments):
radius = 0.05
p1 = (p0[0] + radius * (random.random() - 0.5),
p0[1] + radius * (random.random() - 0.5))
p2 = (p1[0] + radius * (random.random() - 0.5),
p1[1] + radius * (random.random() - 0.5))
p3 = (p2[0] + radius * (random.random() - 0.5),
p2[1] + radius * (random.random() - 0.5))
points.append(p1)
points.append(p2)
if j < num_segments - 1:
points.append(p3)
p0 = p3
points = torch.tensor(points)
points[:, 0] *= canvas_width
points[:, 1] *= canvas_height
path = pydiffvg.Path(num_control_points=num_control_points,
points=points,
stroke_width=torch.tensor(1.0),
is_closed=True)
shapes.append(path)
path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor(
[len(shapes) - 1]),
fill_color=torch.tensor([
random.random(),
random.random(),
random.random(),
random.random()
]))
shape_groups.append(path_group)
else:
for i in range(num_paths):
num_segments = random.randint(1, 3)
num_control_points = torch.zeros(num_segments,
dtype=torch.int32) + 2
points = []
p0 = (random.random(), random.random())
points.append(p0)
for j in range(num_segments):
radius = 0.05
p1 = (p0[0] + radius * (random.random() - 0.5),
p0[1] + radius * (random.random() - 0.5))
p2 = (p1[0] + radius * (random.random() - 0.5),
p1[1] + radius * (random.random() - 0.5))
p3 = (p2[0] + radius * (random.random() - 0.5),
p2[1] + radius * (random.random() - 0.5))
points.append(p1)
points.append(p2)
points.append(p3)
p0 = p3
points = torch.tensor(points)
points[:, 0] *= canvas_width
points[:, 1] *= canvas_height
#points = torch.rand(3 * num_segments + 1, 2) * min(canvas_width, canvas_height)
path = pydiffvg.Path(num_control_points=num_control_points,
points=points,
stroke_width=torch.tensor(1.0),
is_closed=False)
shapes.append(path)
path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor(
[len(shapes) - 1]),
fill_color=None,
stroke_color=torch.tensor([
random.random(),
random.random(),
random.random(),
random.random()
]))
shape_groups.append(path_group)
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
pydiffvg.imwrite(img.cpu(),
'results/painterly_rendering/init.png',
gamma=gamma)
points_vars = []
stroke_width_vars = []
color_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
if not args.use_blob:
for path in shapes:
path.stroke_width.requires_grad = True
stroke_width_vars.append(path.stroke_width)
if args.use_blob:
for group in shape_groups:
group.fill_color.requires_grad = True
color_vars.append(group.fill_color)
else:
for group in shape_groups:
group.stroke_color.requires_grad = True
color_vars.append(group.stroke_color)
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=1.0)
if len(stroke_width_vars) > 0:
width_optim = torch.optim.Adam(stroke_width_vars, lr=0.1)
color_optim = torch.optim.Adam(color_vars, lr=0.01)
# Adam iterations.
for t in range(args.num_iter):
print('iteration:', t)
points_optim.zero_grad()
if len(stroke_width_vars) > 0:
width_optim.zero_grad()
color_optim.zero_grad()
# Forward pass: render the image.
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
t, # seed
None,
*scene_args)
# Compose img with white background
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(
img.shape[0], img.shape[1], 3,
device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'results/painterly_rendering/iter_{}.png'.format(t),
gamma=gamma)
img = img[:, :, :3]
# Convert img from HWC to NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
if args.use_lpips_loss:
loss = perception_loss(
img, target) + (img.mean() - target.mean()).pow(2)
else:
loss = (img - target).pow(2).mean()
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
points_optim.step()
if len(stroke_width_vars) > 0:
width_optim.step()
color_optim.step()
if len(stroke_width_vars) > 0:
for path in shapes:
path.stroke_width.data.clamp_(1.0, max_width)
if args.use_blob:
for group in shape_groups:
group.fill_color.data.clamp_(0.0, 1.0)
else:
for group in shape_groups:
group.stroke_color.data.clamp_(0.0, 1.0)
if t % 10 == 0 or t == args.num_iter - 1:
pydiffvg.save_svg(
'results/painterly_rendering/iter_{}.svg'.format(t),
canvas_width, canvas_height, shapes, shape_groups)
# Render the final result.
img = render(
target.shape[1], # width
target.shape[0], # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(),
'results/painterly_rendering/final.png'.format(t),
gamma=gamma)
# Convert the intermediate renderings to a video.
from subprocess import call
call([
"ffmpeg", "-framerate", "24", "-i",
"results/painterly_rendering/iter_%d.png", "-vb", "20M",
"results/painterly_rendering/out.mp4"
])
infile = 'linux.svg'
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(infile)
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(
canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), 'test_old.png', gamma=1.0)
#optim=OptimizableSvg('linux.svg',verbose=True)
optim = OptimizableSvg(infile, verbose=True)
scene = optim.build_scene()
scene_args = pydiffvg.RenderFunction.serialize_scene(*scene)
render = pydiffvg.RenderFunction.apply
img = render(
scene[0], # width
scene[1], # height
2, # num_samples_x
2, # num_samples_y
0, # seed
*scene_args)
fill_color = torch.tensor([0.3, 0.6, 0.3, 1.0]))
shape_groups = [path_group]
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups,
output_type = pydiffvg.OutputType.sdf)
render = pydiffvg.RenderFunction.apply
img = render(510, # width
510, # height
1, # num_samples_x
1, # num_samples_y
0, # seed
None, # background_image
*scene_args)
img = img / 510 # Normalize SDF to [0, 1]
pydiffvg.imwrite(img.cpu(), 'results/single_path_sdf/target.png', gamma=1.0)
target = img.clone()
# Move the path to produce initial guess
# normalize points for easier learning rate
noise = torch.FloatTensor(shapes[0].points.shape).uniform_(0.0, 1.0)
points_n = (shapes[0].points.clone() + (noise * 60 - 30)) / 510.0
points_n.requires_grad = True
color = torch.tensor([0.3, 0.2, 0.5, 1.0], requires_grad=True)
shapes[0].points = points_n * 510
path_group.fill_color = color
scene_args = pydiffvg.RenderFunction.serialize_scene(\
canvas_width, canvas_height, shapes, shape_groups,
output_type = pydiffvg.OutputType.sdf)
img = render(510, # width
510, # height
def generate_samples(args):
chkpt = VAE_OUTPUT
if args.conditional:
chkpt += "_conditional"
if args.fc:
chkpt += "_fc"
meta = ttools.Checkpointer.load_meta(chkpt, prefix="g_")
if meta is None:
LOG.info("No metadata in checkpoint (or no checkpoint), aborting.")
return
model = VectorMNISTVAE(**meta)
checkpointer = ttools.Checkpointer(chkpt, model, prefix="g_")
checkpointer.load_latest()
model.eval()
# Sample some latent vectors
n = 8
bs = n*n
z = th.randn(bs, model.zdim)
imsize = 28
dataset = Dataset(args.data_dir, imsize)
dataloader = DataLoader(dataset, batch_size=bs,
num_workers=1, shuffle=True)
for batch in dataloader:
ref, label = batch
break
autoencode = True
if autoencode:
LOG.info("Sampling with auto-encoder code")
if not args.conditional:
label = None
mu, logvar = model.encode(ref, label)
z = model.reparameterize(mu, logvar)
else:
label = None
if args.conditional:
label = th.clamp(th.rand(bs)*10, 0, 9).long()
if args.digit is not None:
label[:] = args.digit
with th.no_grad():
images, aux = model.decode(z, label=label)
scenes = aux["scenes"]
images += 1.0
images /= 2.0
h = w = model.imsize
images = images.view(n, n, h, w).permute(0, 2, 1, 3)
images = images.contiguous().view(n*h, n*w)
images = th.clamp(images, 0, 1).cpu().numpy()
path = os.path.join(chkpt, "samples.png")
pydiffvg.imwrite(images, path, gamma=2.2)
if autoencode:
ref += 1.0
ref /= 2.0
ref = ref.view(n, n, h, w).permute(0, 2, 1, 3)
ref = ref.contiguous().view(n*h, n*w)
ref = th.clamp(ref, 0, 1).cpu().numpy()
path = os.path.join(chkpt, "ref.png")
pydiffvg.imwrite(ref, path, gamma=2.2)
# merge scenes
all_shapes = []
all_shape_groups = []
cur_id = 0
for idx, s in enumerate(scenes):
shapes, shape_groups, _ = s
# width, height = sizes
# Shift digit on canvas
center_x = idx % n
center_y = idx // n
for shape in shapes:
shape.points[:, 0] += center_x * model.imsize
shape.points[:, 1] += center_y * model.imsize
all_shapes.append(shape)
for grp in shape_groups:
grp.shape_ids[:] = cur_id
cur_id += 1
all_shape_groups.append(grp)
LOG.info("Generated %d shapes", len(all_shapes))
fname = os.path.join(chkpt, "digits.svg")
pydiffvg.save_svg(fname, n*model.imsize, n*model.imsize, all_shapes,
all_shape_groups, use_gamma=False)
LOG.info("Results saved to %s", chkpt)
def main(target_path, svg_path, output_dir, num_iter=1000, use_lpips_loss=False):
perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device())
target = torch.from_numpy(skimage.io.imread(target_path, as_gray=False, pilmode="RGB")).to(torch.float32) / 255.0
print("target", target.size())
target = target.pow(gamma)
target = target.to(pydiffvg.get_device())
target = target.unsqueeze(0)
target = target.permute(0, 3, 1, 2) # NHWC -> NCHW
canvas_width, canvas_height, shapes, shape_groups = \
pydiffvg.svg_to_scene(svg_path)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
render = pydiffvg.RenderFunction.apply
img = render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(), f'{output_dir}/init.png', gamma=gamma)
points_vars = []
for path in shapes:
path.points.requires_grad = True
points_vars.append(path.points)
color_vars = {}
for group in shape_groups:
group.fill_color.requires_grad = True
color_vars[group.fill_color.data_ptr()] = group.fill_color
# color_vars = list(color_vars.values())
# Optimize
points_optim = torch.optim.Adam(points_vars, lr=0.1)
# color_optim = torch.optim.Adam(color_vars, lr=0.01)
# Adam iterations.
for t in range(num_iter):
print('iteration:', t)
points_optim.zero_grad()
# color_optim.zero_grad()
# Forward pass: render the image.
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
img = render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# Compose img with white background
img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4])
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(), f'{output_dir}/iter_{t}.png', gamma=gamma)
img = img[:, :, :3]
# Convert img from HWC to NCHW
img = img.unsqueeze(0)
img = img.permute(0, 3, 1, 2) # NHWC -> NCHW
# print(img.size())
# print(target.size())
if use_lpips_loss:
loss = perception_loss(img, target)
else:
loss = (img - target).pow(2).mean()
print('render loss:', loss.item())
# Backpropagate the gradients.
loss.backward()
# Take a gradient descent step.
points_optim.step()
# color_optim.step()
for group in shape_groups:
group.fill_color.data.clamp_(0.0, 1.0)
if t % 10 == 0 or t == num_iter - 1:
pydiffvg.save_svg_paths_only(f'{output_dir}/iter_{t}.svg',
canvas_width, canvas_height, shapes, shape_groups)
# Render the final result.
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width, canvas_height, shapes, shape_groups)
img = render(canvas_width, # width
canvas_height, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None, # bg
*scene_args)
# Save the intermediate render.
pydiffvg.imwrite(img.cpu(), f'{output_dir}/final.png', gamma=gamma)
# Convert the intermediate renderings to a video.
from subprocess import call
call(["ffmpeg", "-framerate", "24", "-i",
f"{output_dir}/iter_%d.png", "-vb", "20M",
f"{output_dir}/out.mp4"])
shape_groups=shape_groups,
filter=pydiffvg.PixelFilter(type=diffvg.FilterType.hann,
radius=torch.tensor(8.0)))
render = pydiffvg.RenderFunction.apply
img = render(
256, # width
256, # height
2, # num_samples_x
2, # num_samples_y
0, # seed
None,
*scene_args)
# The output image is in linear RGB space. Do Gamma correction before saving the image.
pydiffvg.imwrite(img.cpu(),
'results/optimize_pixel_filter/target.png',
gamma=2.2)
target = img.clone()
# Change the pixel filter radius
radius = torch.tensor(1.0, requires_grad=True)
scene_args = pydiffvg.RenderFunction.serialize_scene(
canvas_width=canvas_width,
canvas_height=canvas_height,
shapes=shapes,
shape_groups=shape_groups,
filter=pydiffvg.PixelFilter(type=diffvg.FilterType.hann, radius=radius))
img = render(
256, # width
256, # height
2, # num_samples_x
