def loss(self, x, y): self.forward(x) criterion = nn.MSELoss()#nn.BCELoss() x_recon = self.cs[-1] # torch.sigmoid(self.cs[-1]) pydiffvg.imwrite(y[0].reshape([3,32, 32]).cpu().permute(1,2,0), './in.png') pydiffvg.imwrite(x_recon[0].reshape([3,32, 32]).cpu().permute(1,2,0), './out.png') # Reconstruction loss. # Only want to average across the mini-batch, hence, multiply by the image dimensions. Lx = criterion(x_recon, y) * self.A * self.B * self.channel # Latent loss. Lz = 0 for t in range(self.T): mu_2 = self.mus[t] * self.mus[t] sigma_2 = self.sigmas[t] * self.sigmas[t] logsigma = self.logsigmas[t] kl_loss = 0.5*torch.sum(mu_2 + sigma_2 - 2*logsigma, 1) - 0.5*self.T Lz += kl_loss Lz = torch.mean(Lz) net_loss = Lx + Lz return net_loss
def comp_loss_and_grad(img, tgt, it, sz): dif = img-tgt loss = dif.pow(2).mean() dif = dif.detach() cdif = dif.clone().abs() cdif[:, :, 3] = 1. resdif = torch.nn.functional.interpolate(cdif.permute(2, 0, 1).unsqueeze(0), sz, mode='bilinear').squeeze().permute(1, 2, 0).abs() pydiffvg.imwrite(resdif[:, :, 0:4], 'results/simple_transform_svg/dif_{:04}.png'.format(it)) dif = dif.numpy() padded = np.pad(dif, [(1, 1), (1, 1), (0, 0)], mode='edge') # print(padded[:-2,:,:].shape) grad_x = (padded[:-2, :, :]-padded[2:, :, :])[:, 1:-1, :] grad_y = (padded[:, :-2, :]-padded[:, 2:, :])[1:-1, :, :] resshape = dif.shape resshape = (resshape[0], resshape[1], 2) res = np.zeros(resshape) for x in range(resshape[0]): for y in range(resshape[1]): A = np.concatenate((grad_x[x, y, :][:, np.newaxis], grad_y[x, y, :][:, np.newaxis]), axis=1) b = -dif[x, y, :] v = np.linalg.lstsq(np.dot(A.T, A), np.dot(A.T, b)) res[x, y, :] = v[0] return loss, res
def interpolate(args): chkpt = VAE_OUTPUT if args.conditional: chkpt += "_conditional" if args.fc: chkpt += "_fc" meta = ttools.Checkpointer.load_meta(chkpt, prefix="g_") if meta is None: LOG.info("No metadata in checkpoint (or no checkpoint), aborting.") return model = VectorMNISTVAE(imsize=128, **meta) checkpointer = ttools.Checkpointer(chkpt, model, prefix="g_") checkpointer.load_latest() model.eval() # Sample some latent vectors bs = 10 z = th.randn(bs, model.zdim) label = None label = th.arange(0, 10) animation = [] nframes = 60 with th.no_grad(): for idx, _z in enumerate(z): if idx == 0: # skip first continue _z0 = z[idx-1].unsqueeze(0).repeat(nframes, 1) _z = _z.unsqueeze(0).repeat(nframes, 1) if args.conditional: _label = label[idx].unsqueeze(0).repeat(nframes) else: _label = None # interp weights alpha = th.linspace(0, 1, nframes).view(nframes, 1) batch = alpha*_z + (1.0 - alpha)*_z0 images, aux = model.decode(batch, label=_label) images += 1.0 images /= 2.0 animation.append(images) anim_dir = os.path.join(chkpt, "interpolation") os.makedirs(anim_dir, exist_ok=True) animation = th.cat(animation, 0) for idx, frame in enumerate(animation): frame = frame.squeeze() frame = th.clamp(frame, 0, 1).cpu().numpy() path = os.path.join(anim_dir, "frame%03d.png" % idx) pydiffvg.imwrite(frame, path, gamma=2.2) LOG.info("Results saved to %s", anim_dir)
def main(args): pydiffvg.set_device(th.device('cuda:1')) # Load SVG svg = os.path.join(args.svg) canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(svg) # Save initial state ref = render(canvas_width, canvas_height, shapes, shape_groups) pydiffvg.imwrite(ref.cpu(), args.out, gamma=2.2)
def print_gradimg(gradimg, it, shape=None): out = torch.zeros((gradimg.shape[0], gradimg.shape[1], 3), requires_grad=False, dtype=torch.float32) for x in range(gradimg.shape[0]): for y in range(gradimg.shape[1]): # h = math.atan2(gradimg[x, y, 1], gradimg[x, y, 0]) # s = math.tanh(np.linalg.norm(gradimg[x, y, :])) # v = 1. vec = (gradimg[x, y, :].clip(min=-1, max=1)/2)+.5 # out[x,y,:]=torch.tensor(colorsys.hsv_to_rgb(h,s,v),dtype=torch.float32) out[x, y, :] = torch.tensor([vec[0], vec[1], 0]) if shape is not None: out = torch.nn.functional.interpolate(out.permute(2, 0, 1).unsqueeze(0), shape, mode='bilinear').squeeze().permute(1, 2, 0) pydiffvg.imwrite(out.cpu(), 'results/simple_transform_svg/grad_{:04}.png'.format(it))
def main(svg_dirs): pydiffvg.set_device(th.device('cuda:1')) assert os.path.exists(svg_dirs) svg_files = os.listdir(svg_dirs) for svg_file in svg_files: if '.svg' not in svg_file: continue svg_file_path = os.path.join(svg_dirs, svg_file) out_file_path = svg_file_path.replace('.svg', '.png') # Load SVG canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(svg_file_path) # Save initial state ref = render(canvas_width, canvas_height, shapes, shape_groups) pydiffvg.imwrite(ref.cpu(), out_file_path, gamma=2.2)
def gen_image_from_curves(self, t, shapes, shape_groups, gamma, background_image): render = pydiffvg.RenderFunction.apply scene_args = pydiffvg.RenderFunction.serialize_scene( \ self.canvas_width, self.canvas_height, shapes, shape_groups) img = render(self.canvas_width, self.canvas_height, 2, 2, t, background_image, *scene_args) img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4]) img = img[:, :, :3] dir_ = "./gens/" if not os.path.exists(dir_): os.mkdir(dir_) if t % 200 == 1: pydiffvg.imwrite(img.cpu(), os.path.join(dir_, 'iter_{}.png'.format(int(t / 5))), gamma=gamma) img = img[:, :, :3] img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) return img
def main(args): pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene( args.content_file) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Transform to gamma space pydiffvg.imwrite(img.cpu(), 'results/style_transfer/init.png', gamma=1.0) # HWC -> NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW loader = transforms.Compose([transforms.ToTensor() ]) # transform it into a torch tensor def image_loader(image_name): image = Image.open(image_name) # fake batch dimension required to fit network's input dimensions image = loader(image).unsqueeze(0) return image.to(pydiffvg.get_device(), torch.float) style_img = image_loader(args.style_img) # alpha blend content with a gray background content_img = img[:, :3, :, :] * img[:, 3, :, :] + \ 0.5 * torch.ones([1, 3, img.shape[2], img.shape[3]]) * \ (1 - img[:, 3, :, :]) assert style_img.size() == content_img.size(), \ "we need to import style and content images of the same size" # unloader = transforms.ToPILImage() # reconvert into PIL image class ContentLoss(nn.Module): def __init__( self, target, ): super(ContentLoss, self).__init__() # we 'detach' the target content from the tree used # to dynamically compute the gradient: this is a stated value, # not a variable. Otherwise the forward method of the criterion # will throw an error. self.target = target.detach() def forward(self, input): self.loss = F.mse_loss(input, self.target) return input def gram_matrix(input): a, b, c, d = input.size() # a=batch size(=1) # b=number of feature maps # (c,d)=dimensions of a f. map (N=c*d) features = input.view(a * b, c * d) # resise F_XL into \hat F_XL G = torch.mm(features, features.t()) # compute the gram product # we 'normalize' the values of the gram matrix # by dividing by the number of element in each feature maps. return G.div(a * b * c * d) class StyleLoss(nn.Module): def __init__(self, target_feature): super(StyleLoss, self).__init__() self.target = gram_matrix(target_feature).detach() def forward(self, input): G = gram_matrix(input) self.loss = F.mse_loss(G, self.target) return input device = pydiffvg.get_device() cnn = models.vgg19(pretrained=True).features.to(device).eval() cnn_normalization_mean = torch.tensor([0.485, 0.456, 0.406]).to(device) cnn_normalization_std = torch.tensor([0.229, 0.224, 0.225]).to(device) # create a module to normalize input image so we can easily put it in a # nn.Sequential class Normalization(nn.Module): def __init__(self, mean, std): super(Normalization, self).__init__() # .view the mean and std to make them [C x 1 x 1] so that they can # directly work with image Tensor of shape [B x C x H x W]. # B is batch size. C is number of channels. H is height and W is width. self.mean = mean.clone().view(-1, 1, 1) self.std = std.clone().view(-1, 1, 1) def forward(self, img): # normalize img return (img - self.mean) / self.std # desired depth layers to compute style/content losses : content_layers_default = ['conv_4'] style_layers_default = ['conv_1', 'conv_2', 'conv_3', 'conv_4', 'conv_5'] def get_style_model_and_losses(cnn, normalization_mean, normalization_std, style_img, content_img, content_layers=content_layers_default, style_layers=style_layers_default): cnn = copy.deepcopy(cnn) # normalization module normalization = Normalization(normalization_mean, normalization_std).to(device) # just in order to have an iterable access to or list of content/syle # losses content_losses = [] style_losses = [] # assuming that cnn is a nn.Sequential, so we make a new nn.Sequential # to put in modules that are supposed to be activated sequentially model = nn.Sequential(normalization) i = 0 # increment every time we see a conv for layer in cnn.children(): if isinstance(layer, nn.Conv2d): i += 1 name = 'conv_{}'.format(i) elif isinstance(layer, nn.ReLU): name = 'relu_{}'.format(i) # The in-place version doesn't play very nicely with the ContentLoss # and StyleLoss we insert below. So we replace with out-of-place # ones here. layer = nn.ReLU(inplace=False) elif isinstance(layer, nn.MaxPool2d): name = 'pool_{}'.format(i) elif isinstance(layer, nn.BatchNorm2d): name = 'bn_{}'.format(i) else: raise RuntimeError('Unrecognized layer: {}'.format( layer.__class__.__name__)) model.add_module(name, layer) if name in content_layers: # add content loss: target = model(content_img).detach() content_loss = ContentLoss(target) model.add_module("content_loss_{}".format(i), content_loss) content_losses.append(content_loss) if name in style_layers: # add style loss: target_feature = model(style_img).detach() style_loss = StyleLoss(target_feature) model.add_module("style_loss_{}".format(i), style_loss) style_losses.append(style_loss) # now we trim off the layers after the last content and style losses for i in range(len(model) - 1, -1, -1): if isinstance(model[i], ContentLoss) or isinstance( model[i], StyleLoss): break model = model[:(i + 1)] return model, style_losses, content_losses def run_style_transfer(cnn, normalization_mean, normalization_std, content_img, style_img, canvas_width, canvas_height, shapes, shape_groups, num_steps=500, style_weight=5000, content_weight=1): """Run the style transfer.""" print('Building the style transfer model..') model, style_losses, content_losses = get_style_model_and_losses( cnn, normalization_mean, normalization_std, style_img, content_img) point_params = [] color_params = [] stroke_width_params = [] for shape in shapes: if isinstance(shape, pydiffvg.Path): point_params.append(shape.points.requires_grad_()) stroke_width_params.append(shape.stroke_width.requires_grad_()) for shape_group in shape_groups: if isinstance(shape_group.fill_color, torch.Tensor): color_params.append(shape_group.fill_color.requires_grad_()) elif isinstance(shape_group.fill_color, pydiffvg.LinearGradient): point_params.append( shape_group.fill_color.begin.requires_grad_()) point_params.append( shape_group.fill_color.end.requires_grad_()) color_params.append( shape_group.fill_color.stop_colors.requires_grad_()) if isinstance(shape_group.stroke_color, torch.Tensor): color_params.append(shape_group.stroke_color.requires_grad_()) elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient): point_params.append( shape_group.stroke_color.begin.requires_grad_()) point_params.append( shape_group.stroke_color.end.requires_grad_()) color_params.append( shape_group.stroke_color.stop_colors.requires_grad_()) point_optimizer = optim.Adam(point_params, lr=1.0) color_optimizer = optim.Adam(color_params, lr=0.01) stroke_width_optimizers = optim.Adam(stroke_width_params, lr=0.1) print('Optimizing..') run = [0] while run[0] <= num_steps: point_optimizer.zero_grad() color_optimizer.zero_grad() stroke_width_optimizers.zero_grad() scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # alpha blend img with a gray background img = img[:, :, :3] * img[:, :, 3:4] + \ 0.5 * torch.ones([img.shape[0], img.shape[1], 3]) * \ (1 - img[:, :, 3:4]) pydiffvg.imwrite(img.cpu(), 'results/style_transfer/step_{}.png'.format( run[0]), gamma=1.0) # HWC to NCHW img = img.permute([2, 0, 1]).unsqueeze(0) model(img) style_score = 0 content_score = 0 for sl in style_losses: style_score += sl.loss for cl in content_losses: content_score += cl.loss style_score *= style_weight content_score *= content_weight loss = style_score + content_score loss.backward() run[0] += 1 if run[0] % 1 == 0: print("run {}:".format(run)) print('Style Loss : {:4f} Content Loss: {:4f}'.format( style_score.item(), content_score.item())) print() point_optimizer.step() color_optimizer.step() stroke_width_optimizers.step() for color in color_params: color.data.clamp_(0, 1) for w in stroke_width_params: w.data.clamp_(0.5, 4.0) return shapes, shape_groups shapes, shape_groups = run_style_transfer(cnn, cnn_normalization_mean, cnn_normalization_std, content_img, style_img, canvas_width, canvas_height, shapes, shape_groups) scene_args = pydiffvg.RenderFunction.serialize_scene(shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Transform to gamma space pydiffvg.imwrite(img.cpu(), 'results/style_transfer/output.png', gamma=1.0)
def main(args): canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(args.svg_file) w = int(canvas_width * args.size_scale) h = int(canvas_height * args.size_scale) pfilter = pydiffvg.PixelFilter(type=diffvg.FilterType.box, radius=torch.tensor(0.5)) use_prefiltering = False scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) num_samples_x = 16 num_samples_y = 16 render = pydiffvg.RenderFunction.apply img = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, *scene_args) pydiffvg.imwrite(img.cpu(), 'results/finite_difference_comp/img.png', gamma=1.0) epsilon = 0.1 def perturb_scene(axis, epsilon): shapes[2].points[:, axis] += epsilon # for s in shapes: # if isinstance(s, pydiffvg.Circle): # s.center[axis] += epsilon # elif isinstance(s, pydiffvg.Ellipse): # s.center[axis] += epsilon # elif isinstance(s, pydiffvg.Path): # s.points[:, axis] += epsilon # elif isinstance(s, pydiffvg.Polygon): # s.points[:, axis] += epsilon # elif isinstance(s, pydiffvg.Rect): # s.p_min[axis] += epsilon # s.p_max[axis] += epsilon # for s in shape_groups: # if isinstance(s.fill_color, pydiffvg.LinearGradient): # s.fill_color.begin[axis] += epsilon # s.fill_color.end[axis] += epsilon perturb_scene(0, epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) render = pydiffvg.RenderFunction.apply img0 = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, *scene_args) forward_diff = (img0 - img) / (epsilon) forward_diff = forward_diff.sum(axis=2) x_diff_max = 1.5 x_diff_min = -1.5 print(forward_diff.max()) print(forward_diff.min()) forward_diff = cm.viridis( normalize(forward_diff, x_diff_min, x_diff_max).cpu().numpy()) pydiffvg.imwrite( forward_diff, 'results/finite_difference_comp/shared_edge_forward_diff.png', gamma=1.0) perturb_scene(0, -2 * epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) img1 = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, *scene_args) backward_diff = (img - img1) / (epsilon) backward_diff = backward_diff.sum(axis=2) print(backward_diff.max()) print(backward_diff.min()) backward_diff = cm.viridis( normalize(backward_diff, x_diff_min, x_diff_max).cpu().numpy()) pydiffvg.imwrite( backward_diff, 'results/finite_difference_comp/shared_edge_backward_diff.png', gamma=1.0) perturb_scene(0, epsilon) num_samples_x = 4 num_samples_y = 4 scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) render_grad = pydiffvg.RenderFunction.render_grad img_grad = render_grad( torch.ones(h, w, 4), w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed *scene_args) print(img_grad[:, :, 0].max()) print(img_grad[:, :, 0].min()) x_diff = cm.viridis( normalize(img_grad[:, :, 0], x_diff_min, x_diff_max).cpu().numpy()) pydiffvg.imwrite(x_diff, 'results/finite_difference_comp/ours_x_diff.png', gamma=1.0)
def main(args): case_name = args.svg_file.split('/')[-1].split('.')[0] canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(args.svg_file) w = int(canvas_width * args.size_scale) h = int(canvas_height * args.size_scale) print(w, h) curve_counts = 0 for s in shapes: if isinstance(s, pydiffvg.Circle): curve_counts += 1 elif isinstance(s, pydiffvg.Ellipse): curve_counts += 1 elif isinstance(s, pydiffvg.Path): curve_counts += len(s.num_control_points) elif isinstance(s, pydiffvg.Polygon): curve_counts += len(s.points) - 1 if s.is_closed: curve_counts += 1 elif isinstance(s, pydiffvg.Rect): curve_counts += 1 print('curve_counts:', curve_counts) pfilter = pydiffvg.PixelFilter(type=diffvg.FilterType.box, radius=torch.tensor(0.5)) use_prefiltering = args.use_prefiltering print('use_prefiltering:', use_prefiltering) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) num_samples_x = args.num_spp num_samples_y = args.num_spp if (use_prefiltering): num_samples_x = 1 num_samples_y = 1 render = pydiffvg.RenderFunction.apply img = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, # background_image *scene_args) pydiffvg.imwrite( img.cpu(), f'results/finite_difference_comp/{case_name}_{use_prefiltering}/img.png', gamma=1.0) epsilon = 0.1 def perturb_scene(axis, epsilon): for s in shapes: if isinstance(s, pydiffvg.Circle): s.center[axis] += epsilon elif isinstance(s, pydiffvg.Ellipse): s.center[axis] += epsilon elif isinstance(s, pydiffvg.Path): s.points[:, axis] += epsilon elif isinstance(s, pydiffvg.Polygon): s.points[:, axis] += epsilon elif isinstance(s, pydiffvg.Rect): s.p_min[axis] += epsilon s.p_max[axis] += epsilon for s in shape_groups: if isinstance(s.fill_color, pydiffvg.LinearGradient): s.fill_color.begin[axis] += epsilon s.fill_color.end[axis] += epsilon perturb_scene(0, epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) render = pydiffvg.RenderFunction.apply img0 = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, # background_image *scene_args) perturb_scene(0, -2 * epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) img1 = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, # background_image *scene_args) x_diff = (img0 - img1) / (2 * epsilon) x_diff = x_diff.sum(axis=2) x_diff_max = x_diff.max() * args.clamping_factor x_diff_min = x_diff.min() * args.clamping_factor print(x_diff.max()) print(x_diff.min()) x_diff = cm.viridis( normalize(x_diff, x_diff_min, x_diff_max).cpu().numpy()) pydiffvg.imwrite( x_diff, f'results/finite_difference_comp//{case_name}_{use_prefiltering}/finite_x_diff.png', gamma=1.0) perturb_scene(0, epsilon) perturb_scene(1, epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) render = pydiffvg.RenderFunction.apply img0 = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, # background_image *scene_args) perturb_scene(1, -2 * epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) img1 = render( w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, # background_image *scene_args) y_diff = (img0 - img1) / (2 * epsilon) y_diff = y_diff.sum(axis=2) y_diff_max = y_diff.max() * args.clamping_factor y_diff_min = y_diff.min() * args.clamping_factor y_diff = cm.viridis( normalize(y_diff, y_diff_min, y_diff_max).cpu().numpy()) pydiffvg.imwrite( y_diff, f'results/finite_difference_comp/{case_name}_{use_prefiltering}/finite_y_diff.png', gamma=1.0) perturb_scene(1, epsilon) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, filter=pfilter, use_prefiltering=use_prefiltering) render_grad = pydiffvg.RenderFunction.render_grad img_grad = render_grad( torch.ones(h, w, 4, device=pydiffvg.get_device()), w, # width h, # height num_samples_x, # num_samples_x num_samples_y, # num_samples_y 0, # seed None, # background_image *scene_args) print(img_grad[:, :, 0].max()) print(img_grad[:, :, 0].min()) x_diff = cm.viridis( normalize(img_grad[:, :, 0], x_diff_min, x_diff_max).cpu().numpy()) y_diff = cm.viridis( normalize(img_grad[:, :, 1], y_diff_min, y_diff_max).cpu().numpy()) pydiffvg.imwrite( x_diff, f'results/finite_difference_comp/{case_name}_{use_prefiltering}/ours_x_diff.png', gamma=1.0) pydiffvg.imwrite( y_diff, f'results/finite_difference_comp/{case_name}_{use_prefiltering}/ours_y_diff.png', gamma=1.0)
def main(): pydiffvg.set_device(th.device('cuda:1')) # Load SVG svg = os.path.join("imgs", "peppers.svg") canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(svg) # Save initial state ref = render(canvas_width, canvas_height, shapes, shape_groups) pydiffvg.imwrite(ref.cpu(), 'results/gaussian_blur/init.png', gamma=2.2) target = F.gaussian_filter(ref.cpu().numpy(), [10, 10, 0]) target = th.from_numpy(target).to(ref.device) pydiffvg.imwrite(target.cpu(), 'results/gaussian_blur/target.png', gamma=2.2) # Collect variables to optimize points_vars = [] width_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) path.stroke_width.requires_grad = True width_vars.append(path.stroke_width) color_vars = [] for group in shape_groups: # do not optimize alpha group.fill_color[..., :3].requires_grad = True color_vars.append(group.fill_color) # Optimize points_optim = th.optim.Adam(points_vars, lr=1.0) width_optim = th.optim.Adam(width_vars, lr=1.0) color_optim = th.optim.Adam(color_vars, lr=0.01) for t in range(20): print('\niteration:', t) points_optim.zero_grad() width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. img = render(canvas_width, canvas_height, shapes, shape_groups) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/gaussian_blur/iter_{}.png'.format(t), gamma=2.2) loss = (img - target)[..., :3].pow(2).mean() print('alpha:', img[..., 3].mean().item()) print('render loss:', loss.item()) # Backpropagate the gradients. loss.backward() # Take a gradient descent step. points_optim.step() width_optim.step() color_optim.step() for group in shape_groups: group.fill_color.data.clamp_(0.0, 1.0) # Final render img = render(canvas_width, canvas_height, shapes, shape_groups) pydiffvg.imwrite(img.cpu(), 'results/gaussian_blur/final.png', gamma=2.2) # Convert the intermediate renderings to a video. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", "results/gaussian_blur/iter_%d.png", "-vb", "20M", "results/gaussian_blur/out.mp4" ])
[0.6, 0.3, 0.6, 0.8])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_open_curve_thickness/target.png', gamma=2.2) target = img.clone() # Move the path to produce initial guess # normalize points for easier learning rate points_n = torch.tensor( [ [100.0 / 256.0, 40.0 / 256.0], # base [155.0 / 256.0, 65.0 / 256.0], # control point [100.0 / 256.0, 180.0 / 256.0], # control point [65.0 / 256.0, 238.0 / 256.0] ], # base requires_grad=True) thickness_n = torch.tensor( [10.0 / 100.0, 10.0 / 100.0, 10.0 / 100.0, 10.0 / 100.0],
fill_color=torch.tensor([0.3, 0.6, 0.3, 1.0])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 510, # width 510, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # background_image *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_path/target.png', gamma=2.2) target = img.clone() # Move the path to produce initial guess # normalize points for easier learning rate noise = torch.FloatTensor(shapes[0].points.shape).uniform_(0.0, 1.0) points_n = (shapes[0].points.clone() + (noise * 60 - 30)) / 510.0 points_n.requires_grad = True color = torch.tensor([0.3, 0.2, 0.5, 1.0], requires_grad=True) shapes[0].points = points_n * 510 path_group.fill_color = color scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( 510, # width 510, # height
shapes = [circle] circle_group = pydiffvg.ShapeGroup(shape_ids = torch.tensor([0]), fill_color = color) shape_groups = [circle_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # background_image *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_gradient/target.png', gamma=2.2) target = img.clone() # Move the circle to produce initial guess # normalize radius & center for easier learning rate radius_n = torch.tensor(20.0 / 256.0, requires_grad=True) center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True) begin_n = torch.tensor([100.0 / 256.0, 100.0 / 256.0], requires_grad=True) end_n = torch.tensor([150.0 / 256.0, 150.0 / 256.0], requires_grad=True) stop_colors = torch.tensor([[0.1, 0.9, 0.2, 1.0], [0.5, 0.3, 0.6, 1.0]], requires_grad=True) color.begin = begin_n * 256 color.end = end_n * 256 color.stop_colors = stop_colors circle.radius = radius_n * 256 circle.center = center_n * 256
canvas_height, shapes, shape_groups, output_type=pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # background_image *scene_args) img = img / 256 # Normalize SDF to [0, 1] pydiffvg.imwrite(img.cpu(), 'results/test_eval_positions/target.png') target = img.clone() # Move the circle to produce initial guess # normalize radius & center for easier learning rate radius_n = torch.tensor(20.0 / 256.0, requires_grad=True) center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True) color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True) circle.radius = radius_n * 256 circle.center = center_n * 256 circle_group.fill_color = color scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups,
shape_to_canvas = torch.eye(3, 3)) shape_groups = [ellipse_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_ellipse_transform/target.png', gamma=2.2) target = img.clone() # Affine transform the ellipse to produce initial guess color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True) affine = torch.zeros(2, 3) affine[0, 0] = 1.3 affine[0, 1] = 0.2 affine[0, 2] = 0.1 affine[1, 0] = 0.2 affine[1, 1] = 0.6 affine[1, 2] = 0.3 affine.requires_grad = True shape_to_canvas = torch.cat((affine, torch.tensor([[0.0, 0.0, 1.0]])), axis=0) ellipse_group.fill_color = color
shapes, shape_groups, output_type=pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 1, # num_samples_x 1, # num_samples_y 0, # seed None, # background_image *scene_args) img /= 256.0 cm = plt.get_cmap('viridis') img = cm(img.cpu().squeeze()) pydiffvg.imwrite(img, 'results/quadratic_distance_approx/ref_sdf.png') scene_args = pydiffvg.RenderFunction.serialize_scene(canvas_width, canvas_height, shapes, shape_groups) img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # background_image *scene_args) pydiffvg.imwrite(img.cpu(), 'results/quadratic_distance_approx/ref_color.png') shapes[0].use_distance_approx = True
settings.global_override(["paths", "optimize_points"], False) settings.global_override(["transforms", "transform_lr"], 1e-2) settings.undefault("linearGradient3152") settings.retrieve( "linearGradient3152")[0]["transforms"]["optimize_transforms"] = False #optim=pydiffvg.OptimizableSvg("note_small.svg",settings,verbose=True) optim = pydiffvg.OptimizableSvg("heart_green.svg", settings, verbose=True) #img=torchvision.transforms.ToTensor()(Image.open("note_transformed.png")).permute(1,2,0) img = torchvision.transforms.ToTensor()( Image.open("heart_green_90.png")).permute(1, 2, 0) name = "heart_green_90" pydiffvg.imwrite(img.cpu(), 'results/simple_transform_svg/target.png') target = img.clone().detach().requires_grad_(False) img = optim.render() pydiffvg.imwrite(img.cpu(), 'results/simple_transform_svg/init.png') def smooth(input, kernel): input = torch.nn.functional.pad(input.permute(2, 0, 1).unsqueeze(0), (2, 2, 2, 2), mode='reflect') output = kernel(input) return output def printimg(optim):
canvas_height, shapes, shape_groups, output_type=pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) img = img / 256 # Normalize SDF to [0, 1] pydiffvg.imwrite(img.cpu(), 'results/single_circle_sdf/target.png') target = img.clone() # Move the circle to produce initial guess # normalize radius & center for easier learning rate radius_n = torch.tensor(20.0 / 256.0, requires_grad=True) center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True) color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True) circle.radius = radius_n * 256 circle.center = center_n * 256 circle_group.fill_color = color scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups,
def main(args): if args.seed: np.random.seed(args.seed) random.seed(args.seed) torch.manual_seed(args.seed) pydiffvg.set_print_timing(False) outdir = os.path.join(args.results_dir, args.prompt, args.subdir) # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 224, 224 margin = args.initial_margin total_paths = args.open_paths + args.closed_paths step = min(args.step, total_paths) if step == 0: step = total_paths fill_color = None stroke_color = None shapes = [] shape_groups = [] losses = [] tt = 0 for num_paths in range(step, total_paths + 1, step): for i in range(num_paths - step, num_paths): num_segments = random.randint(1, args.extra_segments + 1) p0 = (margin + random.random() * (1 - 2 * margin), margin + random.random() * (1 - 2 * margin)) points = [p0] is_closed = i >= args.open_paths if is_closed: num_segments += 2 for j in range(num_segments): p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) if is_closed and j < num_segments - 1 or not is_closed: points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height stroke_width = torch.tensor(1.0) color = torch.tensor([ random.random(), random.random(), random.random(), random.random() ]) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=stroke_width, is_closed=is_closed) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=color if is_closed else None, stroke_color=None if is_closed else color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'init.png'), gamma=gamma) points_vars = [] stroke_width_vars = [] color_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) if not path.is_closed and args.max_width > 1: path.stroke_width.requires_grad = True stroke_width_vars.append(path.stroke_width) for group in shape_groups: if group.fill_color is not None: group.fill_color.requires_grad = True color_vars.append(group.fill_color) else: group.stroke_color.requires_grad = True color_vars.append(group.stroke_color) # Embed prompt text_features = clip_utils.embed_text(args.prompt) # Optimize points_optim = torch.optim.Adam(points_vars, lr=args.points_lr) if len(stroke_width_vars) > 0: width_optim = torch.optim.Adam(stroke_width_vars, lr=args.width_lr) color_optim = torch.optim.Adam(color_vars, lr=args.color_lr) # Adam iterations. final = False this_step_iters = max(1, round(args.num_iter * step / total_paths)) if num_paths + step > total_paths: final = True this_step_iters += args.extra_iter for t in range(this_step_iters): points_optim.zero_grad() if len(stroke_width_vars) > 0: width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y tt, # seed None, *scene_args) # Save the intermediate render. with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'iter_{}.png'.format(tt)), gamma=gamma) image_features = clip_utils.embed_image(img) loss = -torch.cosine_similarity( text_features, image_features, dim=-1).mean() # Backpropagate the gradients. loss.backward() losses.append(loss.item()) # Take a gradient descent step. points_optim.step() if len(stroke_width_vars) > 0: width_optim.step() color_optim.step() for path in shapes: path.points.data[:, 0].clamp_(0.0, canvas_width) path.points.data[:, 1].clamp_(0.0, canvas_height) if not path.is_closed: path.stroke_width.data.clamp_(1.0, args.max_width) for group in shape_groups: if group.fill_color is not None: group.fill_color.data[:3].clamp_(0.0, 1.0) group.fill_color.data[3].clamp_(args.min_alpha, 1.0) else: group.stroke_color.data[:3].clamp_(0.0, 1.0) group.stroke_color.data[3].clamp_(args.min_alpha, 1.0) if tt % 10 == 0 or final and t == this_step_iters - 1: print('%d loss=%.3f' % (tt, 1 + losses[-1])) pydiffvg.save_svg( os.path.join(outdir, 'iter_{}.svg'.format(tt)), canvas_width, canvas_height, shapes, shape_groups) clip_utils.plot_losses(losses, outdir) tt += 1 # Render the final result. img = render( args.final_px, # width args.final_px, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Save the intermediate render with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'final.png'), gamma=gamma) # Convert the intermediate renderings to a video with a white background. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", os.path.join(outdir, "iter_%d.png"), "-vb", "20M", "-filter_complex", "color=white,format=rgb24[c];[c][0]scale2ref[c][i];[c][i]overlay=format=auto:shortest=1,setsar=1", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-profile:v", "baseline", "-movflags", "+faststart", os.path.join(outdir, "out.mp4") ])
def run_style_transfer(cnn, normalization_mean, normalization_std, content_img, style_img, canvas_width, canvas_height, shapes, shape_groups, num_steps=500, style_weight=5000, content_weight=1): """Run the style transfer.""" print('Building the style transfer model..') model, style_losses, content_losses = get_style_model_and_losses( cnn, normalization_mean, normalization_std, style_img, content_img) point_params = [] color_params = [] stroke_width_params = [] for shape in shapes: if isinstance(shape, pydiffvg.Path): point_params.append(shape.points.requires_grad_()) stroke_width_params.append(shape.stroke_width.requires_grad_()) for shape_group in shape_groups: if isinstance(shape_group.fill_color, torch.Tensor): color_params.append(shape_group.fill_color.requires_grad_()) elif isinstance(shape_group.fill_color, pydiffvg.LinearGradient): point_params.append( shape_group.fill_color.begin.requires_grad_()) point_params.append( shape_group.fill_color.end.requires_grad_()) color_params.append( shape_group.fill_color.stop_colors.requires_grad_()) if isinstance(shape_group.stroke_color, torch.Tensor): color_params.append(shape_group.stroke_color.requires_grad_()) elif isinstance(shape_group.stroke_color, pydiffvg.LinearGradient): point_params.append( shape_group.stroke_color.begin.requires_grad_()) point_params.append( shape_group.stroke_color.end.requires_grad_()) color_params.append( shape_group.stroke_color.stop_colors.requires_grad_()) point_optimizer = optim.Adam(point_params, lr=1.0) color_optimizer = optim.Adam(color_params, lr=0.01) stroke_width_optimizers = optim.Adam(stroke_width_params, lr=0.1) print('Optimizing..') run = [0] while run[0] <= num_steps: point_optimizer.zero_grad() color_optimizer.zero_grad() stroke_width_optimizers.zero_grad() scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # alpha blend img with a gray background img = img[:, :, :3] * img[:, :, 3:4] + \ 0.5 * torch.ones([img.shape[0], img.shape[1], 3]) * \ (1 - img[:, :, 3:4]) pydiffvg.imwrite(img.cpu(), 'results/style_transfer/step_{}.png'.format( run[0]), gamma=1.0) # HWC to NCHW img = img.permute([2, 0, 1]).unsqueeze(0) model(img) style_score = 0 content_score = 0 for sl in style_losses: style_score += sl.loss for cl in content_losses: content_score += cl.loss style_score *= style_weight content_score *= content_weight loss = style_score + content_score loss.backward() run[0] += 1 if run[0] % 1 == 0: print("run {}:".format(run)) print('Style Loss : {:4f} Content Loss: {:4f}'.format( style_score.item(), content_score.item())) print() point_optimizer.step() color_optimizer.step() stroke_width_optimizers.step() for color in color_params: color.data.clamp_(0, 1) for w in stroke_width_params: w.data.clamp_(0.5, 4.0) return shapes, shape_groups
def main(): parser = argparse.ArgumentParser() parser.add_argument("--svg", default=os.path.join("imgs", "seamcarving", "hokusai.svg")) parser.add_argument("--optim_steps", default=10, type=int) parser.add_argument("--lr", default=1e-1, type=int) args = parser.parse_args() name = os.path.splitext(os.path.basename(args.svg))[0] root = os.path.join("results", "seam_carving", name) svg_root = os.path.join(root, "svg") os.makedirs(root, exist_ok=True) os.makedirs(os.path.join(root, "svg"), exist_ok=True) pydiffvg.set_use_gpu(False) # pydiffvg.set_device(th.device('cuda')) # Load SVG print("loading svg %s" % args.svg) canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(args.svg) print("done loading") max_size = 512 scale_factor = max_size / max(canvas_width, canvas_height) print("rescaling from %dx%d with scale %f" % (canvas_width, canvas_height, scale_factor)) canvas_width = int(canvas_width * scale_factor) canvas_height = int(canvas_height * scale_factor) print("new shape %dx%d" % (canvas_width, canvas_height)) vector_rescale(shapes, scale_x=scale_factor, scale_y=scale_factor) # Shrink image by 33 % # num_seams_to_remove = 2 num_seams_to_remove = canvas_width // 3 new_canvas_width = canvas_width - num_seams_to_remove scaling = new_canvas_width * 1.0 / canvas_width # Naive scaling baseline print("rendering naive rescaling...") vector_rescale(shapes, scale_x=scaling) resized = render(new_canvas_width, canvas_height, shapes, shape_groups) pydiffvg.imwrite(resized.cpu(), os.path.join(root, 'uniform_scaling.png'), gamma=2.2) pydiffvg.save_svg(os.path.join(svg_root, 'uniform_scaling.svg'), canvas_width, canvas_height, shapes, shape_groups, use_gamma=False) vector_rescale(shapes, scale_x=1.0 / scaling) # bring back original coordinates print("saved naiving scaling") # Save initial state print("rendering initial state...") im = render(canvas_width, canvas_height, shapes, shape_groups) pydiffvg.imwrite(im.cpu(), os.path.join(root, 'init.png'), gamma=2.2) pydiffvg.save_svg(os.path.join(svg_root, 'init.svg'), canvas_width, canvas_height, shapes, shape_groups, use_gamma=False) print("saved initial state") # Optimize # color_optim = th.optim.Adam(color_vars, lr=0.01) retargeted = im[..., :3].cpu().numpy() previous_width = canvas_width print("carving seams") for seam_idx in range(num_seams_to_remove): print('\nseam', seam_idx + 1, 'of', num_seams_to_remove) # Remove a seam retargeted = carve_seam(retargeted) current_width = canvas_width - seam_idx - 1 scale_factor = current_width * 1.0 / previous_width previous_width = current_width padded = np.zeros((canvas_height, canvas_width, 4)) padded[:, :-seam_idx - 1, :3] = retargeted padded[:, :-seam_idx - 1, -1] = 1.0 # alpha padded = th.from_numpy(padded).to(im.device) # Remap points to the smaller canvas and # collect variables to optimize points_vars = [] # width_vars = [] mini, maxi = canvas_width, 0 for path in shapes: path.points.requires_grad = False x = path.points[..., 0] y = path.points[..., 1] # rescale x = x * scale_factor # clip to canvas path.points[..., 0] = th.clamp(x, 0, current_width) path.points[..., 1] = th.clamp(y, 0, canvas_height) path.points.requires_grad = True points_vars.append(path.points) path.stroke_width.requires_grad = True # width_vars.append(path.stroke_width) mini = min(mini, path.points.min().item()) maxi = max(maxi, path.points.max().item()) print("points", mini, maxi, "scale", scale_factor) # recreate an optimizer so we don't carry over the previous update # (momentum)? geom_optim = th.optim.Adam(points_vars, lr=args.lr) for step in range(args.optim_steps): geom_optim.zero_grad() img = render(canvas_width, canvas_height, shapes, shape_groups, samples=2) pydiffvg.imwrite(img.cpu(), os.path.join( root, "seam_%03d_iter_%02d.png" % (seam_idx, step)), gamma=2.2) # NO alpha loss = (img - padded)[..., :3].pow(2).mean() # loss = (img - padded).pow(2).mean() print('render loss:', loss.item()) # Backpropagate the gradients. loss.backward() # Take a gradient descent step. geom_optim.step() pydiffvg.save_svg(os.path.join(svg_root, "seam%03d.svg" % seam_idx), canvas_width - seam_idx, canvas_height, shapes, shape_groups, use_gamma=False) for path in shapes: mini = min(mini, path.points.min().item()) maxi = max(maxi, path.points.max().item()) print("points", mini, maxi) img = render(canvas_width, canvas_height, shapes, shape_groups) img = img[:, :-num_seams_to_remove] pydiffvg.imwrite(img.cpu(), os.path.join(root, 'final.png'), gamma=2.2) pydiffvg.imwrite(retargeted, os.path.join(root, 'ref.png'), gamma=2.2) pydiffvg.save_svg(os.path.join(svg_root, 'final.svg'), canvas_width - num_seams_to_remove + 1, canvas_height, shapes, shape_groups, use_gamma=False) # Convert the intermediate renderings to a video. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", os.path.join(root, "seam_%03d_iter_00.png"), "-vb", "20M", os.path.join(root, "out.mp4") ])
shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # background_image *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_curve_outline/target.png', gamma=2.2) target = img.clone() # Move the path to produce initial guess # normalize points for easier learning rate points_n = torch.tensor( [ [100.0 / 256.0, 40.0 / 256.0], # base [155.0 / 256.0, 65.0 / 256.0], # control point [100.0 / 256.0, 180.0 / 256.0], # control point [65.0 / 256.0, 238.0 / 256.0], # base [100.0 / 256.0, 200.0 / 256.0], # control point [170.0 / 256.0, 55.0 / 256.0], # control point [220.0 / 256.0, 100.0 / 256.0], # base [210.0 / 256.0, 80.0 / 256.0], # control point
shape_groups = [circle_group] scene_args = pydiffvg.RenderFunction.serialize_scene(canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'logs/single_circle/target.png', gamma=2.2) target = img.clone() # Move the circle to produce initial guess # normalize radius & center for easier learning rate radius_n = torch.tensor(20.0 / 256.0, requires_grad=True) center_n = torch.tensor([108.0 / 256.0, 138.0 / 256.0], requires_grad=True) color = torch.tensor([0.3, 0.2, 0.8, 1.0], requires_grad=True) circle.radius = radius_n * 256 circle.center = center_n * 256 circle_group.fill_color = color scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( 256, # width 256, # height
def main(args): # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device()) #target = torch.from_numpy(skimage.io.imread('imgs/lena.png')).to(torch.float32) / 255.0 target = torch.from_numpy(skimage.io.imread(args.target)).to( torch.float32) / 255.0 target = target.pow(gamma) target = target.to(pydiffvg.get_device()) target = target.unsqueeze(0) target = target.permute(0, 3, 1, 2) # NHWC -> NCHW #target = torch.nn.functional.interpolate(target, size = [256, 256], mode = 'area') canvas_width, canvas_height = target.shape[3], target.shape[2] num_paths = args.num_paths max_width = args.max_width random.seed(1234) torch.manual_seed(1234) shapes = [] shape_groups = [] if args.use_blob: for i in range(num_paths): num_segments = random.randint(3, 5) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) if j < num_segments - 1: points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) shape_groups.append(path_group) else: for i in range(num_paths): num_segments = random.randint(1, 3) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height #points = torch.rand(3 * num_segments + 1, 2) * min(canvas_width, canvas_height) path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/init.png', gamma=gamma) points_vars = [] stroke_width_vars = [] color_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) if not args.use_blob: for path in shapes: path.stroke_width.requires_grad = True stroke_width_vars.append(path.stroke_width) if args.use_blob: for group in shape_groups: group.fill_color.requires_grad = True color_vars.append(group.fill_color) else: for group in shape_groups: group.stroke_color.requires_grad = True color_vars.append(group.stroke_color) # Optimize points_optim = torch.optim.Adam(points_vars, lr=1.0) if len(stroke_width_vars) > 0: width_optim = torch.optim.Adam(stroke_width_vars, lr=0.1) color_optim = torch.optim.Adam(color_vars, lr=0.01) # Adam iterations. for t in range(args.num_iter): print('iteration:', t) points_optim.zero_grad() if len(stroke_width_vars) > 0: width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y t, # seed None, *scene_args) # Compose img with white background img = img[:, :, 3:4] * img[:, :, :3] + torch.ones( img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4]) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/iter_{}.png'.format(t), gamma=gamma) img = img[:, :, :3] # Convert img from HWC to NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW if args.use_lpips_loss: loss = perception_loss( img, target) + (img.mean() - target.mean()).pow(2) else: loss = (img - target).pow(2).mean() print('render loss:', loss.item()) # Backpropagate the gradients. loss.backward() # Take a gradient descent step. points_optim.step() if len(stroke_width_vars) > 0: width_optim.step() color_optim.step() if len(stroke_width_vars) > 0: for path in shapes: path.stroke_width.data.clamp_(1.0, max_width) if args.use_blob: for group in shape_groups: group.fill_color.data.clamp_(0.0, 1.0) else: for group in shape_groups: group.stroke_color.data.clamp_(0.0, 1.0) if t % 10 == 0 or t == args.num_iter - 1: pydiffvg.save_svg( 'results/painterly_rendering/iter_{}.svg'.format(t), canvas_width, canvas_height, shapes, shape_groups) # Render the final result. img = render( target.shape[1], # width target.shape[0], # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/final.png'.format(t), gamma=gamma) # Convert the intermediate renderings to a video. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", "results/painterly_rendering/iter_%d.png", "-vb", "20M", "results/painterly_rendering/out.mp4" ])
infile = 'linux.svg' canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(infile) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'test_old.png', gamma=1.0) #optim=OptimizableSvg('linux.svg',verbose=True) optim = OptimizableSvg(infile, verbose=True) scene = optim.build_scene() scene_args = pydiffvg.RenderFunction.serialize_scene(*scene) render = pydiffvg.RenderFunction.apply img = render( scene[0], # width scene[1], # height 2, # num_samples_x 2, # num_samples_y 0, # seed *scene_args)
fill_color = torch.tensor([0.3, 0.6, 0.3, 1.0])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups, output_type = pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render(510, # width 510, # height 1, # num_samples_x 1, # num_samples_y 0, # seed None, # background_image *scene_args) img = img / 510 # Normalize SDF to [0, 1] pydiffvg.imwrite(img.cpu(), 'results/single_path_sdf/target.png', gamma=1.0) target = img.clone() # Move the path to produce initial guess # normalize points for easier learning rate noise = torch.FloatTensor(shapes[0].points.shape).uniform_(0.0, 1.0) points_n = (shapes[0].points.clone() + (noise * 60 - 30)) / 510.0 points_n.requires_grad = True color = torch.tensor([0.3, 0.2, 0.5, 1.0], requires_grad=True) shapes[0].points = points_n * 510 path_group.fill_color = color scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups, output_type = pydiffvg.OutputType.sdf) img = render(510, # width 510, # height
def generate_samples(args): chkpt = VAE_OUTPUT if args.conditional: chkpt += "_conditional" if args.fc: chkpt += "_fc" meta = ttools.Checkpointer.load_meta(chkpt, prefix="g_") if meta is None: LOG.info("No metadata in checkpoint (or no checkpoint), aborting.") return model = VectorMNISTVAE(**meta) checkpointer = ttools.Checkpointer(chkpt, model, prefix="g_") checkpointer.load_latest() model.eval() # Sample some latent vectors n = 8 bs = n*n z = th.randn(bs, model.zdim) imsize = 28 dataset = Dataset(args.data_dir, imsize) dataloader = DataLoader(dataset, batch_size=bs, num_workers=1, shuffle=True) for batch in dataloader: ref, label = batch break autoencode = True if autoencode: LOG.info("Sampling with auto-encoder code") if not args.conditional: label = None mu, logvar = model.encode(ref, label) z = model.reparameterize(mu, logvar) else: label = None if args.conditional: label = th.clamp(th.rand(bs)*10, 0, 9).long() if args.digit is not None: label[:] = args.digit with th.no_grad(): images, aux = model.decode(z, label=label) scenes = aux["scenes"] images += 1.0 images /= 2.0 h = w = model.imsize images = images.view(n, n, h, w).permute(0, 2, 1, 3) images = images.contiguous().view(n*h, n*w) images = th.clamp(images, 0, 1).cpu().numpy() path = os.path.join(chkpt, "samples.png") pydiffvg.imwrite(images, path, gamma=2.2) if autoencode: ref += 1.0 ref /= 2.0 ref = ref.view(n, n, h, w).permute(0, 2, 1, 3) ref = ref.contiguous().view(n*h, n*w) ref = th.clamp(ref, 0, 1).cpu().numpy() path = os.path.join(chkpt, "ref.png") pydiffvg.imwrite(ref, path, gamma=2.2) # merge scenes all_shapes = [] all_shape_groups = [] cur_id = 0 for idx, s in enumerate(scenes): shapes, shape_groups, _ = s # width, height = sizes # Shift digit on canvas center_x = idx % n center_y = idx // n for shape in shapes: shape.points[:, 0] += center_x * model.imsize shape.points[:, 1] += center_y * model.imsize all_shapes.append(shape) for grp in shape_groups: grp.shape_ids[:] = cur_id cur_id += 1 all_shape_groups.append(grp) LOG.info("Generated %d shapes", len(all_shapes)) fname = os.path.join(chkpt, "digits.svg") pydiffvg.save_svg(fname, n*model.imsize, n*model.imsize, all_shapes, all_shape_groups, use_gamma=False) LOG.info("Results saved to %s", chkpt)
def main(target_path, svg_path, output_dir, num_iter=1000, use_lpips_loss=False): perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device()) target = torch.from_numpy(skimage.io.imread(target_path, as_gray=False, pilmode="RGB")).to(torch.float32) / 255.0 print("target", target.size()) target = target.pow(gamma) target = target.to(pydiffvg.get_device()) target = target.unsqueeze(0) target = target.permute(0, 3, 1, 2) # NHWC -> NCHW canvas_width, canvas_height, shapes, shape_groups = \ pydiffvg.svg_to_scene(svg_path) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # bg *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), f'{output_dir}/init.png', gamma=gamma) points_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) color_vars = {} for group in shape_groups: group.fill_color.requires_grad = True color_vars[group.fill_color.data_ptr()] = group.fill_color # color_vars = list(color_vars.values()) # Optimize points_optim = torch.optim.Adam(points_vars, lr=0.1) # color_optim = torch.optim.Adam(color_vars, lr=0.01) # Adam iterations. for t in range(num_iter): print('iteration:', t) points_optim.zero_grad() # color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups) img = render(canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # bg *scene_args) # Compose img with white background img = img[:, :, 3:4] * img[:, :, :3] + torch.ones(img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4]) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), f'{output_dir}/iter_{t}.png', gamma=gamma) img = img[:, :, :3] # Convert img from HWC to NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW # print(img.size()) # print(target.size()) if use_lpips_loss: loss = perception_loss(img, target) else: loss = (img - target).pow(2).mean() print('render loss:', loss.item()) # Backpropagate the gradients. loss.backward() # Take a gradient descent step. points_optim.step() # color_optim.step() for group in shape_groups: group.fill_color.data.clamp_(0.0, 1.0) if t % 10 == 0 or t == num_iter - 1: pydiffvg.save_svg_paths_only(f'{output_dir}/iter_{t}.svg', canvas_width, canvas_height, shapes, shape_groups) # Render the final result. scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups) img = render(canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # bg *scene_args) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), f'{output_dir}/final.png', gamma=gamma) # Convert the intermediate renderings to a video. from subprocess import call call(["ffmpeg", "-framerate", "24", "-i", f"{output_dir}/iter_%d.png", "-vb", "20M", f"{output_dir}/out.mp4"])
shape_groups=shape_groups, filter=pydiffvg.PixelFilter(type=diffvg.FilterType.hann, radius=torch.tensor(8.0))) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/optimize_pixel_filter/target.png', gamma=2.2) target = img.clone() # Change the pixel filter radius radius = torch.tensor(1.0, requires_grad=True) scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width=canvas_width, canvas_height=canvas_height, shapes=shapes, shape_groups=shape_groups, filter=pydiffvg.PixelFilter(type=diffvg.FilterType.hann, radius=radius)) img = render( 256, # width 256, # height 2, # num_samples_x