def save(self, x, save_dir, name): z, log_var = self.encode(x) all_points = self.decode(z) # print(all_points.std(dim=1)) # all_points = ((all_points-0.5)*2 + 0.5)*self.imsize # if type(self.sort_idx) == type(None): # angles = torch.atan(all_points[:,:,1]/all_points[:,:,0]).detach() # self.sort_idx = torch.argsort(angles, dim=1) # Process the batch sequentially outputs = [] for k in range(1): # Get point parameters from network shapes = [] shape_groups = [] points = all_points[k].cpu()#[self.sort_idx[k]] color = torch.cat([torch.tensor([0,0,0,1]),]) num_ctrl_pts = torch.zeros(self.curves, dtype=torch.int32) + 2 path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=color, stroke_color=color) shape_groups.append(path_group) pydiffvg.save_svg(f"{save_dir}{name}/{name}.svg", self.imsize, self.imsize, shapes, shape_groups)
def font_render(all_points, all_widths, force_cpu=True, canvas_size=32): dev = all_points.device if force_cpu: all_points = all_points.to("cpu") all_widths = all_widths.to("cpu") all_points = 0.5 * (all_points + 1.0) * canvas_size eps = 1e-4 all_points = all_points + eps * th.randn_like(all_points) bs, num_strokes, num_pts, _ = all_points.shape num_segments = (num_pts - 1) // 3 n_out = 1 output = th.zeros(bs, n_out, canvas_size, canvas_size, device=all_points.device) scenes = [] for k in range(bs): shapes = [] shape_groups = [] for p in range(num_strokes): points = all_points[k, p].contiguous().cpu() # bezier num_ctrl_pts = th.zeros(num_segments, dtype=th.int32) + 2 width = all_widths[k, p].cpu() color = th.ones(4) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=color, stroke_color=None) shape_groups.append(path_group) # Rasterize scenes.append((canvas_size, canvas_size, shapes, shape_groups)) raster = render(canvas_size, canvas_size, shapes, shape_groups, samples=2) raster = raster.permute(2, 0, 1).view(4, canvas_size, canvas_size) image = raster[:1] output[k] = image output = output.to(dev) return output, scenes
def render(self, out): num_paths = self.num_paths img_size = self.A num_segments = self.num_segments imgs = [] for b in range(out.shape[0]): shapes = [] shape_groups = [] index = 0 for i in range(num_paths): points = img_size * out[b, index: index + 2 * (num_segments + 1) + num_segments].view(-1, 2) + img_size//2 index += 2 * (num_segments + 1) stroke_width = out[b, index] index += 1 num_control_points = torch.zeros(int(np.log2(num_segments)), dtype = torch.int32, device=self.device) + 2 path = pydiffvg.Path(num_control_points = num_control_points, points = points, is_closed = False, stroke_width = stroke_width) shapes.append(path) stroke_color = out[b, index].view(1) index += 1 stroke_color = torch.cat([stroke_color, stroke_color, stroke_color, torch.tensor([1.0]).to(self.device)]) path_group = pydiffvg.ShapeGroup(shape_ids = torch.tensor([len(shapes) - 1]), fill_color = None, stroke_color = stroke_color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(img_size, img_size, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(img_size, # width img_size, # height 2, # num_samples_x 2, # num_samples_y random.randint(0, 1048576), # seed None, *scene_args) imgs.append(img[:,:,:3]) img = torch.stack(imgs, dim = 0) # channel last to channel first img = img.permute(0,3,1,2) return img.reshape(out.shape[0], -1)
def forward(self, z): out = self.fc(z) # construct paths imgs = [] for b in range(out.shape[0]): index = 0 shapes = [] shape_groups = [] for i in range(num_paths): points = img_size * out[b, index: index + 2 * (num_segments + 1)].view(-1, 2).cpu() index += 2 * (num_segments + 1) stroke_width = img_size * out[b, index].view(1).cpu() index += 1 num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=stroke_width, is_closed=False) shapes.append(path) stroke_color = out[b, index].view(1).cpu() index += 1 stroke_color = torch.cat([stroke_color, torch.tensor([0.0, 0.0, 1.0])]) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(shapes) - 1]), fill_color=None, stroke_color=stroke_color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(img_size, img_size, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(img_size, # width img_size, # height 2, # num_samples_x 2, # num_samples_y random.randint(0, 1048576), # seed None, *scene_args) img = img[:, :, :1] # HWC -> NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW imgs.append(img) img = torch.cat(imgs, dim=0) return img
def build_random_path(num_segments, canvas_width, canvas_height): num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.1 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=False) return path
def save(self, all_points, save_dir, name, verbose=False, white_background=True): # note that this if for a single shape and bs dimension should have multiple curves # print('1:', process.memory_info().rss*1e-6) render_size = self.imsize bs = all_points.shape[0] if verbose: render_size = render_size * 2 all_points = all_points * render_size num_ctrl_pts = torch.zeros(self.curves, dtype=torch.int32) + 2 shapes = [] shape_groups = [] for k in range(bs): # Get point parameters from network color = make_tensor(color[k]) points = all_points[k].cpu().contiguous() #[self.sort_idx[k]] if verbose: np.random.seed(0) colors = np.random.rand(self.curves, 4) high = np.array((0.565, 0.392, 0.173, 1)) low = np.array((0.094, 0.310, 0.635, 1)) diff = (high - low) / (self.curves) colors[:, 3] = 1 for i in range(self.curves): scale = diff * i color = low + scale color[3] = 1 color = torch.tensor(color) num_ctrl_pts = torch.zeros(1, dtype=torch.int32) + 2 if i * 3 + 4 > self.curves * 3: curve_points = torch.stack([ points[i * 3], points[i * 3 + 1], points[i * 3 + 2], points[0] ]) else: curve_points = points[i * 3:i * 3 + 4] path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=curve_points, is_closed=False, stroke_width=torch.tensor(4)) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [i]), fill_color=None, stroke_color=color) shapes.append(path) shape_groups.append(path_group) for i in range(self.curves * 3): scale = diff * (i // 3) color = low + scale color[3] = 1 color = torch.tensor(color) if i % 3 == 0: # color = torch.tensor(colors[i//3]) #green shape = pydiffvg.Rect(p_min=points[i] - 8, p_max=points[i] + 8) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [self.curves + i]), fill_color=color) else: # color = torch.tensor(colors[i//3]) #purple shape = pydiffvg.Circle(radius=torch.tensor(8.0), center=points[i]) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [self.curves + i]), fill_color=color) shapes.append(shape) shape_groups.append(group) else: path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=color, stroke_color=color) shape_groups.append(path_group) pydiffvg.save_svg(f"{save_dir}{name}/{name}.svg", self.imsize, self.imsize, shapes, shape_groups)
pydiffvg.set_print_timing(True) # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 256, 256 num_control_points = torch.tensor([2]) points = torch.tensor([ [120.0, 30.0], # base [150.0, 60.0], # control point [90.0, 198.0], # control point [60.0, 218.0] ]) # base thickness = torch.tensor([10.0, 5.0, 4.0, 20.0]) path = pydiffvg.Path(num_control_points=num_control_points, points=points, is_closed=False, stroke_width=thickness) shapes = [path] path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([0]), fill_color=None, stroke_color=torch.tensor( [0.6, 0.3, 0.6, 0.8])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x
def gen_sample(self, z, label=None): bs = z.shape[0] if self.conditional: if label is None: raise ValueError("GAN is conditional, please provide a label") # One-hot encoding of the image label label_onehot = _onehot(label) # get some embedding in_ = th.cat([z, label_onehot.float()], 1) else: in_ = z feats = self.trunk(in_) all_points = self.point_predictor(feats) all_points = all_points.view(bs, self.paths, -1, 2) if False: all_alphas = th.ones(bs, self.paths) else: all_alphas = self.alpha_predictor(feats) # stroke size between 0.5 and 3.5 px if False: all_widths = th.ones(bs, self.paths) * 1 else: all_widths = self.width_predictor(feats) all_widths = 1.5*all_widths + 0.5 all_points = all_points*(self.imsize//2) + self.imsize//2 # Process the batch sequentially outputs = [] for k in range(bs): # Get point parameters from network shapes = [] shape_groups = [] for p in range(self.paths): points = all_points[k, p].contiguous().cpu() # num_ctrl_pts = th.zeros(self.segments, dtype=th.int32)+0 num_ctrl_pts = th.zeros(self.segments, dtype=th.int32)+2 width = all_widths[k, p].cpu() alpha = all_alphas[k, p].cpu() color = th.cat([th.ones(3), alpha.view(1,)]) path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=th.tensor([len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize out = render(self.imsize, self.imsize, shapes, shape_groups, samples=self.samples) # Torch format, discard alpha, make gray out = out.permute(2, 0, 1).view(4, self.imsize, self.imsize)[:3].mean(0, keepdim=True) outputs.append(out) output = th.stack(outputs).to(z.device) aux_data = { "points": all_points, "raw_vector": output, } # output = self.postprocessor(output) # map to [-1, 1] output = output*2.0 - 1.0 return output, aux_data
def decode(self, z, label=None): bs = z.shape[0] feats = self._decode_features(z, label) if self.raster: out = self.raster_decoder(feats).view(bs, 1, self.imsize, self.imsize) return out, {} all_points = self.point_predictor(feats) all_points = all_points.view(bs, self.paths, -1, 2) all_points = all_points*(self.imsize//2-2) + self.imsize//2 if False: all_widths = th.ones(bs, self.paths) * 0.5 else: all_widths = self.width_predictor(feats) * 1.5 + .25 if False: all_alphas = th.ones(bs, self.paths) else: all_alphas = self.alpha_predictor(feats) # Process the batch sequentially outputs = [] scenes = [] for k in range(bs): # Get point parameters from network shapes = [] shape_groups = [] for p in range(self.paths): points = all_points[k, p].contiguous().cpu() width = all_widths[k, p].cpu() alpha = all_alphas[k, p].cpu() color = th.cat([th.ones(3), alpha.view(1,)]) num_ctrl_pts = th.zeros(self.segments, dtype=th.int32) + 2 path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=th.tensor([len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) scenes.append( [shapes, shape_groups, (self.imsize, self.imsize)]) # Rasterize out = render(self.imsize, self.imsize, shapes, shape_groups, samples=self.samples) # Torch format, discard alpha, make gray out = out.permute(2, 0, 1).view(4, self.imsize, self.imsize)[:3].mean(0, keepdim=True) outputs.append(out) output = th.stack(outputs).to(z.device) auxdata = { "points": all_points, "scenes": scenes, } # map to [-1, 1] output = output*2.0 - 1.0 return output, auxdata
def main(args): # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device()) #target = torch.from_numpy(skimage.io.imread('imgs/lena.png')).to(torch.float32) / 255.0 target = torch.from_numpy(skimage.io.imread(args.target)).to( torch.float32) / 255.0 target = target.pow(gamma) target = target.to(pydiffvg.get_device()) target = target.unsqueeze(0) target = target.permute(0, 3, 1, 2) # NHWC -> NCHW #target = torch.nn.functional.interpolate(target, size = [256, 256], mode = 'area') canvas_width, canvas_height = target.shape[3], target.shape[2] num_paths = args.num_paths max_width = args.max_width random.seed(1234) torch.manual_seed(1234) shapes = [] shape_groups = [] if args.use_blob: for i in range(num_paths): num_segments = random.randint(3, 5) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) if j < num_segments - 1: points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) shape_groups.append(path_group) else: for i in range(num_paths): num_segments = random.randint(1, 3) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height #points = torch.rand(3 * num_segments + 1, 2) * min(canvas_width, canvas_height) path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/init.png', gamma=gamma) points_vars = [] stroke_width_vars = [] color_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) if not args.use_blob: for path in shapes: path.stroke_width.requires_grad = True stroke_width_vars.append(path.stroke_width) if args.use_blob: for group in shape_groups: group.fill_color.requires_grad = True color_vars.append(group.fill_color) else: for group in shape_groups: group.stroke_color.requires_grad = True color_vars.append(group.stroke_color) # Optimize points_optim = torch.optim.Adam(points_vars, lr=1.0) if len(stroke_width_vars) > 0: width_optim = torch.optim.Adam(stroke_width_vars, lr=0.1) color_optim = torch.optim.Adam(color_vars, lr=0.01) # Adam iterations. for t in range(args.num_iter): print('iteration:', t) points_optim.zero_grad() if len(stroke_width_vars) > 0: width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y t, # seed None, *scene_args) # Compose img with white background img = img[:, :, 3:4] * img[:, :, :3] + torch.ones( img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4]) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/iter_{}.png'.format(t), gamma=gamma) img = img[:, :, :3] # Convert img from HWC to NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW if args.use_lpips_loss: loss = perception_loss( img, target) + (img.mean() - target.mean()).pow(2) else: loss = (img - target).pow(2).mean() print('render loss:', loss.item()) # Backpropagate the gradients. loss.backward() # Take a gradient descent step. points_optim.step() if len(stroke_width_vars) > 0: width_optim.step() color_optim.step() if len(stroke_width_vars) > 0: for path in shapes: path.stroke_width.data.clamp_(1.0, max_width) if args.use_blob: for group in shape_groups: group.fill_color.data.clamp_(0.0, 1.0) else: for group in shape_groups: group.stroke_color.data.clamp_(0.0, 1.0) if t % 10 == 0 or t == args.num_iter - 1: pydiffvg.save_svg( 'results/painterly_rendering/iter_{}.svg'.format(t), canvas_width, canvas_height, shapes, shape_groups) # Render the final result. img = render( target.shape[1], # width target.shape[0], # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/final.png'.format(t), gamma=gamma) # Convert the intermediate renderings to a video. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", "results/painterly_rendering/iter_%d.png", "-vb", "20M", "results/painterly_rendering/out.mp4" ])
canvas_width, canvas_height = 256, 256 num_control_points = torch.tensor([2, 2, 2]) points = torch.tensor([ [120.0, 30.0], # base [150.0, 60.0], # control point [90.0, 198.0], # control point [60.0, 218.0], # base [90.0, 180.0], # control point [200.0, 65.0], # control point [210.0, 98.0], # base [220.0, 70.0], # control point [130.0, 55.0] ]) # control point path = pydiffvg.Path(num_control_points=num_control_points, points=points, is_closed=True) shapes = [path] path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([0]), fill_color=torch.tensor([0.3, 0.6, 0.3, 1.0])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups, output_type = pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 1, # num_samples_x 1, # num_samples_y
def __init__(self, num_paths=100, n_path_segments=3, max_width=4.0, min_width=1.0): self.num_paths = 100 self.max_width = max_width self.min_width = min_width assert self.max_width > self.min_width self.n_path_segments = n_path_segments self.classification_model = models.inception_v3( pretrained=True, transform_input=False).cuda() self.classification_model.eval() self.canvas_width = 229 # sync with inception model inputs self.canvas_height = 229 self.frequency_plot = 100 self.shapes = list() self.shape_groups = list() # fill shapes and shape_groups with random strokes for i in range(num_paths): num_control_points = torch.zeros(self.n_path_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(self.n_path_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= self.canvas_width points[:, 1] *= self.canvas_height path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor( 0.5 * (self.max_width - self.min_width)), is_closed=False) self.shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(self.shapes) - 1]), fill_color=None, stroke_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) self.shape_groups.append(path_group) self.render = pydiffvg.RenderFunction.apply # initialize variables self.point_variables = list() self.widths_variables = list() self.color_variables = list() for path in self.shapes: path.points.requires_grad = True path.stroke_width.requires_grad = True self.point_variables.append(path.points) self.widths_variables.append(path.stroke_width) for group in self.shape_groups: group.stroke_color.requires_grad = True self.color_variables.append(group.stroke_color)
def bezier_render(all_points, all_widths, all_alphas, force_cpu=True, canvas_size=32, colors=None): dev = all_points.device if force_cpu: all_points = all_points.to("cpu") all_widths = all_widths.to("cpu") all_alphas = all_alphas.to("cpu") if colors is not None: colors = colors.to("cpu") all_points = 0.5 * (all_points + 1.0) * canvas_size eps = 1e-4 all_points = all_points + eps * th.randn_like(all_points) bs, num_strokes, num_pts, _ = all_points.shape num_segments = (num_pts - 1) // 3 n_out = 3 if colors is not None else 1 output = th.zeros(bs, n_out, canvas_size, canvas_size, device=all_points.device) scenes = [] for k in range(bs): shapes = [] shape_groups = [] for p in range(num_strokes): points = all_points[k, p].contiguous().cpu() # bezier num_ctrl_pts = th.zeros(num_segments, dtype=th.int32) + 2 width = all_widths[k, p].cpu() alpha = all_alphas[k, p].cpu() if colors is not None: color = colors[k, p] else: color = th.ones(3, device=alpha.device) color = th.cat([color, alpha.view(1, )]) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize scenes.append((canvas_size, canvas_size, shapes, shape_groups)) raster = render(canvas_size, canvas_size, shapes, shape_groups, samples=2) raster = raster.permute(2, 0, 1).view(4, canvas_size, canvas_size) alpha = raster[3:4] if colors is not None: # color output image = raster[:3] alpha = alpha.repeat(3, 1, 1) else: image = raster[:1] # alpha compositing image = image * alpha output[k] = image output = output.to(dev) return output, scenes
def build_translated_path(path, dx, dy): pts = copy.deepcopy(path.points) pts[:, 0] += dx pts[:, 1] += dy return pydiffvg.Path(num_control_points=path.num_control_points, points=pts, stroke_width=path.stroke_width, is_closed=False)
# import skimage import torch # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 256, 256 num_control_points = torch.tensor([1]) points = torch.tensor([ [50.0, 30.0], # base [125.0, 400.0], # control point [170.0, 30.0] ]) # base path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor([30.0]), is_closed=False, use_distance_approx=False) shapes = [path] path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([0]), fill_color=None, stroke_color=torch.tensor( [0.5, 0.5, 0.5, 0.5])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, output_type=pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply
def main(args): if args.seed: np.random.seed(args.seed) random.seed(args.seed) torch.manual_seed(args.seed) pydiffvg.set_print_timing(False) outdir = os.path.join(args.results_dir, args.prompt, args.subdir) # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 224, 224 margin = args.initial_margin total_paths = args.open_paths + args.closed_paths step = min(args.step, total_paths) if step == 0: step = total_paths fill_color = None stroke_color = None shapes = [] shape_groups = [] losses = [] tt = 0 for num_paths in range(step, total_paths + 1, step): for i in range(num_paths - step, num_paths): num_segments = random.randint(1, args.extra_segments + 1) p0 = (margin + random.random() * (1 - 2 * margin), margin + random.random() * (1 - 2 * margin)) points = [p0] is_closed = i >= args.open_paths if is_closed: num_segments += 2 for j in range(num_segments): p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) if is_closed and j < num_segments - 1 or not is_closed: points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height stroke_width = torch.tensor(1.0) color = torch.tensor([ random.random(), random.random(), random.random(), random.random() ]) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=stroke_width, is_closed=is_closed) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=color if is_closed else None, stroke_color=None if is_closed else color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'init.png'), gamma=gamma) points_vars = [] stroke_width_vars = [] color_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) if not path.is_closed and args.max_width > 1: path.stroke_width.requires_grad = True stroke_width_vars.append(path.stroke_width) for group in shape_groups: if group.fill_color is not None: group.fill_color.requires_grad = True color_vars.append(group.fill_color) else: group.stroke_color.requires_grad = True color_vars.append(group.stroke_color) # Embed prompt text_features = clip_utils.embed_text(args.prompt) # Optimize points_optim = torch.optim.Adam(points_vars, lr=args.points_lr) if len(stroke_width_vars) > 0: width_optim = torch.optim.Adam(stroke_width_vars, lr=args.width_lr) color_optim = torch.optim.Adam(color_vars, lr=args.color_lr) # Adam iterations. final = False this_step_iters = max(1, round(args.num_iter * step / total_paths)) if num_paths + step > total_paths: final = True this_step_iters += args.extra_iter for t in range(this_step_iters): points_optim.zero_grad() if len(stroke_width_vars) > 0: width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y tt, # seed None, *scene_args) # Save the intermediate render. with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'iter_{}.png'.format(tt)), gamma=gamma) image_features = clip_utils.embed_image(img) loss = -torch.cosine_similarity( text_features, image_features, dim=-1).mean() # Backpropagate the gradients. loss.backward() losses.append(loss.item()) # Take a gradient descent step. points_optim.step() if len(stroke_width_vars) > 0: width_optim.step() color_optim.step() for path in shapes: path.points.data[:, 0].clamp_(0.0, canvas_width) path.points.data[:, 1].clamp_(0.0, canvas_height) if not path.is_closed: path.stroke_width.data.clamp_(1.0, args.max_width) for group in shape_groups: if group.fill_color is not None: group.fill_color.data[:3].clamp_(0.0, 1.0) group.fill_color.data[3].clamp_(args.min_alpha, 1.0) else: group.stroke_color.data[:3].clamp_(0.0, 1.0) group.stroke_color.data[3].clamp_(args.min_alpha, 1.0) if tt % 10 == 0 or final and t == this_step_iters - 1: print('%d loss=%.3f' % (tt, 1 + losses[-1])) pydiffvg.save_svg( os.path.join(outdir, 'iter_{}.svg'.format(tt)), canvas_width, canvas_height, shapes, shape_groups) clip_utils.plot_losses(losses, outdir) tt += 1 # Render the final result. img = render( args.final_px, # width args.final_px, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Save the intermediate render with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'final.png'), gamma=gamma) # Convert the intermediate renderings to a video with a white background. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", os.path.join(outdir, "iter_%d.png"), "-vb", "20M", "-filter_complex", "color=white,format=rgb24[c];[c][0]scale2ref[c][i];[c][i]overlay=format=auto:shortest=1,setsar=1", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-profile:v", "baseline", "-movflags", "+faststart", os.path.join(outdir, "out.mp4") ])
import pydiffvg import torch import skimage # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 256, 256 num_control_points = torch.tensor([2]) points = torch.tensor([[120.0, 30.0], # base [150.0, 60.0], # control point [ 90.0, 198.0], # control point [ 60.0, 218.0]]) # base path = pydiffvg.Path(num_control_points = num_control_points, points = points, thickness = None, is_closed = False, stroke_width = torch.tensor(5.0)) shapes = [path] path_group = pydiffvg.ShapeGroup(shape_ids = torch.tensor([0]), fill_color = None, stroke_color = torch.tensor([0.6, 0.3, 0.6, 0.8])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(256, # width 256, # height 2, # num_samples_x 2, # num_samples_y
def my_render(curve_points, curve_widths, curve_alphas, circle_centers, circle_radiuses, circle_widths, circle_alphas, canvas_size=32, colors=None): dev = curve_points.device curve_points = 0.5*(curve_points + 1.0) * canvas_size circle_centers = 0.5*(circle_centers + 1.0) * canvas_size circle_radiuses = circle_radiuses * canvas_size / 2 eps = 1e-4 curve_points = curve_points + eps*torch.randn_like(curve_points) bs, num_strokes, num_pts, _ = curve_points.shape num_segments = (num_pts - 1) // 3 num_circles = circle_centers.shape[1] n_out = 3 if colors is not None else 1 output = torch.zeros(bs, n_out, canvas_size, canvas_size, device=curve_points.device) scenes = [] for k in range(bs): shapes = [] shape_groups = [] for p in range(num_strokes): points = curve_points[k, p].contiguous().cuda() # bezier num_ctrl_pts = torch.zeros(num_segments, dtype=torch.int32) + 2 width = curve_widths[k, p].cuda() alpha = curve_alphas[k, p].cuda() if colors is not None: color = colors[k, p] else: color = torch.ones(3, device=alpha.device) color = torch.cat([color, alpha.view(1,)]) path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) for c in range(num_circles): center = circle_centers[k, c] radius = circle_radiuses[k, c] width = circle_widths[k, c] alpha = circle_alphas[k, c] circle = pydiffvg.Circle(radius=radius, center=center, stroke_width=width) shapes.append(circle) if colors is not None: color = colors[k, p] else: color = torch.ones(3, device=alpha.device) color = torch.cat([color, alpha.view(1,)]) circle_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(shapes) - 1]), fill_color=torch.tensor([0, 0, 0, 0.0]), stroke_color=color, ) shape_groups.append(circle_group) # Rasterize scenes.append((canvas_size, canvas_size, shapes, shape_groups)) raster = render(canvas_size, canvas_size, shapes, shape_groups, samples=2) raster = raster.permute(2, 0, 1).view(4, canvas_size, canvas_size) alpha = raster[3:4] if colors is not None: # color output image = raster[:3] alpha = alpha.repeat(3, 1, 1) else: image = raster[:1] # alpha compositing image = image*alpha output[k] = image output = output.to(dev) return output, scenes
# [150.0, 60.0], # control point # [ 90.0, 198.0], # control point # [ 60.0, 218.0], # base # [ 90.0, 180.0], # control point # [200.0, 65.0], # control point # [210.0, 98.0], # base # [220.0, 70.0], # control point # [130.0, 55.0]]) # control point points = torch.tensor([ [20.0, 128.0], # base [50.0, 128.0], # control point [170.0, 128.0], # control point [200.0, 128.0] ]) # base path = pydiffvg.Path(num_control_points=num_control_points, points=points, is_closed=False, stroke_width=torch.tensor(10.0)) shapes = [path] path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([0]), fill_color=None, stroke_color=torch.tensor( [0.3, 0.6, 0.3, 1.0])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups, output_type = pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height
def opacityStroke2diffvg(strokes, canvas_size=128, debug=False, relative=True, force_cpu=True): dev = strokes.device if force_cpu: strokes = strokes.to("cpu") # pydiffvg.set_use_gpu(False) # if strokes.is_cuda: # pydiffvg.set_use_gpu(True) """Rasterize strokes given in (dx, dy, opacity) sequence format.""" bs, nsegs, dims = strokes.shape out = [] start = time.time() for batch_idx, stroke in enumerate(strokes): if relative: # Absolute coordinates all_points = stroke[..., :2].cumsum(0) else: all_points = stroke[..., :2] all_opacities = stroke[..., 2] # Transform from [-1, 1] to canvas coordinates # Make sure points are in canvas all_points = 0.5 * (all_points + 1.0) * canvas_size # all_points = th.clamp(0.5*(all_points + 1.0), 0, 1) * canvas_size # Avoid overlapping points eps = 1e-4 all_points = all_points + eps * th.randn_like(all_points) shapes = [] shape_groups = [] for start_idx in range(0, nsegs - 1): points = all_points[start_idx:start_idx + 2].contiguous().float() opacity = all_opacities[start_idx] num_ctrl_pts = th.zeros(points.shape[0] - 1, dtype=th.int32) width = th.ones(1) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) color = th.cat( [th.ones(3, device=opacity.device), opacity.unsqueeze(0)], 0) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize only if there are shapes if shapes: inner_start = time.time() out.append( render(canvas_size, canvas_size, shapes, shape_groups, samples=4)) if debug: inner_elapsed = time.time() - inner_start print("diffvg call took %.2fms" % inner_elapsed) else: out.append( th.zeros(canvas_size, canvas_size, 4, device=strokes.device)) if debug: elapsed = (time.time() - start) * 1000 print("rendering took %.2fms" % elapsed) images = th.stack(out, 0).permute(0, 3, 1, 2).contiguous() # Return data on the same device as input return images.to(dev)
def stroke2diffvg(strokes, canvas_size=128): """Rasterize strokes given some sequential data.""" bs, nsegs, dims = strokes.shape out = [] for stroke_idx, stroke in enumerate(strokes): end_of_stroke = stroke[:, 4] == 1 last = end_of_stroke.cpu().numpy().argmax() stroke = stroke[:last + 1, :] # stroke = stroke[~end_of_stroke] # TODO: stop at the first end of stroke # import ipdb; ipdb.set_trace() split_idx = stroke[:, 3].nonzero().squeeze(1) # Absolute coordinates all_points = stroke[..., :2].cumsum(0) # Transform to canvas coordinates all_points[..., 0] += 0.5 all_points[..., 0] *= canvas_size all_points[..., 1] += 0.5 all_points[..., 1] *= canvas_size # Make sure points are in canvas all_points[..., :2] = th.clamp(all_points[..., :2], 0, canvas_size) shape_groups = [] shapes = [] start_idx = 0 for count, end_idx in enumerate(split_idx): points = all_points[start_idx:end_idx + 1].contiguous().float() if points.shape[0] <= 2: # we need at least 2 points for a line continue num_ctrl_pts = th.zeros(points.shape[0] - 1, dtype=th.int32) width = th.ones(1) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) start_idx = end_idx + 1 shapes.append(path) color = th.ones(4, 1) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize if shapes: # draw only if there are shapes out.append( render(canvas_size, canvas_size, shapes, shape_groups, samples=2)) else: out.append( th.zeros(canvas_size, canvas_size, 4, device=strokes.device)) return th.stack(out, 0).permute(0, 3, 1, 2)[:, :3].contiguous()
def raster(all_points, color=[0, 0, 0, 1], verbose=False, white_background=True): assert len(color) == 4 # print('1:', process.memory_info().rss*1e-6) render_size = 512 paths = int(all_points.shape[0] / 3) bs = 1 #all_points.shape[0] outputs = [] scaling = torch.zeros([1, 2]) scaling[:, 0] = 512 / 24 scaling[:, 1] = 512 / 24 print(scaling) all_points = all_points * scaling num_ctrl_pts = torch.zeros(paths, dtype=torch.int32) + 2 color = make_tensor(color) for k in range(bs): # Get point parameters from network shapes = [] shape_groups = [] points = all_points.cpu().contiguous() # [self.sort_idx[k]] if verbose: np.random.seed(0) colors = np.random.rand(paths, 4) high = np.array((0.565, 0.392, 0.173, 1)) low = np.array((0.094, 0.310, 0.635, 1)) diff = (high - low) / (paths) colors[:, 3] = 1 for i in range(paths): scale = diff * i color = low + scale color[3] = 1 color = torch.tensor(color) num_ctrl_pts = torch.zeros(1, dtype=torch.int32) + 2 if i * 3 + 4 > paths * 3: curve_points = torch.stack([ points[i * 3], points[i * 3 + 1], points[i * 3 + 2], points[0] ]) else: curve_points = points[i * 3:i * 3 + 4] path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=curve_points, is_closed=False, stroke_width=torch.tensor(4)) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([i]), fill_color=None, stroke_color=color) shapes.append(path) shape_groups.append(path_group) for i in range(paths * 3): scale = diff * (i // 3) color = low + scale color[3] = 1 color = torch.tensor(color) if i % 3 == 0: # color = torch.tensor(colors[i//3]) #green shape = pydiffvg.Rect(p_min=points[i] - 8, p_max=points[i] + 8) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [paths + i]), fill_color=color) else: # color = torch.tensor(colors[i//3]) #purple shape = pydiffvg.Circle(radius=torch.tensor(8.0), center=points[i]) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [paths + i]), fill_color=color) shapes.append(shape) shape_groups.append(group) else: path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=color, stroke_color=color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene( render_size, render_size, shapes, shape_groups) out = render( render_size, # width render_size, # height 2, # num_samples_x 2, # num_samples_y 102, # seed None, *scene_args) out = out.permute(2, 0, 1).view(4, render_size, render_size) # [:3]#.mean(0, keepdim=True) outputs.append(out) output = torch.stack(outputs).to(all_points.device) alpha = output[:, 3:4, :, :] # map to [-1, 1] if white_background: output_white_bg = output[:, :3, :, :] * alpha + (1 - alpha) output = torch.cat([output_white_bg, alpha], dim=1) del num_ctrl_pts, color return output