def save(self, x, save_dir, name): z, log_var = self.encode(x) all_points = self.decode(z) # print(all_points.std(dim=1)) # all_points = ((all_points-0.5)*2 + 0.5)*self.imsize # if type(self.sort_idx) == type(None): # angles = torch.atan(all_points[:,:,1]/all_points[:,:,0]).detach() # self.sort_idx = torch.argsort(angles, dim=1) # Process the batch sequentially outputs = [] for k in range(1): # Get point parameters from network shapes = [] shape_groups = [] points = all_points[k].cpu()#[self.sort_idx[k]] color = torch.cat([torch.tensor([0,0,0,1]),]) num_ctrl_pts = torch.zeros(self.curves, dtype=torch.int32) + 2 path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=color, stroke_color=color) shape_groups.append(path_group) pydiffvg.save_svg(f"{save_dir}{name}/{name}.svg", self.imsize, self.imsize, shapes, shape_groups)
def setup_parameters(*args, **kwargs): shapes = [] shape_groups = [] for i in range(self.num_paths): num_segments = random.randint(1, 3) path = path_helpers.build_random_path(num_segments) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(shapes) - 1]), fill_color=None, stroke_color=torch.tensor(self.stroke_color)) shape_groups.append(path_group) for i in range(self.n_forms): num_segments = 10 path = path_helpers.build_random_path(num_segments) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(shapes) - 1]), fill_color=torch.tensor(self.stroke_color), stroke_color=torch.tensor(self.stroke_color)) shape_groups.append(path_group) return shapes, shape_groups
def font_render(all_points, all_widths, force_cpu=True, canvas_size=32): dev = all_points.device if force_cpu: all_points = all_points.to("cpu") all_widths = all_widths.to("cpu") all_points = 0.5 * (all_points + 1.0) * canvas_size eps = 1e-4 all_points = all_points + eps * th.randn_like(all_points) bs, num_strokes, num_pts, _ = all_points.shape num_segments = (num_pts - 1) // 3 n_out = 1 output = th.zeros(bs, n_out, canvas_size, canvas_size, device=all_points.device) scenes = [] for k in range(bs): shapes = [] shape_groups = [] for p in range(num_strokes): points = all_points[k, p].contiguous().cpu() # bezier num_ctrl_pts = th.zeros(num_segments, dtype=th.int32) + 2 width = all_widths[k, p].cpu() color = th.ones(4) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=color, stroke_color=None) shape_groups.append(path_group) # Rasterize scenes.append((canvas_size, canvas_size, shapes, shape_groups)) raster = render(canvas_size, canvas_size, shapes, shape_groups, samples=2) raster = raster.permute(2, 0, 1).view(4, canvas_size, canvas_size) image = raster[:1] output[k] = image output = output.to(dev) return output, scenes
def gen_func(self): canvas_width, canvas_height, shapes, shape_groups = pydiffvg.svg_to_scene(self.input_path) new_shapes = list() new_groups = list() for dx in range(0, self.multiplier_x): for dy in range(0, self.multiplier_y): for i in range(len(shapes)): new_shape = build_translated_path(shapes[i], dy * canvas_height, dx * canvas_width) new_shapes.append(new_shape) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(new_shapes) - 1]), fill_color=None, stroke_color=torch.tensor(self.stroke_color)) new_groups.append(path_group) def gen(): return new_shapes, new_groups return gen
def render(self, out): num_paths = self.num_paths img_size = self.A num_segments = self.num_segments imgs = [] for b in range(out.shape[0]): shapes = [] shape_groups = [] index = 0 for i in range(num_paths): points = img_size * out[b, index: index + 2 * (num_segments + 1) + num_segments].view(-1, 2) + img_size//2 index += 2 * (num_segments + 1) stroke_width = out[b, index] index += 1 num_control_points = torch.zeros(int(np.log2(num_segments)), dtype = torch.int32, device=self.device) + 2 path = pydiffvg.Path(num_control_points = num_control_points, points = points, is_closed = False, stroke_width = stroke_width) shapes.append(path) stroke_color = out[b, index].view(1) index += 1 stroke_color = torch.cat([stroke_color, stroke_color, stroke_color, torch.tensor([1.0]).to(self.device)]) path_group = pydiffvg.ShapeGroup(shape_ids = torch.tensor([len(shapes) - 1]), fill_color = None, stroke_color = stroke_color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(img_size, img_size, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(img_size, # width img_size, # height 2, # num_samples_x 2, # num_samples_y random.randint(0, 1048576), # seed None, *scene_args) imgs.append(img[:,:,:3]) img = torch.stack(imgs, dim = 0) # channel last to channel first img = img.permute(0,3,1,2) return img.reshape(out.shape[0], -1)
def forward(self, z): out = self.fc(z) # construct paths imgs = [] for b in range(out.shape[0]): index = 0 shapes = [] shape_groups = [] for i in range(num_paths): points = img_size * out[b, index: index + 2 * (num_segments + 1)].view(-1, 2).cpu() index += 2 * (num_segments + 1) stroke_width = img_size * out[b, index].view(1).cpu() index += 1 num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=stroke_width, is_closed=False) shapes.append(path) stroke_color = out[b, index].view(1).cpu() index += 1 stroke_color = torch.cat([stroke_color, torch.tensor([0.0, 0.0, 1.0])]) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(shapes) - 1]), fill_color=None, stroke_color=stroke_color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(img_size, img_size, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(img_size, # width img_size, # height 2, # num_samples_x 2, # num_samples_y random.randint(0, 1048576), # seed None, *scene_args) img = img[:, :, :1] # HWC -> NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW imgs.append(img) img = torch.cat(imgs, dim=0) return img
def opacityStroke2diffvg(strokes, canvas_size=128, debug=False, relative=True, force_cpu=True): dev = strokes.device if force_cpu: strokes = strokes.to("cpu") # pydiffvg.set_use_gpu(False) # if strokes.is_cuda: # pydiffvg.set_use_gpu(True) """Rasterize strokes given in (dx, dy, opacity) sequence format.""" bs, nsegs, dims = strokes.shape out = [] start = time.time() for batch_idx, stroke in enumerate(strokes): if relative: # Absolute coordinates all_points = stroke[..., :2].cumsum(0) else: all_points = stroke[..., :2] all_opacities = stroke[..., 2] # Transform from [-1, 1] to canvas coordinates # Make sure points are in canvas all_points = 0.5 * (all_points + 1.0) * canvas_size # all_points = th.clamp(0.5*(all_points + 1.0), 0, 1) * canvas_size # Avoid overlapping points eps = 1e-4 all_points = all_points + eps * th.randn_like(all_points) shapes = [] shape_groups = [] for start_idx in range(0, nsegs - 1): points = all_points[start_idx:start_idx + 2].contiguous().float() opacity = all_opacities[start_idx] num_ctrl_pts = th.zeros(points.shape[0] - 1, dtype=th.int32) width = th.ones(1) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) color = th.cat( [th.ones(3, device=opacity.device), opacity.unsqueeze(0)], 0) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize only if there are shapes if shapes: inner_start = time.time() out.append( render(canvas_size, canvas_size, shapes, shape_groups, samples=4)) if debug: inner_elapsed = time.time() - inner_start print("diffvg call took %.2fms" % inner_elapsed) else: out.append( th.zeros(canvas_size, canvas_size, 4, device=strokes.device)) if debug: elapsed = (time.time() - start) * 1000 print("rendering took %.2fms" % elapsed) images = th.stack(out, 0).permute(0, 3, 1, 2).contiguous() # Return data on the same device as input return images.to(dev)
import pydiffvg import torch # import skimage # import numpy as np # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width = 256 canvas_height = 256 circle = pydiffvg.Circle(radius=torch.tensor(40.0), center=torch.tensor([128.0, 128.0])) shapes = [circle] circle_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([0]), fill_color=torch.tensor( [0.3, 0.6, 0.3, 1.0])) shape_groups = [circle_group] scene_args = pydiffvg.RenderFunction.serialize_scene( canvas_width, canvas_height, shapes, shape_groups, output_type=pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed
# [210.0, 98.0], # base # [220.0, 70.0], # control point # [130.0, 55.0]]) # control point points = torch.tensor([ [20.0, 128.0], # base [50.0, 128.0], # control point [170.0, 128.0], # control point [200.0, 128.0] ]) # base path = pydiffvg.Path(num_control_points=num_control_points, points=points, is_closed=False, stroke_width=torch.tensor(10.0)) shapes = [path] path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([0]), fill_color=None, stroke_color=torch.tensor( [0.3, 0.6, 0.3, 1.0])) shape_groups = [path_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups, output_type = pydiffvg.OutputType.sdf) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 1, # num_samples_x 1, # num_samples_y 0, # seed None, # background_image *scene_args)
import pydiffvg import torch import skimage import numpy as np # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 256, 256 ellipse = pydiffvg.Ellipse(radius=torch.tensor([60.0, 30.0]), center=torch.tensor([128.0, 128.0])) shapes = [ellipse] ellipse_group = pydiffvg.ShapeGroup(\ shape_ids = torch.tensor([0]), fill_color = torch.tensor([0.3, 0.6, 0.3, 1.0]), shape_to_canvas = torch.eye(3, 3)) shape_groups = [ellipse_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( 256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_ellipse_transform/target.png',
def save(self, all_points, save_dir, name, verbose=False, white_background=True): # note that this if for a single shape and bs dimension should have multiple curves # print('1:', process.memory_info().rss*1e-6) render_size = self.imsize bs = all_points.shape[0] if verbose: render_size = render_size * 2 all_points = all_points * render_size num_ctrl_pts = torch.zeros(self.curves, dtype=torch.int32) + 2 shapes = [] shape_groups = [] for k in range(bs): # Get point parameters from network color = make_tensor(color[k]) points = all_points[k].cpu().contiguous() #[self.sort_idx[k]] if verbose: np.random.seed(0) colors = np.random.rand(self.curves, 4) high = np.array((0.565, 0.392, 0.173, 1)) low = np.array((0.094, 0.310, 0.635, 1)) diff = (high - low) / (self.curves) colors[:, 3] = 1 for i in range(self.curves): scale = diff * i color = low + scale color[3] = 1 color = torch.tensor(color) num_ctrl_pts = torch.zeros(1, dtype=torch.int32) + 2 if i * 3 + 4 > self.curves * 3: curve_points = torch.stack([ points[i * 3], points[i * 3 + 1], points[i * 3 + 2], points[0] ]) else: curve_points = points[i * 3:i * 3 + 4] path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=curve_points, is_closed=False, stroke_width=torch.tensor(4)) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [i]), fill_color=None, stroke_color=color) shapes.append(path) shape_groups.append(path_group) for i in range(self.curves * 3): scale = diff * (i // 3) color = low + scale color[3] = 1 color = torch.tensor(color) if i % 3 == 0: # color = torch.tensor(colors[i//3]) #green shape = pydiffvg.Rect(p_min=points[i] - 8, p_max=points[i] + 8) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [self.curves + i]), fill_color=color) else: # color = torch.tensor(colors[i//3]) #purple shape = pydiffvg.Circle(radius=torch.tensor(8.0), center=points[i]) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [self.curves + i]), fill_color=color) shapes.append(shape) shape_groups.append(group) else: path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=color, stroke_color=color) shape_groups.append(path_group) pydiffvg.save_svg(f"{save_dir}{name}/{name}.svg", self.imsize, self.imsize, shapes, shape_groups)
def raster(all_points, color=[0, 0, 0, 1], verbose=False, white_background=True): assert len(color) == 4 # print('1:', process.memory_info().rss*1e-6) render_size = 512 paths = int(all_points.shape[0] / 3) bs = 1 #all_points.shape[0] outputs = [] scaling = torch.zeros([1, 2]) scaling[:, 0] = 512 / 24 scaling[:, 1] = 512 / 24 print(scaling) all_points = all_points * scaling num_ctrl_pts = torch.zeros(paths, dtype=torch.int32) + 2 color = make_tensor(color) for k in range(bs): # Get point parameters from network shapes = [] shape_groups = [] points = all_points.cpu().contiguous() # [self.sort_idx[k]] if verbose: np.random.seed(0) colors = np.random.rand(paths, 4) high = np.array((0.565, 0.392, 0.173, 1)) low = np.array((0.094, 0.310, 0.635, 1)) diff = (high - low) / (paths) colors[:, 3] = 1 for i in range(paths): scale = diff * i color = low + scale color[3] = 1 color = torch.tensor(color) num_ctrl_pts = torch.zeros(1, dtype=torch.int32) + 2 if i * 3 + 4 > paths * 3: curve_points = torch.stack([ points[i * 3], points[i * 3 + 1], points[i * 3 + 2], points[0] ]) else: curve_points = points[i * 3:i * 3 + 4] path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=curve_points, is_closed=False, stroke_width=torch.tensor(4)) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([i]), fill_color=None, stroke_color=color) shapes.append(path) shape_groups.append(path_group) for i in range(paths * 3): scale = diff * (i // 3) color = low + scale color[3] = 1 color = torch.tensor(color) if i % 3 == 0: # color = torch.tensor(colors[i//3]) #green shape = pydiffvg.Rect(p_min=points[i] - 8, p_max=points[i] + 8) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [paths + i]), fill_color=color) else: # color = torch.tensor(colors[i//3]) #purple shape = pydiffvg.Circle(radius=torch.tensor(8.0), center=points[i]) group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [paths + i]), fill_color=color) shapes.append(shape) shape_groups.append(group) else: path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=color, stroke_color=color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene( render_size, render_size, shapes, shape_groups) out = render( render_size, # width render_size, # height 2, # num_samples_x 2, # num_samples_y 102, # seed None, *scene_args) out = out.permute(2, 0, 1).view(4, render_size, render_size) # [:3]#.mean(0, keepdim=True) outputs.append(out) output = torch.stack(outputs).to(all_points.device) alpha = output[:, 3:4, :, :] # map to [-1, 1] if white_background: output_white_bg = output[:, :3, :, :] * alpha + (1 - alpha) output = torch.cat([output_white_bg, alpha], dim=1) del num_ctrl_pts, color return output
import numpy as np # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 256, 256 color = pydiffvg.LinearGradient(\ begin = torch.tensor([50.0, 50.0]), end = torch.tensor([200.0, 200.0]), offsets = torch.tensor([0.0, 1.0]), stop_colors = torch.tensor([[0.2, 0.5, 0.7, 1.0], [0.7, 0.2, 0.5, 1.0]])) circle = pydiffvg.Circle(radius = torch.tensor(40.0), center = torch.tensor([128.0, 128.0])) shapes = [circle] circle_group = pydiffvg.ShapeGroup(shape_ids = torch.tensor([0]), fill_color = color) shape_groups = [circle_group] scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render(256, # width 256, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, # background_image *scene_args) # The output image is in linear RGB space. Do Gamma correction before saving the image. pydiffvg.imwrite(img.cpu(), 'results/single_gradient/target.png', gamma=2.2) target = img.clone()
def decode(self, z, label=None): bs = z.shape[0] feats = self._decode_features(z, label) if self.raster: out = self.raster_decoder(feats).view(bs, 1, self.imsize, self.imsize) return out, {} all_points = self.point_predictor(feats) all_points = all_points.view(bs, self.paths, -1, 2) all_points = all_points*(self.imsize//2-2) + self.imsize//2 if False: all_widths = th.ones(bs, self.paths) * 0.5 else: all_widths = self.width_predictor(feats) * 1.5 + .25 if False: all_alphas = th.ones(bs, self.paths) else: all_alphas = self.alpha_predictor(feats) # Process the batch sequentially outputs = [] scenes = [] for k in range(bs): # Get point parameters from network shapes = [] shape_groups = [] for p in range(self.paths): points = all_points[k, p].contiguous().cpu() width = all_widths[k, p].cpu() alpha = all_alphas[k, p].cpu() color = th.cat([th.ones(3), alpha.view(1,)]) num_ctrl_pts = th.zeros(self.segments, dtype=th.int32) + 2 path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=th.tensor([len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) scenes.append( [shapes, shape_groups, (self.imsize, self.imsize)]) # Rasterize out = render(self.imsize, self.imsize, shapes, shape_groups, samples=self.samples) # Torch format, discard alpha, make gray out = out.permute(2, 0, 1).view(4, self.imsize, self.imsize)[:3].mean(0, keepdim=True) outputs.append(out) output = th.stack(outputs).to(z.device) auxdata = { "points": all_points, "scenes": scenes, } # map to [-1, 1] output = output*2.0 - 1.0 return output, auxdata
def gen_sample(self, z, label=None): bs = z.shape[0] if self.conditional: if label is None: raise ValueError("GAN is conditional, please provide a label") # One-hot encoding of the image label label_onehot = _onehot(label) # get some embedding in_ = th.cat([z, label_onehot.float()], 1) else: in_ = z feats = self.trunk(in_) all_points = self.point_predictor(feats) all_points = all_points.view(bs, self.paths, -1, 2) if False: all_alphas = th.ones(bs, self.paths) else: all_alphas = self.alpha_predictor(feats) # stroke size between 0.5 and 3.5 px if False: all_widths = th.ones(bs, self.paths) * 1 else: all_widths = self.width_predictor(feats) all_widths = 1.5*all_widths + 0.5 all_points = all_points*(self.imsize//2) + self.imsize//2 # Process the batch sequentially outputs = [] for k in range(bs): # Get point parameters from network shapes = [] shape_groups = [] for p in range(self.paths): points = all_points[k, p].contiguous().cpu() # num_ctrl_pts = th.zeros(self.segments, dtype=th.int32)+0 num_ctrl_pts = th.zeros(self.segments, dtype=th.int32)+2 width = all_widths[k, p].cpu() alpha = all_alphas[k, p].cpu() color = th.cat([th.ones(3), alpha.view(1,)]) path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=th.tensor([len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize out = render(self.imsize, self.imsize, shapes, shape_groups, samples=self.samples) # Torch format, discard alpha, make gray out = out.permute(2, 0, 1).view(4, self.imsize, self.imsize)[:3].mean(0, keepdim=True) outputs.append(out) output = th.stack(outputs).to(z.device) aux_data = { "points": all_points, "raw_vector": output, } # output = self.postprocessor(output) # map to [-1, 1] output = output*2.0 - 1.0 return output, aux_data
def __init__(self, num_paths=100, n_path_segments=3, max_width=4.0, min_width=1.0): self.num_paths = 100 self.max_width = max_width self.min_width = min_width assert self.max_width > self.min_width self.n_path_segments = n_path_segments self.classification_model = models.inception_v3( pretrained=True, transform_input=False).cuda() self.classification_model.eval() self.canvas_width = 229 # sync with inception model inputs self.canvas_height = 229 self.frequency_plot = 100 self.shapes = list() self.shape_groups = list() # fill shapes and shape_groups with random strokes for i in range(num_paths): num_control_points = torch.zeros(self.n_path_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(self.n_path_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= self.canvas_width points[:, 1] *= self.canvas_height path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor( 0.5 * (self.max_width - self.min_width)), is_closed=False) self.shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(self.shapes) - 1]), fill_color=None, stroke_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) self.shape_groups.append(path_group) self.render = pydiffvg.RenderFunction.apply # initialize variables self.point_variables = list() self.widths_variables = list() self.color_variables = list() for path in self.shapes: path.points.requires_grad = True path.stroke_width.requires_grad = True self.point_variables.append(path.points) self.widths_variables.append(path.stroke_width) for group in self.shape_groups: group.stroke_color.requires_grad = True self.color_variables.append(group.stroke_color)
def main(args): # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) perception_loss = ttools.modules.LPIPS().to(pydiffvg.get_device()) #target = torch.from_numpy(skimage.io.imread('imgs/lena.png')).to(torch.float32) / 255.0 target = torch.from_numpy(skimage.io.imread(args.target)).to( torch.float32) / 255.0 target = target.pow(gamma) target = target.to(pydiffvg.get_device()) target = target.unsqueeze(0) target = target.permute(0, 3, 1, 2) # NHWC -> NCHW #target = torch.nn.functional.interpolate(target, size = [256, 256], mode = 'area') canvas_width, canvas_height = target.shape[3], target.shape[2] num_paths = args.num_paths max_width = args.max_width random.seed(1234) torch.manual_seed(1234) shapes = [] shape_groups = [] if args.use_blob: for i in range(num_paths): num_segments = random.randint(3, 5) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) if j < num_segments - 1: points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=True) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) shape_groups.append(path_group) else: for i in range(num_paths): num_segments = random.randint(1, 3) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 points = [] p0 = (random.random(), random.random()) points.append(p0) for j in range(num_segments): radius = 0.05 p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height #points = torch.rand(3 * num_segments + 1, 2) * min(canvas_width, canvas_height) path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=torch.tensor(1.0), is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=torch.tensor([ random.random(), random.random(), random.random(), random.random() ])) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/init.png', gamma=gamma) points_vars = [] stroke_width_vars = [] color_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) if not args.use_blob: for path in shapes: path.stroke_width.requires_grad = True stroke_width_vars.append(path.stroke_width) if args.use_blob: for group in shape_groups: group.fill_color.requires_grad = True color_vars.append(group.fill_color) else: for group in shape_groups: group.stroke_color.requires_grad = True color_vars.append(group.stroke_color) # Optimize points_optim = torch.optim.Adam(points_vars, lr=1.0) if len(stroke_width_vars) > 0: width_optim = torch.optim.Adam(stroke_width_vars, lr=0.1) color_optim = torch.optim.Adam(color_vars, lr=0.01) # Adam iterations. for t in range(args.num_iter): print('iteration:', t) points_optim.zero_grad() if len(stroke_width_vars) > 0: width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y t, # seed None, *scene_args) # Compose img with white background img = img[:, :, 3:4] * img[:, :, :3] + torch.ones( img.shape[0], img.shape[1], 3, device=pydiffvg.get_device()) * (1 - img[:, :, 3:4]) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/iter_{}.png'.format(t), gamma=gamma) img = img[:, :, :3] # Convert img from HWC to NCHW img = img.unsqueeze(0) img = img.permute(0, 3, 1, 2) # NHWC -> NCHW if args.use_lpips_loss: loss = perception_loss( img, target) + (img.mean() - target.mean()).pow(2) else: loss = (img - target).pow(2).mean() print('render loss:', loss.item()) # Backpropagate the gradients. loss.backward() # Take a gradient descent step. points_optim.step() if len(stroke_width_vars) > 0: width_optim.step() color_optim.step() if len(stroke_width_vars) > 0: for path in shapes: path.stroke_width.data.clamp_(1.0, max_width) if args.use_blob: for group in shape_groups: group.fill_color.data.clamp_(0.0, 1.0) else: for group in shape_groups: group.stroke_color.data.clamp_(0.0, 1.0) if t % 10 == 0 or t == args.num_iter - 1: pydiffvg.save_svg( 'results/painterly_rendering/iter_{}.svg'.format(t), canvas_width, canvas_height, shapes, shape_groups) # Render the final result. img = render( target.shape[1], # width target.shape[0], # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Save the intermediate render. pydiffvg.imwrite(img.cpu(), 'results/painterly_rendering/final.png'.format(t), gamma=gamma) # Convert the intermediate renderings to a video. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", "results/painterly_rendering/iter_%d.png", "-vb", "20M", "results/painterly_rendering/out.mp4" ])
def bezier_render(all_points, all_widths, all_alphas, force_cpu=True, canvas_size=32, colors=None): dev = all_points.device if force_cpu: all_points = all_points.to("cpu") all_widths = all_widths.to("cpu") all_alphas = all_alphas.to("cpu") if colors is not None: colors = colors.to("cpu") all_points = 0.5 * (all_points + 1.0) * canvas_size eps = 1e-4 all_points = all_points + eps * th.randn_like(all_points) bs, num_strokes, num_pts, _ = all_points.shape num_segments = (num_pts - 1) // 3 n_out = 3 if colors is not None else 1 output = th.zeros(bs, n_out, canvas_size, canvas_size, device=all_points.device) scenes = [] for k in range(bs): shapes = [] shape_groups = [] for p in range(num_strokes): points = all_points[k, p].contiguous().cpu() # bezier num_ctrl_pts = th.zeros(num_segments, dtype=th.int32) + 2 width = all_widths[k, p].cpu() alpha = all_alphas[k, p].cpu() if colors is not None: color = colors[k, p] else: color = th.ones(3, device=alpha.device) color = th.cat([color, alpha.view(1, )]) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize scenes.append((canvas_size, canvas_size, shapes, shape_groups)) raster = render(canvas_size, canvas_size, shapes, shape_groups, samples=2) raster = raster.permute(2, 0, 1).view(4, canvas_size, canvas_size) alpha = raster[3:4] if colors is not None: # color output image = raster[:3] alpha = alpha.repeat(3, 1, 1) else: image = raster[:1] # alpha compositing image = image * alpha output[k] = image output = output.to(dev) return output, scenes
def stroke2diffvg(strokes, canvas_size=128): """Rasterize strokes given some sequential data.""" bs, nsegs, dims = strokes.shape out = [] for stroke_idx, stroke in enumerate(strokes): end_of_stroke = stroke[:, 4] == 1 last = end_of_stroke.cpu().numpy().argmax() stroke = stroke[:last + 1, :] # stroke = stroke[~end_of_stroke] # TODO: stop at the first end of stroke # import ipdb; ipdb.set_trace() split_idx = stroke[:, 3].nonzero().squeeze(1) # Absolute coordinates all_points = stroke[..., :2].cumsum(0) # Transform to canvas coordinates all_points[..., 0] += 0.5 all_points[..., 0] *= canvas_size all_points[..., 1] += 0.5 all_points[..., 1] *= canvas_size # Make sure points are in canvas all_points[..., :2] = th.clamp(all_points[..., :2], 0, canvas_size) shape_groups = [] shapes = [] start_idx = 0 for count, end_idx in enumerate(split_idx): points = all_points[start_idx:end_idx + 1].contiguous().float() if points.shape[0] <= 2: # we need at least 2 points for a line continue num_ctrl_pts = th.zeros(points.shape[0] - 1, dtype=th.int32) width = th.ones(1) path = pydiffvg.Path(num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) start_idx = end_idx + 1 shapes.append(path) color = th.ones(4, 1) path_group = pydiffvg.ShapeGroup(shape_ids=th.tensor( [len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) # Rasterize if shapes: # draw only if there are shapes out.append( render(canvas_size, canvas_size, shapes, shape_groups, samples=2)) else: out.append( th.zeros(canvas_size, canvas_size, 4, device=strokes.device)) return th.stack(out, 0).permute(0, 3, 1, 2)[:, :3].contiguous()
def parse_shape(node, transform, fill_color, shapes, shape_groups, defs): tag = remove_namespaces(node.tag) new_transform, new_fill_color, stroke_color, stroke_width, use_even_odd_rule = \ parse_common_attrib(node, transform, fill_color, defs) if tag == 'path': d = node.attrib['d'] name = '' if 'id' in node.attrib: name = node.attrib['id'] force_closing = new_fill_color is not None paths = pydiffvg.from_svg_path(d, new_transform, force_closing) for idx, path in enumerate(paths): assert (path.points.shape[1] == 2) path.stroke_width = stroke_width path.source_id = name path.id = "{}-{}".format(name, idx) if len(paths) > 1 else name prev_shapes_size = len(shapes) shapes = shapes + paths shape_ids = torch.tensor(list(range(prev_shapes_size, len(shapes)))) shape_groups.append(pydiffvg.ShapeGroup(\ shape_ids = shape_ids, fill_color = new_fill_color, stroke_color = stroke_color, use_even_odd_rule = use_even_odd_rule, id = name)) elif tag == 'polygon': name = '' if 'id' in node.attrib: name = node.attrib['id'] force_closing = new_fill_color is not None pts = node.attrib['points'].strip() pts = pts.split(' ') # import ipdb; ipdb.set_trace() pts = [[float(y) for y in re.split(',| ', x)] for x in pts if x] pts = torch.tensor(pts, dtype=torch.float32).view(-1, 2) polygon = pydiffvg.Polygon(pts, force_closing) polygon.stroke_width = stroke_width shape_ids = torch.tensor([len(shapes)]) shapes.append(polygon) shape_groups.append(pydiffvg.ShapeGroup(\ shape_ids = shape_ids, fill_color = new_fill_color, stroke_color = stroke_color, use_even_odd_rule = use_even_odd_rule, shape_to_canvas = new_transform, id = name)) elif tag == 'line': x1 = float(node.attrib['x1']) y1 = float(node.attrib['y1']) x2 = float(node.attrib['x2']) y2 = float(node.attrib['y2']) p1 = torch.tensor([x1, y1]) p2 = torch.tensor([x2, y2]) points = torch.stack((p1, p2)) line = pydiffvg.Polygon(points, False) line.stroke_width = stroke_width shape_ids = torch.tensor([len(shapes)]) shapes.append(line) shape_groups.append(pydiffvg.ShapeGroup(\ shape_ids = shape_ids, fill_color = new_fill_color, stroke_color = stroke_color, use_even_odd_rule = use_even_odd_rule, shape_to_canvas = new_transform)) elif tag == 'circle': radius = float(node.attrib['r']) cx = float(node.attrib['cx']) cy = float(node.attrib['cy']) name = '' if 'id' in node.attrib: name = node.attrib['id'] center = torch.tensor([cx, cy]) circle = pydiffvg.Circle(radius=torch.tensor(radius), center=center) circle.stroke_width = stroke_width shape_ids = torch.tensor([len(shapes)]) shapes.append(circle) shape_groups.append(pydiffvg.ShapeGroup(\ shape_ids = shape_ids, fill_color = new_fill_color, stroke_color = stroke_color, use_even_odd_rule = use_even_odd_rule, shape_to_canvas = new_transform)) elif tag == 'ellipse': rx = float(node.attrib['rx']) ry = float(node.attrib['ry']) cx = float(node.attrib['cx']) cy = float(node.attrib['cy']) name = '' if 'id' in node.attrib: name = node.attrib['id'] center = torch.tensor([cx, cy]) circle = pydiffvg.Circle(radius=torch.tensor(radius), center=center) circle.stroke_width = stroke_width shape_ids = torch.tensor([len(shapes)]) shapes.append(circle) shape_groups.append(pydiffvg.ShapeGroup(\ shape_ids = shape_ids, fill_color = new_fill_color, stroke_color = stroke_color, use_even_odd_rule = use_even_odd_rule, shape_to_canvas = new_transform)) elif tag == 'rect': x = 0.0 y = 0.0 if x in node.attrib: x = float(node.attrib['x']) if y in node.attrib: y = float(node.attrib['y']) w = float(node.attrib['width']) h = float(node.attrib['height']) p_min = torch.tensor([x, y]) p_max = torch.tensor([x + w, x + h]) rect = pydiffvg.Rect(p_min=p_min, p_max=p_max) rect.stroke_width = stroke_width shape_ids = torch.tensor([len(shapes)]) shapes.append(rect) shape_groups.append(pydiffvg.ShapeGroup(\ shape_ids = shape_ids, fill_color = new_fill_color, stroke_color = stroke_color, use_even_odd_rule = use_even_odd_rule, shape_to_canvas = new_transform)) return shapes, shape_groups
def my_render(curve_points, curve_widths, curve_alphas, circle_centers, circle_radiuses, circle_widths, circle_alphas, canvas_size=32, colors=None): dev = curve_points.device curve_points = 0.5*(curve_points + 1.0) * canvas_size circle_centers = 0.5*(circle_centers + 1.0) * canvas_size circle_radiuses = circle_radiuses * canvas_size / 2 eps = 1e-4 curve_points = curve_points + eps*torch.randn_like(curve_points) bs, num_strokes, num_pts, _ = curve_points.shape num_segments = (num_pts - 1) // 3 num_circles = circle_centers.shape[1] n_out = 3 if colors is not None else 1 output = torch.zeros(bs, n_out, canvas_size, canvas_size, device=curve_points.device) scenes = [] for k in range(bs): shapes = [] shape_groups = [] for p in range(num_strokes): points = curve_points[k, p].contiguous().cuda() # bezier num_ctrl_pts = torch.zeros(num_segments, dtype=torch.int32) + 2 width = curve_widths[k, p].cuda() alpha = curve_alphas[k, p].cuda() if colors is not None: color = colors[k, p] else: color = torch.ones(3, device=alpha.device) color = torch.cat([color, alpha.view(1,)]) path = pydiffvg.Path( num_control_points=num_ctrl_pts, points=points, stroke_width=width, is_closed=False) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=None, stroke_color=color) shape_groups.append(path_group) for c in range(num_circles): center = circle_centers[k, c] radius = circle_radiuses[k, c] width = circle_widths[k, c] alpha = circle_alphas[k, c] circle = pydiffvg.Circle(radius=radius, center=center, stroke_width=width) shapes.append(circle) if colors is not None: color = colors[k, p] else: color = torch.ones(3, device=alpha.device) color = torch.cat([color, alpha.view(1,)]) circle_group = pydiffvg.ShapeGroup(shape_ids=torch.tensor([len(shapes) - 1]), fill_color=torch.tensor([0, 0, 0, 0.0]), stroke_color=color, ) shape_groups.append(circle_group) # Rasterize scenes.append((canvas_size, canvas_size, shapes, shape_groups)) raster = render(canvas_size, canvas_size, shapes, shape_groups, samples=2) raster = raster.permute(2, 0, 1).view(4, canvas_size, canvas_size) alpha = raster[3:4] if colors is not None: # color output image = raster[:3] alpha = alpha.repeat(3, 1, 1) else: image = raster[:1] # alpha compositing image = image*alpha output[k] = image output = output.to(dev) return output, scenes
def main(args): if args.seed: np.random.seed(args.seed) random.seed(args.seed) torch.manual_seed(args.seed) pydiffvg.set_print_timing(False) outdir = os.path.join(args.results_dir, args.prompt, args.subdir) # Use GPU if available pydiffvg.set_use_gpu(torch.cuda.is_available()) canvas_width, canvas_height = 224, 224 margin = args.initial_margin total_paths = args.open_paths + args.closed_paths step = min(args.step, total_paths) if step == 0: step = total_paths fill_color = None stroke_color = None shapes = [] shape_groups = [] losses = [] tt = 0 for num_paths in range(step, total_paths + 1, step): for i in range(num_paths - step, num_paths): num_segments = random.randint(1, args.extra_segments + 1) p0 = (margin + random.random() * (1 - 2 * margin), margin + random.random() * (1 - 2 * margin)) points = [p0] is_closed = i >= args.open_paths if is_closed: num_segments += 2 for j in range(num_segments): p1 = (p0[0] + radius * (random.random() - 0.5), p0[1] + radius * (random.random() - 0.5)) p2 = (p1[0] + radius * (random.random() - 0.5), p1[1] + radius * (random.random() - 0.5)) p3 = (p2[0] + radius * (random.random() - 0.5), p2[1] + radius * (random.random() - 0.5)) points.append(p1) points.append(p2) if is_closed and j < num_segments - 1 or not is_closed: points.append(p3) p0 = p3 points = torch.tensor(points) points[:, 0] *= canvas_width points[:, 1] *= canvas_height stroke_width = torch.tensor(1.0) color = torch.tensor([ random.random(), random.random(), random.random(), random.random() ]) num_control_points = torch.zeros(num_segments, dtype=torch.int32) + 2 path = pydiffvg.Path(num_control_points=num_control_points, points=points, stroke_width=stroke_width, is_closed=is_closed) shapes.append(path) path_group = pydiffvg.ShapeGroup( shape_ids=torch.tensor([len(shapes) - 1]), fill_color=color if is_closed else None, stroke_color=None if is_closed else color) shape_groups.append(path_group) scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) render = pydiffvg.RenderFunction.apply img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'init.png'), gamma=gamma) points_vars = [] stroke_width_vars = [] color_vars = [] for path in shapes: path.points.requires_grad = True points_vars.append(path.points) if not path.is_closed and args.max_width > 1: path.stroke_width.requires_grad = True stroke_width_vars.append(path.stroke_width) for group in shape_groups: if group.fill_color is not None: group.fill_color.requires_grad = True color_vars.append(group.fill_color) else: group.stroke_color.requires_grad = True color_vars.append(group.stroke_color) # Embed prompt text_features = clip_utils.embed_text(args.prompt) # Optimize points_optim = torch.optim.Adam(points_vars, lr=args.points_lr) if len(stroke_width_vars) > 0: width_optim = torch.optim.Adam(stroke_width_vars, lr=args.width_lr) color_optim = torch.optim.Adam(color_vars, lr=args.color_lr) # Adam iterations. final = False this_step_iters = max(1, round(args.num_iter * step / total_paths)) if num_paths + step > total_paths: final = True this_step_iters += args.extra_iter for t in range(this_step_iters): points_optim.zero_grad() if len(stroke_width_vars) > 0: width_optim.zero_grad() color_optim.zero_grad() # Forward pass: render the image. scene_args = pydiffvg.RenderFunction.serialize_scene(\ canvas_width, canvas_height, shapes, shape_groups) img = render( canvas_width, # width canvas_height, # height 2, # num_samples_x 2, # num_samples_y tt, # seed None, *scene_args) # Save the intermediate render. with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'iter_{}.png'.format(tt)), gamma=gamma) image_features = clip_utils.embed_image(img) loss = -torch.cosine_similarity( text_features, image_features, dim=-1).mean() # Backpropagate the gradients. loss.backward() losses.append(loss.item()) # Take a gradient descent step. points_optim.step() if len(stroke_width_vars) > 0: width_optim.step() color_optim.step() for path in shapes: path.points.data[:, 0].clamp_(0.0, canvas_width) path.points.data[:, 1].clamp_(0.0, canvas_height) if not path.is_closed: path.stroke_width.data.clamp_(1.0, args.max_width) for group in shape_groups: if group.fill_color is not None: group.fill_color.data[:3].clamp_(0.0, 1.0) group.fill_color.data[3].clamp_(args.min_alpha, 1.0) else: group.stroke_color.data[:3].clamp_(0.0, 1.0) group.stroke_color.data[3].clamp_(args.min_alpha, 1.0) if tt % 10 == 0 or final and t == this_step_iters - 1: print('%d loss=%.3f' % (tt, 1 + losses[-1])) pydiffvg.save_svg( os.path.join(outdir, 'iter_{}.svg'.format(tt)), canvas_width, canvas_height, shapes, shape_groups) clip_utils.plot_losses(losses, outdir) tt += 1 # Render the final result. img = render( args.final_px, # width args.final_px, # height 2, # num_samples_x 2, # num_samples_y 0, # seed None, *scene_args) # Save the intermediate render with warnings.catch_warnings(): warnings.simplefilter("ignore") pydiffvg.imwrite(img.cpu(), os.path.join(outdir, 'final.png'), gamma=gamma) # Convert the intermediate renderings to a video with a white background. from subprocess import call call([ "ffmpeg", "-framerate", "24", "-i", os.path.join(outdir, "iter_%d.png"), "-vb", "20M", "-filter_complex", "color=white,format=rgb24[c];[c][0]scale2ref[c][i];[c][i]overlay=format=auto:shortest=1,setsar=1", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-profile:v", "baseline", "-movflags", "+faststart", os.path.join(outdir, "out.mp4") ])