def test_sample_bilinear(): h, w = 2, 3 img = np.float32(np.arange(6).reshape(h, w, 1)) img = img[::-1] # flip y to match OpenGL tests = [[0, 0, 0], [1, 1, 4], [0.5, 0.5, 2], [0.5, 0.0, 0.5], [0.0, 0.5, 1.5], [-1.0, -1.0, 5.0], [w, 1, 3.0], [w - 0.5, h - 0.5, 2.5], [2 * w - 0.5, 2 * h - 0.5, 2.5]] tests = np.float32(tests) uv = np.float32((tests[:, :2] + 0.5) / [w, h]) # normalize UVs expected = tests[:, 2:] with tf.Session() as sess: output = spatial.sample_bilinear(img, uv).eval() assert np.abs(output - expected).max() < 1e-8
def textureStyle3D(self): print("Importing tensorflow...") import tensorflow as tf print("Checking that GPU is visible for tensorflow...") if not tf.test.is_gpu_available(): raise Exception("No GPU available for tensorflow!") print("Importing other libraries...") import os import io import sys from string import Template from pathlib import Path import numpy as np import PIL.Image # import matplotlib.pylab as pl from IPython.display import clear_output, display, Image, HTML # if os.name != 'nt': # from lucid.misc.gl.glcontext import create_opengl_context import OpenGL.GL as gl from lucid.misc.gl import meshutil from lucid.misc.gl import glrenderer import lucid.misc.io.showing as show import lucid.misc.io as lucid_io from lucid.misc.tfutil import create_session from lucid.modelzoo import vision_models from lucid.optvis import objectives from lucid.optvis import param from lucid.optvis.style import StyleLoss, mean_l1_loss from lucid.optvis.param.spatial import sample_bilinear # if os.name != 'nt': # print("Creating OpenGL context...") # create_opengl_context() gl.glGetString(gl.GL_VERSION) print("Loading vision model...") model = vision_models.InceptionV1() model.load_graphdef() def prepare_image(fn, size=None): data = lucid_io.reading.read(fn) im = PIL.Image.open(io.BytesIO(data)).convert('RGB') if size: im = im.resize(size, PIL.Image.ANTIALIAS) return np.float32(im) / 255.0 self.loadCameras() print("Loading input model from '{}'...".format(self.input_model_path)) mesh = meshutil.load_obj(self.input_model_path) if self.cameras is None: mesh = meshutil.normalize_mesh(mesh) print("Loading input texture from '{}'...".format( self.input_texture_path)) original_texture = prepare_image( self.input_texture_path, (self.texture_size, self.texture_size)) print("Loading style from '{}'...".format(self.style_path)) style = prepare_image(self.style_path) rendering_width = self.rendering_width rendering_height = int(rendering_width // self.aspect_ratio) print("Creating renderer with resolution {}x{}...".format( rendering_width, rendering_height)) renderer = glrenderer.MeshRenderer((rendering_width, rendering_height)) if self.cameras is not None: print(" renderer fovy: {:.2f} degrees".format(self.max_fovy)) renderer.fovy = self.max_fovy sess = create_session(timeout_sec=0) # t_fragments is used to feed rasterized UV coordinates for the current view. # Channels: [U, V, _, Alpha]. Alpha is 1 for pixels covered by the object, and # 0 for background. t_fragments = tf.placeholder(tf.float32, [None, None, 4]) t_uv = t_fragments[..., :2] t_alpha = t_fragments[..., 3:] # Texture atlas to optimize t_texture = param.image(self.texture_size, fft=True, decorrelate=True)[0] # Variable to store the original mesh texture used to render content views content_var = tf.Variable(tf.zeros( [self.texture_size, self.texture_size, 3]), trainable=False) # Sample current and original textures with provided pixel data t_joined_texture = tf.concat([t_texture, content_var], -1) t_joined_frame = sample_bilinear(t_joined_texture, t_uv) * t_alpha t_frame_current, t_frame_content = t_joined_frame[ ..., :3], t_joined_frame[..., 3:] t_joined_frame = tf.stack([t_frame_current, t_frame_content], 0) # Feeding the rendered frames to the Neural Network t_input = tf.placeholder_with_default(t_joined_frame, [None, None, None, 3]) model.import_graph(t_input) # style loss style_layers = [ sess.graph.get_tensor_by_name('import/%s:0' % s)[0] for s in self.googlenet_style_layers ] # L1-loss seems to be more stable for GoogleNet # Note that we use style_decay>0 to average style-describing Gram matrices # over the recent viewports. Please refer to StyleLoss for the details. sl = StyleLoss(style_layers, self.style_decay, loss_func=mean_l1_loss) # content loss content_layer = sess.graph.get_tensor_by_name( 'import/%s:0' % self.googlenet_content_layer) content_loss = mean_l1_loss(content_layer[0], content_layer[1]) * self.content_weight # setup optimization total_loss = content_loss + sl.style_loss t_lr = tf.constant(0.05) trainer = tf.train.AdamOptimizer(t_lr) train_op = trainer.minimize(total_loss) init_op = tf.global_variables_initializer() loss_log = [] def reset(style_img, content_texture): del loss_log[:] init_op.run() sl.set_style({t_input: style_img[None, ...]}) content_var.load(content_texture) def sample_random_view(): if self.cameras is None: return meshutil.sample_view(10.0, 12.0) else: rand_m = self.cameras[np.random.randint(0, len( self.cameras))]["transformToCamera"].copy() return rand_m def run(mesh, step_n=400): app = QtWidgets.QApplication.instance() for i in range(step_n): fragments = renderer.render_mesh( modelview=sample_random_view(), position=mesh['position'], uv=mesh['uv'], face=mesh['face']) _, loss = sess.run([train_op, [content_loss, sl.style_loss]], {t_fragments: fragments}) loss_log.append(loss) if i == 0 or (i + 1) % 50 == 0: # clear_output() last_frame, last_content = sess.run( [t_frame_current, t_frame_content], {t_fragments: fragments}) # show.images([last_frame, last_content], ['current frame', 'content']) if i == 0 or (i + 1) % 10 == 0: print(len(loss_log), loss) pass # Show progress self.pBar.setValue( (i + step_n // 10 + 1) / (step_n + step_n // 10) * 100) app.processEvents() reset(style, original_texture) print("Running {} iterations...".format(self.steps_number)) run(mesh, step_n=self.steps_number) print("Finished!") texture = t_texture.eval() print("Exporting result texture to '{}'...".format( self.output_texture_path)) lucid_io.save(texture, self.output_texture_path, quality=90) sess.close() print("Importing result model to Metashape '{}'...".format( self.result_model_path)) chunk.model = None chunk.importModel(self.result_model_path) chunk.model.label = self.style_name Metashape.app.messageBox( "Everything worked fine!\n" "Please save project and RESTART Metashape!\n" "Because video memory was not released by TensorFlow!")
def render_frame(frame_): textures_ = textures # Add noise to textures textures_ = textures_ * tf.random_uniform( [3], texture_channel_multiplicative_noise_[0], texture_channel_multiplicative_noise_[1]) textures_ = textures_ + tf.random_uniform( [3], texture_channel_additive_noise_[0], texture_channel_additive_noise_[1]) textures_ = textures_ * tf.random_uniform( [], texture_pixel_multiplicative_noise_[0], texture_pixel_multiplicative_noise_[1]) textures_ = textures_ + tf.random_uniform( [], texture_pixel_additive_noise_[0], texture_pixel_additive_noise_[1]) textures_ = textures_ + tf.random_normal( textures_.shape, stddev=tf.random_uniform([], texture_gaussian_noise_stddev_[0], texture_gaussian_noise_stddev_[1])) #textures_ = tf.clip_by_value(textures_, 0.0, 1.0) # Render uvf_ = frame_[..., :3] image_ = sample_bilinear(textures_, uvf_) # Composite onto background # FIXME: This only really works with batch_size=1 alpha_ = frame_[..., 3:] image_ = image_ * alpha_ + backgrounds_[0] * (1 - alpha_) # Blur image image_ = image_[tf.newaxis, :, :, :] image_ = tf.nn.depthwise_conv2d(image_, image_gaussian_blur_kernel_, strides=[1, 1, 1, 1], padding='SAME') image_ = image_[0] # Blur alpha alpha_ = alpha_[tf.newaxis, :, :, :] alpha_ = tf.nn.depthwise_conv2d( alpha_, image_gaussian_blur_kernel_[:, :, :1, :], strides=[1, 1, 1, 1], padding='SAME') alpha_ = alpha_[0] # Recomposite blurred image onto background # FIXME: This only really works with batch_size=1 image_ = image_ * alpha_ + backgrounds_[0] * (1 - alpha_) # Add noise to image image_ = image_ * tf.random_uniform( [3], image_channel_multiplicative_noise_[0], image_channel_multiplicative_noise_[1]) image_ = image_ + tf.random_uniform([3], image_channel_additive_noise_[0], image_channel_additive_noise_[1]) image_ = image_ * tf.random_uniform( [], image_pixel_multiplicative_noise_[0], image_pixel_multiplicative_noise_[1]) image_ = image_ + tf.random_uniform([], image_pixel_additive_noise_[0], image_pixel_additive_noise_[1]) image_ = image_ + tf.random_normal( tf.shape(image_), stddev=tf.random_uniform([], image_gaussian_noise_stddev_[0], image_gaussian_noise_stddev_[1])) #image_ = tf.clip_by_value(image_, 0.0, 1.0) return image_
def index(): # return 'Hello World!' model = vision_models.InceptionV1() model.load_graphdef() TEXTURE_SIZE = 1024 mesh = meshutil.load_obj('article_models/bunny.obj') mesh = meshutil.normalize_mesh(mesh) original_texture = prepare_image('article_models/bunny.png', (TEXTURE_SIZE, TEXTURE_SIZE)) style_url = 'https://upload.wikimedia.org/wikipedia/commons/d/db/RIAN_archive_409362_Literaturnaya_Gazeta_article_about_YuriGagarin%2C_first_man_in_space.jpg' style = prepare_image(style_url) renderer = glrenderer.MeshRenderer((512, 512)) googlenet_style_layers = [ 'conv2d2', 'mixed3a', 'mixed3b', 'mixed4a', 'mixed4b', 'mixed4c', ] googlenet_content_layer = 'mixed3b' content_weight = 100.0 # Style Gram matrix weighted average decay coefficient style_decay = 0.95 sess = create_session(timeout_sec=0) # t_fragments is used to feed rasterized UV coordinates for the current view. # Channels: [U, V, _, Alpha]. Alpha is 1 for pixels covered by the object, and # 0 for background. t_fragments = tf.placeholder(tf.float32, [None, None, 4]) t_uv = t_fragments[..., :2] t_alpha = t_fragments[..., 3:] # Texture atlas to optimize t_texture = param.image(TEXTURE_SIZE, fft=True, decorrelate=True)[0] # Variable to store the original mesh texture used to render content views content_var = tf.Variable(tf.zeros([TEXTURE_SIZE, TEXTURE_SIZE, 3]), trainable=False) # Sample current and original textures with provided pixel data t_joined_texture = tf.concat([t_texture, content_var], -1) t_joined_frame = sample_bilinear(t_joined_texture, t_uv) * t_alpha t_frame_current, t_frame_content = t_joined_frame[..., :3], t_joined_frame[ ..., 3:] t_joined_frame = tf.stack([t_frame_current, t_frame_content], 0) # Feeding the rendered frames to the Neural Network t_input = tf.placeholder_with_default(t_joined_frame, [None, None, None, 3]) model.import_graph(t_input) # style loss style_layers = [ sess.graph.get_tensor_by_name('import/%s:0' % s)[0] for s in googlenet_style_layers ] # L1-loss seems to be more stable for GoogleNet # Note that we use style_decay>0 to average style-describing Gram matrices # over the recent viewports. Please refer to StyleLoss for the details. sl = StyleLoss(style_layers, style_decay, loss_func=mean_l1_loss) # content loss content_layer = sess.graph.get_tensor_by_name('import/%s:0' % googlenet_content_layer) content_loss = mean_l1_loss(content_layer[0], content_layer[1]) * content_weight # setup optimization total_loss = content_loss + sl.style_loss t_lr = tf.constant(0.05) trainer = tf.train.AdamOptimizer(t_lr) train_op = trainer.minimize(total_loss) init_op = tf.global_variables_initializer() loss_log = [] reset(style, original_texture) run(mesh) texture = t_texture.eval() return show.textured_mesh(mesh, texture)
# t_fragments is used to feed rasterized UV coordinates for the current view. # Channels: [U, V, _, Alpha]. Alpha is 1 for pixels covered by the object, and # 0 for background. t_fragments = tf.placeholder(tf.float32, [None, None, 4]) t_uv = t_fragments[...,:2] t_alpha = t_fragments[...,3:] # Texture atlas to optimize t_texture = param.image(TEXTURE_SIZE, fft=True, decorrelate=True)[0] # Variable to store the original mesh texture used to render content views content_var = tf.Variable(tf.zeros([TEXTURE_SIZE, TEXTURE_SIZE, 3]), trainable=False) # Sample current and original textures with provided pixel data t_joined_texture = tf.concat([t_texture, content_var], -1) t_joined_frame = sample_bilinear(t_joined_texture, t_uv) * t_alpha t_frame_current, t_frame_content = t_joined_frame[...,:3], t_joined_frame[...,3:] t_joined_frame = tf.stack([t_frame_current, t_frame_content], 0) # Feeding the rendered frames to the Neural Network t_input = tf.placeholder_with_default(t_joined_frame, [None, None, None, 3]) model.import_graph(t_input) # style loss style_layers = [sess.graph.get_tensor_by_name('import/%s:0'%s)[0] for s in googlenet_style_layers] # L1-loss seems to be more stable for GoogleNet # Note that we use style_decay>0 to average style-describing Gram matrices # over the recent viewports. Please refer to StyleLoss for the details. sl = StyleLoss(style_layers, style_decay, loss_func=mean_l1_loss) # content loss