def test_sample_bilinear():
h, w = 2, 3
img = np.float32(np.arange(6).reshape(h, w, 1))
img = img[::-1] # flip y to match OpenGL
tests = [[0, 0, 0], [1, 1, 4], [0.5, 0.5, 2], [0.5, 0.0, 0.5],
[0.0, 0.5, 1.5], [-1.0, -1.0, 5.0], [w, 1, 3.0],
[w - 0.5, h - 0.5, 2.5], [2 * w - 0.5, 2 * h - 0.5, 2.5]]
tests = np.float32(tests)
uv = np.float32((tests[:, :2] + 0.5) / [w, h]) # normalize UVs
expected = tests[:, 2:]
with tf.Session() as sess:
output = spatial.sample_bilinear(img, uv).eval()
assert np.abs(output - expected).max() < 1e-8
def textureStyle3D(self):
print("Importing tensorflow...")
import tensorflow as tf
print("Checking that GPU is visible for tensorflow...")
if not tf.test.is_gpu_available():
raise Exception("No GPU available for tensorflow!")
print("Importing other libraries...")
import os
import io
import sys
from string import Template
from pathlib import Path
import numpy as np
import PIL.Image
# import matplotlib.pylab as pl
from IPython.display import clear_output, display, Image, HTML
# if os.name != 'nt':
# from lucid.misc.gl.glcontext import create_opengl_context
import OpenGL.GL as gl
from lucid.misc.gl import meshutil
from lucid.misc.gl import glrenderer
import lucid.misc.io.showing as show
import lucid.misc.io as lucid_io
from lucid.misc.tfutil import create_session
from lucid.modelzoo import vision_models
from lucid.optvis import objectives
from lucid.optvis import param
from lucid.optvis.style import StyleLoss, mean_l1_loss
from lucid.optvis.param.spatial import sample_bilinear
# if os.name != 'nt':
# print("Creating OpenGL context...")
# create_opengl_context()
gl.glGetString(gl.GL_VERSION)
print("Loading vision model...")
model = vision_models.InceptionV1()
model.load_graphdef()
def prepare_image(fn, size=None):
data = lucid_io.reading.read(fn)
im = PIL.Image.open(io.BytesIO(data)).convert('RGB')
if size:
im = im.resize(size, PIL.Image.ANTIALIAS)
return np.float32(im) / 255.0
self.loadCameras()
print("Loading input model from '{}'...".format(self.input_model_path))
mesh = meshutil.load_obj(self.input_model_path)
if self.cameras is None:
mesh = meshutil.normalize_mesh(mesh)
print("Loading input texture from '{}'...".format(
self.input_texture_path))
original_texture = prepare_image(
self.input_texture_path, (self.texture_size, self.texture_size))
print("Loading style from '{}'...".format(self.style_path))
style = prepare_image(self.style_path)
rendering_width = self.rendering_width
rendering_height = int(rendering_width // self.aspect_ratio)
print("Creating renderer with resolution {}x{}...".format(
rendering_width, rendering_height))
renderer = glrenderer.MeshRenderer((rendering_width, rendering_height))
if self.cameras is not None:
print(" renderer fovy: {:.2f} degrees".format(self.max_fovy))
renderer.fovy = self.max_fovy
sess = create_session(timeout_sec=0)
# t_fragments is used to feed rasterized UV coordinates for the current view.
# Channels: [U, V, _, Alpha]. Alpha is 1 for pixels covered by the object, and
# 0 for background.
t_fragments = tf.placeholder(tf.float32, [None, None, 4])
t_uv = t_fragments[..., :2]
t_alpha = t_fragments[..., 3:]
# Texture atlas to optimize
t_texture = param.image(self.texture_size, fft=True,
decorrelate=True)[0]
# Variable to store the original mesh texture used to render content views
content_var = tf.Variable(tf.zeros(
[self.texture_size, self.texture_size, 3]),
trainable=False)
# Sample current and original textures with provided pixel data
t_joined_texture = tf.concat([t_texture, content_var], -1)
t_joined_frame = sample_bilinear(t_joined_texture, t_uv) * t_alpha
t_frame_current, t_frame_content = t_joined_frame[
..., :3], t_joined_frame[..., 3:]
t_joined_frame = tf.stack([t_frame_current, t_frame_content], 0)
# Feeding the rendered frames to the Neural Network
t_input = tf.placeholder_with_default(t_joined_frame,
[None, None, None, 3])
model.import_graph(t_input)
# style loss
style_layers = [
sess.graph.get_tensor_by_name('import/%s:0' % s)[0]
for s in self.googlenet_style_layers
]
# L1-loss seems to be more stable for GoogleNet
# Note that we use style_decay>0 to average style-describing Gram matrices
# over the recent viewports. Please refer to StyleLoss for the details.
sl = StyleLoss(style_layers, self.style_decay, loss_func=mean_l1_loss)
# content loss
content_layer = sess.graph.get_tensor_by_name(
'import/%s:0' % self.googlenet_content_layer)
content_loss = mean_l1_loss(content_layer[0],
content_layer[1]) * self.content_weight
# setup optimization
total_loss = content_loss + sl.style_loss
t_lr = tf.constant(0.05)
trainer = tf.train.AdamOptimizer(t_lr)
train_op = trainer.minimize(total_loss)
init_op = tf.global_variables_initializer()
loss_log = []
def reset(style_img, content_texture):
del loss_log[:]
init_op.run()
sl.set_style({t_input: style_img[None, ...]})
content_var.load(content_texture)
def sample_random_view():
if self.cameras is None:
return meshutil.sample_view(10.0, 12.0)
else:
rand_m = self.cameras[np.random.randint(0, len(
self.cameras))]["transformToCamera"].copy()
return rand_m
def run(mesh, step_n=400):
app = QtWidgets.QApplication.instance()
for i in range(step_n):
fragments = renderer.render_mesh(
modelview=sample_random_view(),
position=mesh['position'],
uv=mesh['uv'],
face=mesh['face'])
_, loss = sess.run([train_op, [content_loss, sl.style_loss]],
{t_fragments: fragments})
loss_log.append(loss)
if i == 0 or (i + 1) % 50 == 0:
# clear_output()
last_frame, last_content = sess.run(
[t_frame_current, t_frame_content],
{t_fragments: fragments})
# show.images([last_frame, last_content], ['current frame', 'content'])
if i == 0 or (i + 1) % 10 == 0:
print(len(loss_log), loss)
pass
# Show progress
self.pBar.setValue(
(i + step_n // 10 + 1) / (step_n + step_n // 10) * 100)
app.processEvents()
reset(style, original_texture)
print("Running {} iterations...".format(self.steps_number))
run(mesh, step_n=self.steps_number)
print("Finished!")
texture = t_texture.eval()
print("Exporting result texture to '{}'...".format(
self.output_texture_path))
lucid_io.save(texture, self.output_texture_path, quality=90)
sess.close()
print("Importing result model to Metashape '{}'...".format(
self.result_model_path))
chunk.model = None
chunk.importModel(self.result_model_path)
chunk.model.label = self.style_name
Metashape.app.messageBox(
"Everything worked fine!\n"
"Please save project and RESTART Metashape!\n"
"Because video memory was not released by TensorFlow!")
def render_frame(frame_):
textures_ = textures
# Add noise to textures
textures_ = textures_ * tf.random_uniform(
[3], texture_channel_multiplicative_noise_[0],
texture_channel_multiplicative_noise_[1])
textures_ = textures_ + tf.random_uniform(
[3], texture_channel_additive_noise_[0],
texture_channel_additive_noise_[1])
textures_ = textures_ * tf.random_uniform(
[], texture_pixel_multiplicative_noise_[0],
texture_pixel_multiplicative_noise_[1])
textures_ = textures_ + tf.random_uniform(
[], texture_pixel_additive_noise_[0],
texture_pixel_additive_noise_[1])
textures_ = textures_ + tf.random_normal(
textures_.shape,
stddev=tf.random_uniform([], texture_gaussian_noise_stddev_[0],
texture_gaussian_noise_stddev_[1]))
#textures_ = tf.clip_by_value(textures_, 0.0, 1.0)
# Render
uvf_ = frame_[..., :3]
image_ = sample_bilinear(textures_, uvf_)
# Composite onto background
# FIXME: This only really works with batch_size=1
alpha_ = frame_[..., 3:]
image_ = image_ * alpha_ + backgrounds_[0] * (1 - alpha_)
# Blur image
image_ = image_[tf.newaxis, :, :, :]
image_ = tf.nn.depthwise_conv2d(image_,
image_gaussian_blur_kernel_,
strides=[1, 1, 1, 1],
padding='SAME')
image_ = image_[0]
# Blur alpha
alpha_ = alpha_[tf.newaxis, :, :, :]
alpha_ = tf.nn.depthwise_conv2d(
alpha_,
image_gaussian_blur_kernel_[:, :, :1, :],
strides=[1, 1, 1, 1],
padding='SAME')
alpha_ = alpha_[0]
# Recomposite blurred image onto background
# FIXME: This only really works with batch_size=1
image_ = image_ * alpha_ + backgrounds_[0] * (1 - alpha_)
# Add noise to image
image_ = image_ * tf.random_uniform(
[3], image_channel_multiplicative_noise_[0],
image_channel_multiplicative_noise_[1])
image_ = image_ + tf.random_uniform([3],
image_channel_additive_noise_[0],
image_channel_additive_noise_[1])
image_ = image_ * tf.random_uniform(
[], image_pixel_multiplicative_noise_[0],
image_pixel_multiplicative_noise_[1])
image_ = image_ + tf.random_uniform([], image_pixel_additive_noise_[0],
image_pixel_additive_noise_[1])
image_ = image_ + tf.random_normal(
tf.shape(image_),
stddev=tf.random_uniform([], image_gaussian_noise_stddev_[0],
image_gaussian_noise_stddev_[1]))
#image_ = tf.clip_by_value(image_, 0.0, 1.0)
return image_
def index():
# return 'Hello World!'
model = vision_models.InceptionV1()
model.load_graphdef()
TEXTURE_SIZE = 1024
mesh = meshutil.load_obj('article_models/bunny.obj')
mesh = meshutil.normalize_mesh(mesh)
original_texture = prepare_image('article_models/bunny.png',
(TEXTURE_SIZE, TEXTURE_SIZE))
style_url = 'https://upload.wikimedia.org/wikipedia/commons/d/db/RIAN_archive_409362_Literaturnaya_Gazeta_article_about_YuriGagarin%2C_first_man_in_space.jpg'
style = prepare_image(style_url)
renderer = glrenderer.MeshRenderer((512, 512))
googlenet_style_layers = [
'conv2d2',
'mixed3a',
'mixed3b',
'mixed4a',
'mixed4b',
'mixed4c',
]
googlenet_content_layer = 'mixed3b'
content_weight = 100.0
# Style Gram matrix weighted average decay coefficient
style_decay = 0.95
sess = create_session(timeout_sec=0)
# t_fragments is used to feed rasterized UV coordinates for the current view.
# Channels: [U, V, _, Alpha]. Alpha is 1 for pixels covered by the object, and
# 0 for background.
t_fragments = tf.placeholder(tf.float32, [None, None, 4])
t_uv = t_fragments[..., :2]
t_alpha = t_fragments[..., 3:]
# Texture atlas to optimize
t_texture = param.image(TEXTURE_SIZE, fft=True, decorrelate=True)[0]
# Variable to store the original mesh texture used to render content views
content_var = tf.Variable(tf.zeros([TEXTURE_SIZE, TEXTURE_SIZE, 3]),
trainable=False)
# Sample current and original textures with provided pixel data
t_joined_texture = tf.concat([t_texture, content_var], -1)
t_joined_frame = sample_bilinear(t_joined_texture, t_uv) * t_alpha
t_frame_current, t_frame_content = t_joined_frame[..., :3], t_joined_frame[
..., 3:]
t_joined_frame = tf.stack([t_frame_current, t_frame_content], 0)
# Feeding the rendered frames to the Neural Network
t_input = tf.placeholder_with_default(t_joined_frame,
[None, None, None, 3])
model.import_graph(t_input)
# style loss
style_layers = [
sess.graph.get_tensor_by_name('import/%s:0' % s)[0]
for s in googlenet_style_layers
]
# L1-loss seems to be more stable for GoogleNet
# Note that we use style_decay>0 to average style-describing Gram matrices
# over the recent viewports. Please refer to StyleLoss for the details.
sl = StyleLoss(style_layers, style_decay, loss_func=mean_l1_loss)
# content loss
content_layer = sess.graph.get_tensor_by_name('import/%s:0' %
googlenet_content_layer)
content_loss = mean_l1_loss(content_layer[0],
content_layer[1]) * content_weight
# setup optimization
total_loss = content_loss + sl.style_loss
t_lr = tf.constant(0.05)
trainer = tf.train.AdamOptimizer(t_lr)
train_op = trainer.minimize(total_loss)
init_op = tf.global_variables_initializer()
loss_log = []
reset(style, original_texture)
run(mesh)
texture = t_texture.eval()
return show.textured_mesh(mesh, texture)
# t_fragments is used to feed rasterized UV coordinates for the current view.
# Channels: [U, V, _, Alpha]. Alpha is 1 for pixels covered by the object, and
# 0 for background.
t_fragments = tf.placeholder(tf.float32, [None, None, 4])
t_uv = t_fragments[...,:2]
t_alpha = t_fragments[...,3:]
# Texture atlas to optimize
t_texture = param.image(TEXTURE_SIZE, fft=True, decorrelate=True)[0]
# Variable to store the original mesh texture used to render content views
content_var = tf.Variable(tf.zeros([TEXTURE_SIZE, TEXTURE_SIZE, 3]), trainable=False)
# Sample current and original textures with provided pixel data
t_joined_texture = tf.concat([t_texture, content_var], -1)
t_joined_frame = sample_bilinear(t_joined_texture, t_uv) * t_alpha
t_frame_current, t_frame_content = t_joined_frame[...,:3], t_joined_frame[...,3:]
t_joined_frame = tf.stack([t_frame_current, t_frame_content], 0)
# Feeding the rendered frames to the Neural Network
t_input = tf.placeholder_with_default(t_joined_frame, [None, None, None, 3])
model.import_graph(t_input)
# style loss
style_layers = [sess.graph.get_tensor_by_name('import/%s:0'%s)[0] for s in googlenet_style_layers]
# L1-loss seems to be more stable for GoogleNet
# Note that we use style_decay>0 to average style-describing Gram matrices
# over the recent viewports. Please refer to StyleLoss for the details.
sl = StyleLoss(style_layers, style_decay, loss_func=mean_l1_loss)
# content loss