def cluster_analysis(): from sklearn.cluster import KMeans from sklearn.metrics import silhouette_score from sklearn.metrics import mean_squared_error from sklearn import decomposition n_sample = 100 indices = yTrain[np.logical_and(yTrain >= 1, yTrain <= 10)][:n_sample] x_train = xTrain[indices] y_train = yTrain[indices] np.random.shuffle(x_train) pca = decomposition.PCA(n_components=10) _X = x_train.reshape(n_sample, -1) pca.fit_transform(_X) with tf.Session() as sess: image = tf.placeholder('float', shape=[None, 32, 32, 3]) net = vgg.net_preloaded(vgg_weights, image, pooling) X = net[last_layer].eval(feed_dict={image: x_train}) X = X.reshape(n_sample, -1) pca = decomposition.PCA(n_components=10) cluster_X = pca.fit_transform(X) print pca.singular_values_ sil_scores = [] kmin = 2 kmax = 25 for k in range(kmin, kmax): km = KMeans(n_clusters=k, n_init=20).fit(cluster_X) sil_scores.append(silhouette_score(cluster_X, km.labels_)) print sil_scores with tf.Session() as sess: image = tf.placeholder('float', shape=[None, 32, 32, 3]) net = vgg.net_preloaded(vgg_weights_2, image, pooling, apply_pruning=True, target_w=target_w, prune_percent=prune_percent) X = net[last_layer].eval(feed_dict={image: x_train}) X = X.reshape(n_sample, -1) pca = decomposition.PCA(n_components=10) X = pca.fit_transform(X) print pca.singular_values_ cluster_X = pca.fit_transform(X) sil_scores = [] kmin = 2 kmax = 25 for k in range(kmin, kmax): km = KMeans(n_clusters=k, n_init=20).fit(cluster_X) sil_scores.append(silhouette_score(cluster_X, km.labels_)) print sil_scores
def test(): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ content = xVal[0, :, :, :] # compute content features in feedforward mode content = content.reshape((1, ) + content.shape) with tf.Session() as sess: image = tf.placeholder('float', shape=[None, 32, 32, 3]) net = vgg.net_preloaded(vgg_weights, image, pooling) #content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) # retrain happens here # train(sess,image,net,buildModel(vgg_weights),xTrain, yTrain, xVal, yVal, xTest, yTest) for weight_name, weight in net.items(): if weight_name in target_w: filename = 'complete_%s' % (weight_name) save_response(net[last_layer].eval(feed_dict={image: content}), filename) with tf.Session() as sess: image = tf.placeholder('float', shape=[None, 32, 32, 3]) net = vgg.net_preloaded(vgg_weights_2, image, pooling, apply_pruning=True, target_w=target_w, prune_percent=prune_percent) # train(sess,image,net,buildModel2(vgg_weights_2),xTrain, yTrain, xVal, yVal, xTest, yTest) for weight_name, weight in net.items(): if weight_name in target_w: filename = 'pruned_%s_%s' % (weight_name, str(prune_percent[weight_name])) save_response(net[last_layer].eval(feed_dict={image: content}), filename) print "done"
def score_img(content_features, fname, vgg_weights, vgg_mean_pixel): '''Return the scores for a given image, compared with content image ''' try: style = read_img(fname) except: # disregard file if it cannot be read as an image return np.float('inf') * np.ones(len(content_features)) scores = np.zeros(len(content_features)) with tf.Graph().as_default(), tf.Session() as sess: image = tf.placeholder('float', shape=(1, ) + style.shape) net = vgg.net_preloaded(vgg_weights, image, 'max') style_pre = np.array([vgg.preprocess(style, vgg_mean_pixel)]) for i, s in enumerate(scores): content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append( tf.nn.l2_loss(net[content_layer] - content_features[i][content_layer])) content_loss = reduce(tf.add, content_losses) scores[i] = content_loss.eval(feed_dict={image: style_pre}) print(fname) return scores
def calculate_content_feature(self, pooling, content, vgg_mean_pixel): """ Obliczanie właściwości obrazu """ g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session(): self.image = tf.placeholder('float', shape=self.shape) self.net = vgg.net_preloaded(self.vgg_weights, self.image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in self.content_layers: self.content_features[layer] = self.net[layer].eval( feed_dict={self.image: content_pre})
def calculate_style_feature(self, styles, pooling, vgg_mean_pixel): """ Obliczanie właściwości stylu """ for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: self.image = tf.placeholder('float', shape=self.style_shapes[i]) self.net = vgg.net_preloaded(self.vgg_weights, self.image, pooling) style_pre = np.array( [vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in self.style_layers: features = self.net[layer].eval( feed_dict={self.image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size self.style_features[i][layer] = gram
def styleloss(network, image1, image2, layer_weight_exp, pooling): """ Calculate style similarity utilizing style (gram) matrix. This function returns "style loss", which indicates how dissimilar two input images are. """ image1_shape = (1, ) + image1.shape # (1, height, width, number) image2_shape = (1, ) + image2.shape image1_features = {} image2_features = {} vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 layers_weights = {} for layer in LAYERS: layers_weights[layer] = layer_weight layer_weight *= layer_weight_exp # normalize layer weights layer_weights_sum = 0 for layer in LAYERS: layer_weights_sum += layers_weights[layer] for layer in LAYERS: layers_weights[layer] /= layer_weights_sum # compute image1 features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=image1_shape) net = vgg.net_preloaded(vgg_weights, image, pooling) image1_pre = np.array([vgg.preprocess(image1, vgg_mean_pixel)]) for layer in LAYERS: features = net[layer].eval(feed_dict={image: image1_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) image1_features[layer] = gram # compute image2 features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=image2_shape) net = vgg.net_preloaded(vgg_weights, image, pooling) image2_pre = np.array([vgg.preprocess(image2, vgg_mean_pixel)]) for layer in LAYERS: features = net[layer].eval(feed_dict={image: image2_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) image2_features[layer] = gram # calculate style loss from gram matrices with tf.Graph().as_default(): style_loss = 0 style_losses = [] for layer in LAYERS: temp_layer = net[layer] _, height, width, number = map(lambda i: i.value, temp_layer.get_shape()) size = height * width * number image1_gram = image1_features[layer] image2_gram = image2_features[layer] style_losses.append(layers_weights[layer] * tf.nn.l2_loss(image1_gram - image2_gram) / size**2) style_losses = reduce(tf.add, style_losses) with tf.Session() as sess: style_loss = style_losses.eval() return style_loss
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image, loss_vals) at every iteration. However `image` and `loss_vals` are None by default. Each `checkpoint_iterations`, `image` is not None. Each `print_iterations`, `loss_vals` is not None. `loss_vals` is a dict with loss values for the current iteration, e.g. ``{'content': 1.23, 'style': 4.56, 'tv': 7.89, 'total': 13.68}``. :rtype: iterator[tuple[int,image]] """ shape = (1, ) + content.shape style_shapes = [(1, ) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + ( tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append( content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce( tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:, 1:, :, :]) tv_x_size = _tensor_size(image[:, :, 1:, :]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) / tv_y_size) + (tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) / tv_x_size)) # total loss loss = content_loss + style_loss + tv_loss # We use OrderedDict to make sure we have the same order of loss types # (content, tv, style, total) as defined by the initial costruction of # the loss_store dict. This is important for print_progress() and # saving loss_arrs (column order) in the main script. # # Subtle Gotcha (tested with Python 3.5): The syntax # OrderedDict(key1=val1, key2=val2, ...) does /not/ create the same # order since, apparently, it first creates a normal dict with random # order (< Python 3.7) and then wraps that in an OrderedDict. We have # to pass in a data structure which is already ordered. I'd call this a # bug, since both constructor syntax variants result in different # objects. In 3.6, the order is preserved in dict() in CPython, in 3.7 # they finally made it part of the language spec. Thank you! loss_store = OrderedDict([('content', content_loss), ('style', style_loss), ('tv', tv_loss), ('total', loss)]) # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) print('Optimization started...') if (print_iterations and print_iterations != 0): print_progress(get_loss_vals(loss_store)) iteration_times = [] start = time.time() for i in range(iterations): iteration_start = time.time() if i > 0: elapsed = time.time() - start # take average of last couple steps to get time per iteration remaining = np.mean( iteration_times[-10:]) * (iterations - i) print('Iteration %4d/%4d (%s elapsed, %s remaining)' % (i + 1, iterations, hms(elapsed), hms(remaining))) else: print('Iteration %4d/%4d' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): loss_vals = get_loss_vals(loss_store) print_progress(loss_vals) else: loss_vals = None if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray( styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) else: img_out = None yield i + 1 if last_step else i, img_out, loss_vals iteration_end = time.time() iteration_times.append(iteration_end - iteration_start)
layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum #------------------------------------------------------------- # # compute content features in feedforward mode content_features = {} g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: # image = <tf.Tensor 'Placeholder:0' shape=(1, 399, 600, 3) dtype=float32> image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, POOLING) # 기존의 vgg net을 갖다쓴다 # content_pre = (1, 399, 600, 3) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) # 입력으로 줄 이미지를 준비한다 for layer in CONTENT_LAYERS: #content_features.keys() ==> ['relu4_2','relu5_2'] # content 사진을 넣었을 때 net의 값을 저장해놓는다 content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # network를 evaluation해서 각 콘텐츠의 feature를 뽑아낸다 #------------------------------------------------------------- # compute style features in feedforward mode
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1,) + content.shape style_shapes = [(1,) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network)#加载vgg19与训练模型 layer_weight = 1.0#layer权重默认为1 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight#style_layer_weight_exp为图像风格的权重默认为1,否则为指数级增长 layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/gpu:0'), tf.Session() as sess:#使用gpu训练,cpu训练大约2个小时,gpu5分钟 image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/gpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling)#池化层默认为max规则 style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3]))#将一维数组根据图像大小转为三维 gram = np.matmul(features.T, features) / features.size#计算gram矩阵 style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256#得到一个随机白噪音 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial)#将得到的白噪音转为tensorflow对象 net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} #网络的高层特征一般是关于输入图像的物体和布局等信息,低层特征一般表达输入图像的像素信息 #最终选择conv4_2 content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) #计算content loss # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:,1:,:,:]) tv_x_size = _tensor_size(image[:,:,1:,:]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # overall loss #总loos为loss相加 loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress():#输出相关信息 stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() iteration_times = [] start = time.time() for i in range(iterations): iteration_start = time.time() if i > 0: elapsed = time.time() - start # take average of last couple steps to get time per iteration remaining = np.mean(iteration_times[-10:]) * (iterations - i) stderr.write('Iteration %4d/%4d (%s elapsed, %s remaining)\n' % ( i + 1, iterations, hms(elapsed), hms(remaining) )) else: stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. 将风格图像的RGB转为gray # 2. 将风格图像gray转为ycrcb # 3. 将事物图像转为ycrcb # 4. 将图像重组 # 5. 最后转为RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( (None if last_step else i), img_out ) iteration_end = time.time() iteration_times.append(iteration_end - iteration_start)
def stylize(network, initial, initial_noiseblend, content, styles, matte, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, matte_weight, learning_rate, beta1, beta2, epsilon, pooling, output, dest_txt, dest_fig, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1,) + content.shape #rajoute un 1 en tant que 1ere dimension de content style_shapes = [(1,) + style.shape for style in styles] #idem sur les images de style content_features = {} #Création dico style_features = [{} for _ in styles] #idem pour chaque image de style vgg_weights, vgg_mean_pixel = vgg.load_net(network) print('\n',vgg_mean_pixel.shape,'\n') layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight # => relu1_1 : 1 ; relu2_1 : 1*style_layer_weight_exp ; ... ; relu5_1 : (style_layer_weight_exp)**4 layer_weight *= style_layer_weight_exp # (default : style_layer_weight_exp=1) => seulement des 1 # normalize style layer weights => sum=1 layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # => on obtient 1 liste normalisée à 5 élts pour chaque image de style # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: #Toutes les opérations construites dans ce contexte (indentées) seront placées sur le CPU:0 et dans le graphe g #"with Session" ferme la session lorsque c'est terminé image = tf.placeholder('float', shape = shape) net = vgg.net_preloaded(vgg_weights, image, pooling) #dictionnaire associant à chaque élt de VGG19-LAYERS un tensor , shape.len=4 content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) #retourne une matrice image_style[i] - vgg_mean_pixel for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) # print("\n") # print(features) # print("shape",features.shape, features.size) # print("\n") features = np.reshape(features, (-1, features.shape[3])) # print("\n") # print(features) # print("shape",features.shape, features.size) # print("\n") gram = np.matmul(features.T, features) / features.size #matmul = matrix multiplication => gram=[features(transposée) x features] / features.size style_features[i][layer] = gram #style_features = liste de dictionnaires initial_content_noise_coeff = 1.0 - initial_noiseblend #noiseblend = input (optionnel) # make stylized image using backpropogation with tf.Graph().as_default(): #initial = tf.random_normal(shape) * 0 #image de départ = blanche if initial is None: #initial = image de laquelle on part pour construire l'image suivante #noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 #initial non renseignée => aléatoire else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) # initial - mean_pixel initial = initial.astype('float32') #noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) #(default : initial_noiseblend=0) => initial = inchangé image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend #default : content_weight_blend = 1 ==>...['relu4_2]=1 content_layers_weights['relu5_2'] = 1.0 - content_weight_blend #==>...['relu5_2]=0 content_loss = 0 #initialisation inutile mais on garde le même format pour style loss content_losses = [] for content_layer in CONTENT_LAYERS: #CONTENT_LAYERS = ('relu4_2', 'relu5_2') content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( #content_weight = alpha/2 net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) #content_losses = liste de 2 élts #net[content_layer] = features de l'image générée ; content_features[content_layer] = features de l'image d'origine content_loss += reduce(tf.add, content_losses) # = somme des élts de content_losses (on calcule l'erreur sur chaque layer, puis on additionne ces erreurs) #(default : content_layers_weights['relu5_2]=0 => content_loss = content_losses[0]) # style loss style_loss = 0 for i in range(len(styles)): #nb d'images de style style_losses = [] for style_layer in STYLE_LAYERS: #STYLE_LAYERS = ('relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1') layer = net[style_layer] _, height, width, number = map(lambda j: j.value, layer.get_shape()) # "_" => discard the first elt of the tuple #lambda = definit la fonction qui a j associe j.value ; map applique la fonction a tous les élts de layer.get_shape) size = height * width * number # print("number ",number) # print("layer.shape",layer.get_shape()) feats = tf.reshape(layer, (-1, number)) #supprime dim0 (=1), dim0=dim1*dim2, dim1=dim3=number => shape = (dim1*dim2 , number) # print("feats.shape",feats.get_shape()) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] #style_features = liste de dictionnaires initialisée dans "compute style featurs in feedforwardmode" style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) #liste contenant les erreurs de tous les layers de l'image i #gram = style representation of generated image ; style_gram = style representation of original image style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) #incrémentation de style_loss : reduce=sum(err layers de im[i]) ; style_weight = poids du style par rapp au content # style_blend_weights[i] = pids de l'im. i par rapp aux autres # += => on somme les losses de toutes les images # matting lapacian loss loader = np.load(matte) lcoo = csr_matrix((loader['data'], loader['indices'], loader['indptr']), shape=loader['shape']).tocoo() lindices = np.mat([lcoo.row, lcoo.col]).transpose() lvalues = tf.constant(lcoo.data, dtype=tf.float32) laplacian = tf.SparseTensor(indices=lindices, values=lvalues, dense_shape=lcoo.shape) matte_loss = 0 matte_losses = [] for i in range(3): imr = tf.reshape(image[:,:,:,i], [-1, 1]) matte_losses.append( tf.matmul(tf.transpose(imr), tf.sparse_tensor_dense_matmul(laplacian, imr))[0][0] ) matte_loss += matte_weight * reduce(tf.add, matte_losses) # total variation denoising (pas très important : a remplacer par une autre loss ?) print("\n total variation denoising") #(possible de désactiver la tv loss avec la commande --tv-weight 0) tv_y_size = _tensor_size(image[:,1:,:,:]) print(tv_y_size) tv_x_size = _tensor_size(image[:,:,1:,:]) print(tv_x_size) print("\n") tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # GAN loss # overall loss loss = content_loss + style_loss + matte_loss + tv_loss # make alpha etc appear (coeffs) # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) # (operation qui met a jour les variables pour que total loss soit minimise) # quelles variables ??? def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) # stderr.write(' matte loss: %g\n' % matte_loss.eval()) stderr.write(' GAN loss: %g\n' % GAN_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') #??? best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) #initialise les variables globales stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): #Si on a rentré un pas pour print_iterations, on affiche avant la 1ere iteration les loss e initial print_progress() c_loss = [] #initialisation des listes de valeurs de loss s_loss = [] t_loss = [] tot_loss = [] x=[i+1 for i in range(iterations)] #initialisation des abscisses des graphes for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() #on minimise loss a chaque itération c_loss.append(content_loss.eval()) #incrémentation des listes de valeurs de loss pour chaque itération s_loss.append(style_loss.eval()) t_loss.append(tv_loss.eval()) tot_loss.append(loss.eval()) last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0) : #i % print_iterations = reste de la diveucl de i par print_iterations print_progress() #On affiche les loss instantannées avec une fréquence = print_iterations if last_step : if dest_txt is None: l=len(output)-4 #Création d'un fichier contenant les losses (même nom que l'output mais .txt) file=output[:l] F=open(''.join([file,'.txt']),'x') #fusionne file et '.txt' F.writelines([' content loss: %g\n' % content_loss.eval() , ' style loss: %g\n' % style_loss.eval() , ' tv loss: %g\n' % tv_loss.eval() , ' total loss: %g\n' % loss.eval()]) F.close else: F=open(dest_txt,'x') F.writelines([' content loss: %g\n' % content_loss.eval() , ' style loss: %g\n' % style_loss.eval() , ' tv loss: %g\n' % tv_loss.eval() , ' total loss: %g\n' % loss.eval()]) F.close if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() #on associe l'image finale à la meilleure loss totale img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: #preserve-colors original_image = np.clip(content, 0, 255) #clip = tous les élts de content >255 -->255, idem <0 -->0 styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( (None if last_step else i), img_out ) #Nom de la destination des courbes if dest_fig is None : l=len(output)-4 file=output[:l] dest_fig=''.join([file,'_fig','.jpg']) print('dest_fig',dest_fig) #Tracé des graphes plt.figure(1) plt.title("Différents types d'erreurs - graphe classique et graphe semi-logarithmique") plt.subplot(2,1,1) plt.plot(x, c_loss, label='content_loss') plt.plot(x, s_loss, label='style_loss') plt.plot(x, t_loss, label='tv_loss') plt.plot(x, tot_loss, label='total_loss') plt.grid('on') plt.axis('tight') plt.legend() plt.ylabel('erreur') plt.subplot(2,1,2) plt.semilogy(x, c_loss, label='content_loss') plt.semilogy(x, s_loss, label='style_loss') plt.semilogy(x, t_loss, label='tv_loss') plt.semilogy(x, tot_loss, label='total_loss') plt.grid('on') plt.axis('tight') plt.xlabel("i (Nombre d'itérations)") plt.ylabel('erreur') plt.savefig(dest_fig)
def stylize(Ray_render, ray_steps, reset_opp, session, network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1, ) + content.shape style_shapes = [(1, ) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel) ]).astype(np.float32) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={image: content_pre}) image = initial - tf.cast(tf.reshape(vgg_mean_pixel, (1, 1, 1, 3)), tf.float32) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = 1 / (1.0 * len(CONTENT_LAYERS)) content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append( content_layers_weights * content_weight * (2 * tf.nn.l2_loss(net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 # overall loss loss = content_loss + style_loss #+ tv_loss # optimizer setup render_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='parameters') with tf.variable_scope('OPTIMIZATION', reuse=tf.AUTO_REUSE): train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize( loss, var_list=render_vars) session.run(tf.initialize_all_variables()) Ray_render.trace(session, ray_steps, reset_opp, num_steps=50) # evals_ = session.run(tf.squeeze(initial,axis=0)) # <= returns jpeg data you can write to disk def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval(session=session)) # stderr.write(' style loss: %g\n' % style_loss.eval(session=session)) stderr.write(' total loss: %g\n' % loss.eval(session=session)) print_progress() # aa= np.squeeze(net[CONTENT_LAYERS[0]].eval(session=session),0) # bb = np.squeeze(content_features[CONTENT_LAYERS[0]],0) # pic_aa=np.squeeze(content) # pic_bb=np.squeeze(initial.eval(session=session),0) # # fig = plt.figure(1) # ax2 = fig.add_subplot(1, 1, 1) # ax2.imshow(pic_aa) # # fig = plt.figure(2) # ax2 = fig.add_subplot(1, 1, 1) # ax2.imshow(pic_bb) # aa.aa=1 # optimization stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run(session=session) Ray_render.trace(session, ray_steps, reset_opp, num_steps=50) last_step = (i == iterations - 1) print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: image_ = image.eval(session=session) img_out = vgg.unprocess(image_.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray(styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ((None if last_step else i), img_out)
vgg_weights, vgg_mean_pixel = vgg.load_net(network) print(5) orig_image = tf.placeholder( 'float', shape=shape) #need to feed it with (1,256,256,3) objects print(orig_image) orig_content = vgg.preprocess(orig_image, vgg_mean_pixel) #tensor (1,256,256,3) print(orig_content) print('G_sample.shape', G_sample.shape) #G_sample = tf.reshape(G_sample,(mb_size,256,256)) #G_sample = tf.stack([G_sample,G_sample,G_sample],axis=3) #tensor (256,256,3) #print('G_sample.shape',G_sample.shape) gen_content = vgg.preprocess(G_sample, vgg_mean_pixel) # gen_content = tf.expand_dims(gen_content,0) print('ok') orig_net = vgg.net_preloaded(vgg_weights, orig_content, pooling) print('ok1') gen_net = vgg.net_preloaded(vgg_weights, gen_content, pooling) #content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) #print('content_pre.shape',content_pre.shape) print(6) """ Losses """ #feat_loss feat_loss = 0 for layer in CONTENT_LAYERS: feat_loss += tf.nn.l2_loss(orig_net[layer] - gen_net[layer]) D_loss_real = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real))) # labels = matrice de 1, même type que logits
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1,) + content.shape #若content.shape=(356, 600, 3) shape=(1,356, 600, 3) style_shapes = [(1,) + style.shape for style in styles] content_features = {} #创建内容features map style_features = [{} for _ in styles] #创建风格features map vgg_weights, vgg_mean_pixel = vgg.load_net(network) #加载预训练模型,得到weights和mean_pixel layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp #若有设置style_layer_weight_exp,则style_layers_weights指数增长, # style_layer_weight_exp默认为1不增长 # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum #更新style_layers_weights应该是比例,使其总和为1 # 首先创建一个image的占位符,然后通过eval()的feed_dict将content_pre传给image, # 启动net的运算过程,得到了content的feature maps # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.compat.v1.Session() as sess: #计算content features image = tf.compat.v1.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) #所有网络在此构建,net为content的features maps content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) #content - vgg_mean_pixel for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) #content_features取值 # print(layer,content_features[layer].shape) # compute style features in feedforward mode for i in range(len(styles)): #计算style features g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.compat.v1.Session() as sess: image = tf.compat.v1.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) #pooling 默认为MAX style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) #styles[i]-vgg_mean_pixel for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) #根据通道数目reshape gram = np.matmul(features.T, features) / features.size #gram矩阵 style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 #初始化图片 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + ( tf.random.normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) ''' image = tf.Variable(initial)初始化了一个TensorFlow的变量,即为我们需要训练的对象。 注意这里我们训练的对象是一张图像,而不是weight和bias。 ''' net = vgg.net_preloaded(vgg_weights, image, pooling) #此处的net为生成图片的features map # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend #内容图片 content weight blend, conv4_2 * blend + conv5_2 * (1-blend) content_layers_weights['relu5_2'] = 1.0 - content_weight_blend #content weight blend默认为1,即只用conv4_2层 content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / #生成图片-内容图片 content_features[content_layer].size)) # tf.nn.l2_loss:output = sum(t ** 2) / 2 content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 ''' 由于style图像可以输入多幅,这里使用for循环。同样的,将style_pre传给image占位符, 启动net运算,得到了style的feature maps,由于style为不同filter响应的内积, 因此在这里增加了一步:gram = np.matmul(features.T, features) / features.size,即为style的feature。 ''' for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size #求得生成图片的gram矩阵 style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:,1:,:,:]) tv_x_size = _tensor_size(image[:,:,1:,:]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # overall loss ''' 接下来定义了Content Loss和Style Loss,结合文中的公式很容易看懂,在代码中, 还增加了total variation denoising,因此总的loss = content_loss + style_loss + tv_loss ''' loss = content_loss + style_loss + tv_loss #总loss为三个loss之和 # optimizer setup # optimizer setup # 创建train_step,使用Adam优化器,优化对象是上面的loss # 优化过程,通过迭代使用train_step来最小化loss,最终得到一个best,即为训练优化的结果 train_step = tf.compat.v1.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.compat.v1.Session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) #还原图片 if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( #相当于return,但用于迭代 (None if last_step else i), img_out )
def stylyze(options, callback): parser = build_parser() if options is None: key = 'TF_CPP_MIN_LOG_LEVEL' if key not in os.environ: os.environ[key] = '2' options = parser.parse_args() if not os.path.isfile(options.network): parser.error("Network %s does not exist. (Did you forget to " "download it?)" % options.network) if [options.checkpoint_iterations, options.checkpoint_output].count(None) == 1: parser.error("use either both of checkpoint_output and " "checkpoint_iterations or neither") if options.checkpoint_output is not None: if re.match(r'^.*(\{.*\}|%.*).*$', options.checkpoint_output) is None: parser.error("To save intermediate images, the checkpoint_output " "parameter must contain placeholders (e.g. " "`foo_{}.jpg` or `foo_%d.jpg`") content_image_arr = [imread(i) for i in options.content] style_images = [imread(style) for style in options.styles] width_arr = options.width for i in range(len(content_image_arr)): width = width_arr[i] content_image = content_image_arr[i] if width is not None: new_shape = (int( math.floor( float(content_image.shape[0]) / content_image.shape[1] * width)), width) content_image = scipy.misc.imresize(content_image, new_shape) content_image_arr[i] = content_image target_shape = content_image.shape for j in range(len(style_images)): style_scale = STYLE_SCALE if options.style_scales is not None: style_scale = options.style_scales[j] style_images[j] = scipy.misc.imresize( style_images[j], style_scale * target_shape[1] / style_images[j].shape[1]) style_blend_weights = options.style_blend_weights if style_blend_weights is None: # default is equal weights style_blend_weights = [1.0 / len(style_images) for _ in style_images] else: total_blend_weight = sum(style_blend_weights) style_blend_weights = [ weight / total_blend_weight for weight in style_blend_weights ] initial_arr = content_image_arr # try saving a dummy image to the output path to make sure that it's writable output_arr = options.output for output in output_arr: if os.path.isfile(output) and not options.overwrite: raise IOError("%s already exists, will not replace it without " "the '--overwrite' flag" % output) try: imsave(output, np.zeros((500, 500, 3))) except: raise IOError('%s is not writable or does not have a valid file ' 'extension for an image file' % output) vgg_weights, vgg_mean_pixel = vgg.load_net(options.network) style_shapes = [(1, ) + style.shape for style in style_images] style_features = [{} for _ in style_images] layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= options.style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute style features in feedforward mode for i in range(len(style_images)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, options.pooling) style_pre = np.array( [vgg.preprocess(style_images[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - options.initial_noiseblend for i in range(len(content_image_arr)): Data.save_step(Data.get_step() + 1) loss_arrs = None for iteration, image, loss_vals in stylize( initial=initial_arr[i], content=content_image_arr[i], preserve_colors=options.preserve_colors, iterations=options.iterations, content_weight=options.content_weight, content_weight_blend=options.content_weight_blend, tv_weight=options.tv_weight, learning_rate=options.learning_rate, beta1=options.beta1, beta2=options.beta2, epsilon=options.epsilon, pooling=options.pooling, initial_content_noise_coeff=initial_content_noise_coeff, style_images=style_images, style_layers_weights=style_layers_weights, style_weight=options.style_weight, style_blend_weights=style_blend_weights, vgg_weights=vgg_weights, vgg_mean_pixel=vgg_mean_pixel, style_features=style_features, print_iterations=options.print_iterations, checkpoint_iterations=options.checkpoint_iterations, callback=callback): if (image is not None) and (options.checkpoint_output is not None): imsave(fmt_imsave(options.checkpoint_output, iteration), image) if (loss_vals is not None) \ and (options.progress_plot or options.progress_write): if loss_arrs is None: itr = [] loss_arrs = OrderedDict( (key, []) for key in loss_vals.keys()) for key, val in loss_vals.items(): loss_arrs[key].append(val) itr.append(iteration) imsave(options.output[i], image) if options.progress_write: fn = "{}/progress.txt".format(os.path.dirname(options.output[i])) tmp = np.empty((len(itr), len(loss_arrs) + 1), dtype=float) tmp[:, 0] = np.array(itr) for ii, val in enumerate(loss_arrs.values()): tmp[:, ii + 1] = np.array(val) np.savetxt(fn, tmp, header=' '.join(['itr'] + list(loss_arrs.keys()))) if options.progress_plot: import matplotlib matplotlib.use('Agg') from matplotlib import pyplot as plt fig, ax = plt.subplots() for key, val in loss_arrs.items(): ax.semilogy(itr, val, label=key) ax.legend() ax.set_xlabel("iterations") ax.set_ylabel("loss") fig.savefig("{}/progress.png".format( os.path.dirname(options.output[i])))
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1, ) + content.shape style_shapes = [(1, ) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.gpu_options.visible_device_list = '0' with g.as_default(), g.device('/cpu:0'), tf.Session(config=config) as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session( config=config) as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + ( tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: ''' Compute the content loss Variables: content_weight: scalar constant we multiply the content_loss by. net[content_layer]: features of the current image, Tensor with shape [1, height, width, channels] content_features[content_layer]: features of the content image, Tensor with shape [1, height, width, channels] ''' # features of the current image [1, height, width, channels] l_content = content_weight * tf.reduce_sum( (net[content_layer] - content_features[content_layer])**2) content_losses.append(content_layers_weights[content_layer] * l_content) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, channels = map(lambda i: i.value, layer.get_shape()) size = height * width * channels ''' Compute the Gram matrix of the layer Variables: layer: features of the current image at style_layer, Tensor with shape [1, height, width, channels] gram: computed gram matrix with shape [channels, channels] ''' feats = tf.reshape(layer, (-1, channels)) gram = tf.matmul(tf.transpose(feats), feats) gram /= size ''' Compute the style loss Variables: style_layers_weights[style_layer]: scalar constant we multiply the content_loss by. gram: computed gram matrix with shape [channels, channels] style_gram: computed gram matrix of the style image at style_layer with shape [channels, channels] ''' style_gram = style_features[i][style_layer] l_style = style_layers_weights[style_layer] * tf.reduce_sum( (gram - style_gram)**2) style_losses.append(l_style) style_loss += style_weight * style_blend_weights[i] * reduce( tf.add, style_losses) # total variation denoising ''' Compute the TV loss Variables: tv_weight: scalar giving the weight to use for the TV loss. image: tensor of shape (1, H, W, 3) holding current image. ''' tv_loss = tv_weight * (tf.reduce_sum( (image[:, 1:, :, :] - image[:, :-1, :, :])**2) + tf.reduce_sum( (image[:, :, 1:, :] - image[:, :, :-1, :])**2)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session(config=config) as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray( styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ((None if last_step else i), img_out)
def main(_): global_step = tf.Variable(0, trainable=False, name='global_step') invert_layer = FLAGS.invert_layer ### Load pre-trained VGG wieghts vgg_mat_file = FLAGS.vgg19 print("pretrained-VGG : {}".format(FLAGS.vgg19)) vgg_weights, vgg_mean_pixel = vgg.load_net(vgg_mat_file) print("vgg_mean_pixel : ", vgg_mean_pixel) ### Read input image image = FLAGS.image print("input image : {}".format(FLAGS.image)) img = read_image(image, 224, 224) scipy.misc.imsave(sample_dir + '/input_image.png', img) img = img - vgg_mean_pixel img = img.astype(np.float32) img = np.expand_dims(img, axis=0) # extend shape for VGG input img_shape = np.shape(img) print("Image shape : ", np.shape(img)) gpu_options = tf.GPUOptions(allow_growth=True) ### Comput content feature of 'invert_layer' X_content_feature = {} content_graph = tf.Graph() with content_graph.as_default(): sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) X_content = tf.placeholder('float32', shape=img_shape) network = vgg.net_preloaded(vgg_weights, img, pooling) X_content_feature = sess.run(network[invert_layer], feed_dict={X_content: img}) ### Define network to learn 'X' # X_sigma = tf.norm(vgg_mean_pixel)*img_shape[1] # roughly... # X_sigma = tf.cast(X_sigma, tf.float32) # X = tf.Variable(tf.random_normal(img_shape))*X_sigma X = tf.Variable(tf.random_normal(img_shape)) invert_net = vgg.net_preloaded(vgg_weights, X, pooling) X_invert_feature = invert_net[invert_layer] l2_loss = tf.norm(X_content_feature - X_invert_feature, 'euclidean') / tf.norm(X_content_feature, 'euclidean') #total_variation_loss = tf.image.total_variation(img+X)[0] total_variation_loss = tf.reduce_sum( tf.image.total_variation(tf.convert_to_tensor(img + X))) sigma_tv = 5e-7 loss = l2_loss + sigma_tv * total_variation_loss train_step = tf.train.AdamOptimizer(learning_rate=0.1, beta1=0.5).minimize( loss, global_step=global_step) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) sess.run(tf.global_variables_initializer()) for step in range(max_tries): _, _loss = sess.run([train_step, loss]) print("step: %06d" % step, "loss: {:.04}".format(_loss)) #_tv = sess.run(total_variation_loss) #print("total_variation_loss : ", sigma_tv*_tv) # testing if not (step + 1) % 100: this_X = sess.run(X) this_X = this_X + vgg_mean_pixel scipy.misc.imsave( sample_dir + '/invert_{}'.format(str(step + 1).zfill(6)) + '.png', this_X[0]) '''
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1,) + content.shape style_shapes = [(1,) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:,1:,:,:]) tv_x_size = _tensor_size(image[:,:,1:,:]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() iteration_times = [] start = time.time() for i in range(iterations): iteration_start = time.time() if i > 0: elapsed = time.time() - start # take average of last couple steps to get time per iteration remaining = np.mean(iteration_times[-10:]) * (iterations - i) stderr.write('Iteration %4d/%4d (%s elapsed, %s remaining)\n' % ( i + 1, iterations, hms(elapsed), hms(remaining) )) else: stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( (None if last_step else i), img_out ) iteration_end = time.time() iteration_times.append(iteration_end - iteration_start)
def stylize(self, network, content, styles, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling): """ Nałożenie stylu na obraz Metoda jest wywoływana iteracyjnie, obliczane są straty i wagi, a potem do rodzica jest przekazywany tuple z iteratorem i tablicą obrazu oraz, jeśli to ostatnia iteracja, z obliczonymi stratami :rtype: iterator[tuple[int,image]] """ self.style_features = [{} for _ in styles] self.content_features = {} self.style_shapes = [(1, ) + style.shape for style in styles] self.shape = (1, ) + content.shape self.vgg_weights, vgg_mean_pixel = vgg.load_net(network) self.layer_weight = 1.0 for style_layer in self.style_layers: self.style_layers_weights[style_layer] = self.layer_weight self.layer_weight *= style_layer_weight_exp self.calculate_sum_weight() self.calculate_content_feature(pooling, content, vgg_mean_pixel) self.calculate_style_feature(styles, pooling, vgg_mean_pixel) # Użycie propagacji wstecznej na stylizowanym obrazie with tf.Graph().as_default(): initial = tf.random_normal(self.shape) * 0.256 self.image = tf.Variable(initial) self.net = vgg.net_preloaded(self.vgg_weights, self.image, pooling) self.calculate_content_loss(content_weight_blend, content_weight) self.calculate_style_loss(styles, style_weight, style_blend_weights) self.denoise_image(tv_weight) self.calculate_total_loss() # konfiguracja optymalizatora train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(self.loss) # optymalizacja best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for i in range(iterations): if i > 0: print('%4d/%4d' % (i + 1, iterations)) else: print('%4d/%4d' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step: loss_vals = self.get_loss_vals(self.loss_store) else: loss_vals = None if last_step: this_loss = self.loss.eval() if this_loss < best_loss: best_loss = this_loss best = self.image.eval() img_out = vgg.unprocess(best.reshape(self.shape[1:]), vgg_mean_pixel) else: img_out = None yield i + 1 if last_step else i, img_out, loss_vals
def stylize(network, initial, initial_noiseblend, content, content_mask, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1, ) + content.shape mask_shape = (1, ) + content.shape[0:2] + (1, ) style_shapes = [(1, ) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] content_mask_features = {} vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) mask = tf.placeholder('float', shape=mask_shape) net = vgg.net_preloaded(vgg_weights, image, pooling) net_mask = vgg.net_downsample(vgg_weights, mask) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={image: content_pre}) for layer in CONTENT_LAYERS + STYLE_LAYERS: content_mask_features[layer] = net_mask[layer].eval(feed_dict={ mask: np.expand_dims(np.expand_dims(content_mask, axis=0), axis=4) }) # plt.imshow(np.squeeze(content_mask_features[layer])) # plt.show() # plt.pause(0.01) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features_bank = sk_image.extract_patches_2d( np.squeeze(features), (kernel_s, kernel_s)) style_features[i][layer] = [features_bank, features] # plt.imshow(np.squeeze(initial).astype(np.uint8)) # plt.show() # plt.pause(0.01) # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: # noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') # initial_content_noise_coeff = 1.0 - initial_noiseblend # noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) # initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) # plt.imshow(np.squeeze(initial)) # plt.show() # plt.pause(0.01) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} for layer in CONTENT_LAYERS: content_layers_weights[layer] = 1. / len(CONTENT_LAYERS) content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: map_ = (net[content_layer] - content_features[content_layer]) # map_ = (net[content_layer] - content_features[content_layer])*(1.-content_mask_features[content_layer]) loss_ = content_layers_weights[content_layer] * content_weight * ( 2 * tf.nn.l2_loss(map_) / content_features[content_layer].size) content_losses.append(loss_) content_loss += reduce(tf.add, content_losses) # plt.imshow(1-np.squeeze(content_mask_features[content_layer])) # plt.show() # plt.pause(100) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: # Calculate normalized layer layer = tf.expand_dims(net[style_layer], axis=4) paddings = [[0, 0], [pad, pad], [pad, pad], [0, 0], [0, 0]] layer_depth = layer.get_shape().as_list()[3] layer_pad = tf.pad(layer, paddings, "CONSTANT") layer_norm = tf.sqrt( tf.nn.conv3d(tf.pow(layer_pad, 2), tf.ones( (kernel_s, kernel_s, layer_depth, 1, 1), dtype=tf.float32), strides=[1, 1, 1, 1, 1], padding='VALID')) # Calculate normalized filter bank style_filters = np.transpose(style_features[i][style_layer][0], (1, 2, 3, 0)) style_filters = np.expand_dims(style_filters, axis=3) style_filters_norm = np.sqrt( np.sum(np.power(style_filters, 2), axis=(0, 1, 2))) style_filters_normalized = style_filters / style_filters_norm # Calculate normalized correlations layer_filtered = tf.nn.conv3d(layer_pad, style_filters_normalized, strides=[1, 1, 1, 1, 1], padding='VALID') / layer_norm # Find maximum response and index into the filters max_filter_response_idx = tf.squeeze( tf.argmax(layer_filtered, axis=4)) max_filter_response_idx = tf.reshape(max_filter_response_idx, [-1]) max_filter_response_weight = tf.squeeze( tf.reduce_max(tf.abs(layer_filtered), axis=4)) max_filter_response_weight = tf.reshape( max_filter_response_weight, [-1]) max_filter_response_weight = max_filter_response_weight / tf.reduce_max( max_filter_response_weight) style_filters_tf = tf.transpose( tf.squeeze(tf.convert_to_tensor(style_filters, np.float32)), (3, 0, 1, 2)) style_filters_tf_gathered = tf.gather(style_filters_tf, max_filter_response_idx) style_filters_tf_gathered = tf.reshape( style_filters_tf_gathered, (style_filters_tf_gathered.get_shape().as_list()[0], -1)) layer_patches = tf.extract_image_patches( tf.squeeze(layer_pad, axis=4), [1, kernel_s, kernel_s, 1], [1, 1, 1, 1], [1, 1, 1, 1], padding="VALID") layer_size = tf.shape(layer_patches) layer_patches = tf.reshape(layer_patches, (-1, layer_size[3])) style_weights = np.reshape(content_mask_features[style_layer], (-1)) # loss_ = tf.reduce_mean(tf.reduce_mean(tf.pow(layer_patches-style_filters_tf_gathered, 2),axis=1)*tf.stop_gradient(max_filter_response_weight)) loss_ = tf.reduce_mean( tf.reduce_mean(tf.pow( layer_patches - style_filters_tf_gathered, 2), axis=1) * style_weights) style_losses.append(style_layers_weights[style_layer] * 2 * loss_) style_loss += style_weight * style_blend_weights[i] * reduce( tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:, 1:, :, :]) tv_x_size = _tensor_size(image[:, :, 1:, :]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) / tv_y_size) + (tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) # print(str(max_filter_response_weight.eval())) # print(' ') train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray( styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ((None if last_step else i), img_out)
def inferenceImg(network, initial_img, initial_noiseblend, content, style, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weight, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations, checkpoint_iterations): content_shape = (1, ) + content.shape style_shape = (1, ) + style.shape content_features = {} style_features = {} vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight = layer_weight * style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum = layer_weights_sum + style_layers_weights[ style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = style_layers_weights[ style_layer] / layer_weights_sum # compute content features in feedforward mode g1 = tf.Graph() with g1.as_default(), g1.device('/cpu:0'), tf.Session() as sess: contentImg = tf.placeholder('float', shape=content_shape) net = vgg.net_preloaded(vgg_weights, contentImg, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={contentImg: content_pre}) # compute style features in feedforward mode g2 = tf.Graph() with g2.as_default(), g2.device('/cpu:0'), tf.Session() as sess: styleImg = tf.placeholder('float', shape=style_shape) net = vgg.net_preloaded(vgg_weights, styleImg, pooling) style_pre = np.array([vgg.preprocess(style, vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={styleImg: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): noise = np.random.normal(size=content_shape, scale=np.std(content) * 0.1) initial = tf.random_normal(content_shape) * 0.256 inferenceImg = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, inferenceImg, pooling) # compute content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append( content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # compute style loss style_loss = 0 style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weight * reduce( tf.add, style_losses) # skip compute variation denoise, in order to shorten the running time # total variation denoising # tv_y_size = _tensor_size(inferenceImg[:, 1:, :, :]) # tv_x_size = _tensor_size(inferenceImg[:, :, 1:, :]) # tv_loss = tv_weight * 2 * ( # (tf.nn.l2_loss(inferenceImg[:, 1:, :, :] - inferenceImg[:, :content_shape[1] - 1, :, :]) / # tv_y_size) + # (tf.nn.l2_loss(inferenceImg[:, :, 1:, :] - inferenceImg[:, :, :content_shape[2] - 1, :]) / # tv_x_size)) tv_loss = 0 # overall loss loss = content_loss + style_loss + tv_loss # optimizer training train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = inferenceImg.eval() img_out = vgg.unprocess(best.reshape(content_shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray( styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ((None if last_step else i), img_out)
def sparsizeALL(network, img, regularisation_coeff, iterations, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1, ) + img.shape img_conteneur = np.zeros(shape, dtype='float32') img_conteneur[0, :, :, :] = img vgg_weights, vgg_mean_pixel = vgg.load_net(network) loss_curve = [] sparsness_curve = [] # make sparse encoded image using backpropogation with tf.Graph().as_default(): init_image = tf.constant(img_conteneur) net_init = vgg.net_preloaded(vgg_weights, init_image, pooling) layer_nonzero_init = [] layer_size = [] for layer in RELU_LAYERS: init_encoding = net_init[layer] layer_nonzero_init.append(tf.count_nonzero(init_encoding)) layer_size.append(tf.size(init_encoding)) nonzero_init = tf.reduce_sum(layer_nonzero_init) size = tf.reduce_sum(layer_size) init_sparsness = 1. - tf.to_float(nonzero_init) / tf.to_float(size) image = tf.Variable(tf.random_normal(shape) * 0.256) net = vgg.net_preloaded(vgg_weights, image, pooling) layer_l1_norm = [] layer_nonzero = [] layer_size = [] for layer in RELU_LAYERS: encoding = net[layer] layer_l1_norm.append(tf.reduce_sum(encoding)) layer_nonzero.append(tf.count_nonzero(encoding)) layer_size.append(tf.size(encoding)) reg_term = tf.reduce_sum(layer_l1_norm) nonzero = tf.reduce_sum(layer_nonzero) sparsness = 1. - tf.to_float(nonzero) / tf.to_float(size) lbda = tf.constant(regularisation_coeff) loss = tf.nn.l2_loss(image - init_image) + lbda * reg_term # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' loss: %g\n' % loss.eval()) stderr.write(' initial sparsness: %g\n' % init_sparsness.eval()) stderr.write(' final sparsness: %g\n' % sparsness.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() iteration_times = [] start = time.time() for i in range(iterations): iteration_start = time.time() if i > 0: elapsed = time.time() - start # take average of last couple steps to get time per iteration remaining = np.mean( iteration_times[-10:]) * (iterations - i) stderr.write( 'Iteration %4d/%4d (%s elapsed, %s remaining)\n' % (i + 1, iterations, hms(elapsed), hms(remaining))) loss_curve.append(sess.run(loss)) sparsness_curve.append(sess.run(sparsness)) else: stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = best.reshape(shape[1:]) yield ((None if last_step else i), img_out) iteration_end = time.time() iteration_times.append(iteration_end - iteration_start) fig1 = plt.figure('loss') plt.xlabel('Iterations') plt.ylabel('Fonction de perte') plt.title('Fonction de perte au fil des itérations') plt.step(np.arange(1, iterations), loss_curve, 'r') fig2 = plt.figure('sparsness') plt.xlabel('Iterations') plt.title("Parcimonie de l'encodage de la nouvelle image") plt.ylabel("Pourcentage de zéro dans l'encodage de la nouvelle image") plt.step(np.arange(1, iterations), sparsness_curve, 'b') # fig1.savefig('body1_transformation/' + layer + '_' + str(regularisation_coeff) + '_loss_plot.png') # fig2.savefig('body1_transformation/' + layer + '_' + str(regularisation_coeff) + '_sparsness_plot.png') plt.show()
def stylize(content, style, initial, initial_noiseblend, content_weight=5e0, content_layer_num=9, style_weight=5e2, style_layer_weight=(0.2, 0.2, 0.2, 0.2, 0.2), tv_weight=1e2, learning_rate=1e1, beta1=0.9, beta2=0.999, epsilon=1e-8, preserve_colors=False, pooling='max', iterations=1000, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1, ) + content.shape content_features = {} style_features = {} style_layers_weights = {} content_layer = CONTENT_LAYERS[content_layer_num] for i, style_layer in enumerate(STYLE_LAYERS): style_layers_weights[style_layer] = style_layer_weight[i] vgg_weights, vgg_mean_pixel = vgg.load_net(network) image = tf.placeholder(tf.float32, shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) style_pre = np.array([vgg.preprocess(style, vgg_mean_pixel)]) # compute content features,style features in feedforward mode with tf.Session() as sess: content_features[content_layer] = sess.run( net[content_layer], feed_dict={image: content_pre}) for layer in STYLE_LAYERS: features = sess.run(net[layer], feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[layer] = gram # make stylized image using backpropogation if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype(np.float32) noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = initial * (1 - initial_noiseblend) + ( tf.random_normal(shape) * 0.256) * initial_noiseblend image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_loss = content_weight * 2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size # style loss style_loss = 0 for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[style_layer] style_loss += style_weight * style_layers_weights[ style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size # total variation denoising tv_y_size = _tensor_size(image[:, 1:, :, :]) tv_x_size = _tensor_size(image[:, :, 1:, :]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) / tv_y_size) + (tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): print(' content loss: %g\n' % content_loss.eval()) print(' style loss: %g\n' % style_loss.eval()) print(' tv loss: %g\n' % tv_loss.eval()) print(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None images = [] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) print('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print('Iteration %4d/%4d\n' % (i + 1, iterations)) print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() styled_image = np.clip( vgg.unprocess(image.eval().reshape(shape[1:]), vgg_mean_pixel), 0, 255) if this_loss < best_loss: best_loss = this_loss best = styled_image if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray(styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 styled_image = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) plt.figure(figsize=(8, 8)) plt.imshow(styled_image.astype(np.uint8)) plt.axis('off') plt.show() images.append(styled_image.astype(np.uint8)) return images, best
def stylize_c(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, prev_style_image, prev_content_image, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1, ) + content.shape style_shapes = [(1, ) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: #noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) noise = content initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + ( tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} #content_layers_weights['conv2_2'] = content_weight_blend content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append( content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss(net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce( tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:, 1:, :, :]) tv_x_size = _tensor_size(image[:, :, 1:, :]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) / tv_y_size) + (tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) / tv_x_size)) #Continuity Loss #K = np.array([[1/256,4/256,6/256,4/256,1/256], #[4/256,16/256,24/256,16/256,4/256], #[6/256,24/256,36/256,24/256,6/256], #[4/256,16/256,24/256,16/256,4/256], #[1/256,4/256,6/256,4/256,1/256]], dtype=np.float32) #G_filt = np.zeros([5,5,3],dtype=np.float32) #G_filt[:,:,0] = K #G_filt[:,:,1] = K #G_filt[:,:,2] = K #filterG = tf.convert_to_tensor(G_filt,dtype=tf.float32) #filterG = tf.reshape(filterG, [5,5,3,1]) #G_filt = tf.reshape(K, [5,5,1,1], name='G_filt') #G_filt = tf.convert_to_tensor(K, dtype=tf.float32) #tf.expand_dims(G_filt,0) #tf.expand_dims(G_filt,0) tf_org_img = tf.convert_to_tensor(content, dtype=tf.float32) tf_org_img = tf.reshape(tf_org_img, tf.shape(image)) tf_prev_img = tf.convert_to_tensor(prev_content_image, dtype=tf.float32) tf_prev_img = tf.reshape(tf_prev_img, tf.shape(image)) tf_prev_styl = tf.convert_to_tensor(prev_style_image, dtype=tf.float32) tf_prev_styl = tf.reshape(tf_prev_styl, tf.shape(image)) #smth_org_frame_diff = tf.nn.conv2d(tf_org_img - tf_prev_img,filterG,strides=[1, 1, 1, 1],padding='VALID') #smth_styl_frame_diff = tf.nn.conv2d(image - tf_prev_styl,filterG,strides=[1, 1, 1, 1],padding='VALID') #org_frame_diff = tf.norm(smth_org_frame_diff) #styl_frame_diff = tf.norm(smth_styl_frame_diff) org_frame_diff = tf.norm(tf_org_img - tf_prev_img) styl_frame_diff = tf.norm(tf_prev_styl - image) hyperparam_cl = 10e4 cl_loss = tf.multiply( hyperparam_cl, tf.divide(styl_frame_diff, org_frame_diff + 3 * content.shape[0] * content.shape[1])) # overall loss loss = content_loss + style_loss + cl_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' Continuity loss: %g\n' % cl_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() iteration_times = [] start = time.time() for i in range(iterations): iteration_start = time.time() if i > 0: elapsed = time.time() - start # take average of last couple steps to get time per iteration remaining = np.mean( iteration_times[-10:]) * (iterations - i) stderr.write( 'Iteration %4d/%4d (%s elapsed, %s remaining)\n' % (i + 1, iterations, hms(elapsed), hms(remaining))) else: stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array( Image.fromarray( styled_grayscale_rgb.astype( np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array( Image.fromarray(original_image.astype( np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array( Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ((None if last_step else i), img_out) iteration_end = time.time() iteration_times.append(iteration_end - iteration_start)
def main(): # This will print all array values in full np.set_printoptions(threshold=np.nan) parser = build_parser() options = parser.parse_args() if not os.path.isfile(options.network): parser.error( "Network %s does not exist. (Did you forget to download it?)" % options.network) # Load the vgg weights in advance vgg_weights, vgg_mean_pixel = vgg.load_net(options.network) content_image = imread(options.content) # Jacob: moved this here since the same image features will be used for each style image content_features = {} g = tf.Graph() shape = (1, ) + content_image.shape with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, options.pooling) content_pre = np.array([vgg.preprocess(content_image, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval( feed_dict={image: content_pre}) print("READY") sys.stdout.flush( ) # Make sure Java can sense this output before Python blocks waiting for input count = 0 #for style in style_images: # loop through separate style inputs individually for line in sys.stdin: # Assumes a single line of input will be a json for one image style = jsonimread(line) width = options.width if width is not None: new_shape = (int( math.floor( float(content_image.shape[0]) / content_image.shape[1] * width)), width) content_image = scipy.misc.imresize(content_image, new_shape) target_shape = content_image.shape # This batch of code was in a loop for each style input before style_scale = STYLE_SCALE if options.style_scales is not None: style_scale = options.style_scales[i] style = scipy.misc.imresize( style, style_scale * target_shape[1] / style.shape[1]) # Removed code for blanding between multiple styles style_blend_weights = [1.0] initial = options.initial if initial is not None: initial = scipy.misc.imresize(imread(initial), content_image.shape[:2]) # Initial guess is specified, but not noiseblend - no noise should be blended if options.initial_noiseblend is None: options.initial_noiseblend = 0.0 else: # Neither inital, nor noiseblend is provided, falling back to random generated initial guess if options.initial_noiseblend is None: options.initial_noiseblend = 1.0 if options.initial_noiseblend < 1.0: initial = content_image if options.checkpoint_output and "%s" not in options.checkpoint_output: parser.error("To save intermediate images, the checkpoint output " "parameter must contain `%s` (e.g. `foo%s.jpg`)") for iteration, image in stylize( network=options.network, initial=initial, initial_noiseblend=options.initial_noiseblend, content=content_image, styles=[style ], # Changed this to be a list of only one style image preserve_colors=options.preserve_colors, iterations=options.iterations, content_weight=options.content_weight, content_weight_blend=options.content_weight_blend, style_weight=options.style_weight, style_layer_weight_exp=options.style_layer_weight_exp, style_blend_weights=style_blend_weights, tv_weight=options.tv_weight, learning_rate=options.learning_rate, beta1=options.beta1, beta2=options.beta2, epsilon=options.epsilon, pooling=options.pooling, print_iterations=options.print_iterations, checkpoint_iterations=options.checkpoint_iterations, # These vgg settings are now loaded only once vgg_weights=vgg_weights, vgg_mean_pixel=vgg_mean_pixel, content_features=content_features): output_file = None combined_rgb = image if iteration is not None: if options.checkpoint_output: output_file = options.checkpoint_output % iteration else: # Change final output files to simply be numbered output_file = "%d.JPG" % count count = count + 1 if output_file: # No longer save image to file #imsave(output_file, combined_rgb) # Output json String print(json.dumps(combined_rgb.tolist())) sys.stdout.flush( ) # Make sure Java can sense this output before Python blocks waiting for input print("DONE")
def model(self): vgg_mean_pixel = tf.cast(self.vgg_mean_pixel, tf.float32) fake_y = self.G(self.x_image) fake_x = self.F(self.y_image) # cycle loss cycle_loss = self.cycle_consistency_loss(self.G, self.F, self.x_image, self.y_image, fake_x, fake_y) # ink_loss #ink_loss = self.discriminator_loss(self.D_Y, self.y_image, fake_y, gan="ink_loss") # identity loss #id_loss = self.cycle_consistency_loss(self.G, self.F, self.x_image, self.y_image, self.y_image, self.x_image) pre_x_image = tf.subtract(self.x_image, vgg_mean_pixel) vgg_x = vgg.net_preloaded(self.vgg_weights, pre_x_image, max) pre_fake_y_image = tf.subtract(self.fake_y, vgg_mean_pixel) vgg_fake_y = vgg.net_preloaded(self.vgg_weights, pre_fake_y_image, max) pre_y_image = tf.subtract(self.y_image, vgg_mean_pixel) vgg_y = vgg.net_preloaded(self.vgg_weights, pre_y_image, max) pre_fake_x_image = tf.subtract(self.fake_x, vgg_mean_pixel) vgg_fake_x = vgg.net_preloaded(self.vgg_weights, pre_fake_x_image, max) # content loss index = CONTENT_LAYERS content_loss = self.content_loss(vgg_x[index], vgg_y[index], vgg_fake_x[index], vgg_fake_y[index]) # X -> Y G_gan_loss = self.generator_loss(self.D_Y, fake_y, gan=cfg.gan) G_loss = G_gan_loss + cycle_loss + content_loss D_Y_loss = self.discriminator_loss(self.D_Y, self.y_image, self.fake_y, gan=cfg.gan) # Y -> X F_gan_loss = self.generator_loss(self.D_X, fake_x, gan=cfg.gan) F_loss = F_gan_loss + cycle_loss + content_loss D_X_loss = self.discriminator_loss(self.D_X, self.x_image, self.fake_x, gan=cfg.gan) # summary tf.summary.histogram('D_Y/true', tf.reduce_mean(self.D_Y(self.y_image))) tf.summary.histogram('D_Y/fake', tf.reduce_mean(self.D_Y(self.G(self.x_image)))) tf.summary.histogram('D_X/true', tf.reduce_mean(self.D_X(self.x_image))) tf.summary.histogram('D_X/fake', tf.reduce_mean(self.D_X(self.F(self.y_image)))) tf.summary.scalar('loss/G', G_gan_loss) tf.summary.scalar('loss/D_Y', D_Y_loss) tf.summary.scalar('loss/F', F_gan_loss) tf.summary.scalar('loss/D_X', D_X_loss) tf.summary.scalar('loss/cycle', cycle_loss) #tf.summary.scalar('loss/ink', ink_loss) #tf.summary.scalar('loss/id', id_loss) tf.summary.scalar('loss/id', content_loss) x_generate = fake_y x_reconstruct = self.F(fake_y) y_generate = fake_x y_reconstruct = self.G(fake_x) tf.summary.scalar('debug/real_x_mean', tf.reduce_mean(self.x_image)) tf.summary.scalar('debug/fake_x_mean', tf.reduce_mean(y_generate)) tf.summary.scalar('debug/real_y_mean', tf.reduce_mean(self.y_image)) tf.summary.scalar('debug/fake_y_mean', tf.reduce_mean(x_generate)) tf.summary.image('X/input', utils.batch_convert2int(self.x_image[:, :, :, :3])) tf.summary.image('X/generated', utils.batch_convert2int(x_generate)) tf.summary.image('X/reconstruction', utils.batch_convert2int(x_reconstruct[:, :, :, :3])) tf.summary.image('Y/input', utils.batch_convert2int(self.y_image)) tf.summary.image('Y/generated', utils.batch_convert2int(y_generate[:, :, :, :3])) tf.summary.image('Y/reconstruction', utils.batch_convert2int(y_reconstruct)) return G_loss, D_Y_loss, F_loss, D_X_loss, fake_y, fake_x
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None): """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Otherwise tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ # The shape information in the comment is based on the content image 1-content.jpg with shape (533, 400, 3) # and 1-style.jpg (316, 400, 3) # This should be changed with different images. shape = (1,) + content.shape # (1, 533, 400, 3) style_shapes = [(1,) + style.shape for style in styles] # (1, 316, 400, 3) content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) # Load the VGG-19 model. layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight # {'relu1_1': 1.0, 'relu2_1': 1.0, 'relu3_1': 1.0, 'relu4_1': 1.0, 'relu5_1': 1.0} layer_weight *= style_layer_weight_exp # 1.0 # VGG19 layers: # 'conv1_1', 'relu1_1', 'conv1_2', 'relu1_2', 'pool1', # 'conv2_1', 'relu2_1', 'conv2_2', 'relu2_2', 'pool2', # 'conv3_1', 'relu3_1', 'conv3_2', 'relu3_2', 'conv3_3', 'relu3_3', 'conv3_4', 'relu3_4', 'pool3', # 'conv4_1', 'relu4_1', 'conv4_2', 'relu4_2', 'conv4_3', 'relu4_3', 'conv4_4', 'relu4_4', 'pool4', # 'conv5_1', 'relu5_1', 'conv5_2', 'relu5_2', 'conv5_3', 'relu5_3', 'conv5_4', 'relu5_4' # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: # ('relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1') layer_weights_sum += style_layers_weights[style_layer] # 5.0 for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # {'relu1_1': 0.2, 'relu2_1': 0.2, 'relu3_1': 0.2, 'relu4_1': 0.2, 'relu5_1': 0.2} # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) # {'conv1_1': Tensor..., relu1_1: Tensor...} content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) # (1, 533, 400, 3) subtract with the mean pixel for layer in CONTENT_LAYERS: # (relu4_2, relu5_2) content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # Find the feature values for (relu4_2, relu5_2) # compute style features in feed forward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) # (1, 316, 400, 3) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: # # ('relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1') features = net[layer].eval(feed_dict={image: style_pre}) # For relu1_1 layer (1, 316, 400, 64) features = np.reshape(features, (-1, features.shape[3])) # (126400, 64) gram = np.matmul(features.T, features) / features.size # (64, 64) Gram matrix - measure the dependency of features. style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # 0 # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) # Generate a random image with SD the same as the content image. initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: # {'relu5_2'} # Use MSE as content losses content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] # For relu1_1: (1, 533, 400, 64) _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) # (213200, 64) gram = tf.matmul(tf.transpose(feats), feats) / size # Gram matrix for the features in relu1_1 for the result image. style_gram = style_features[i][style_layer] # Gram matrix for the style # Style loss is the MSE for the difference of the 2 Gram matrix style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # Total variation denoising: Add cost to penalize neighboring pixel is very different. # This help to reduce noise. tv_y_size = _tensor_size(image[:,1:,:,:]) tv_x_size = _tensor_size(image[:,:,1:,:]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( (None if last_step else i), img_out )
def stylize(network_path='imagenet-vgg-very0.001p-19.mat', content, styles, iterations=1000, content_weight=5e0, content_weight_blend=1, style_weight=5e2, style_layer_weight_exp=1, style_blend_weights=None, tv_weight=1e2, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-08, pooling='avg', print_iterations=100, checkpoint_iterations=100, checkpoint_path=None, output_path=None): """ This is a function to stylelize images, given the content image, list of style images, path to the network and all the hypter parameters. Returns ------- stylized_img : np.ndarray N x H x W x C image. """ # calculate the shape of the network input tensor according to the content image shape = (1,) + content.shape style_shapes = [(1,) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network_path) # scale the importance of each sytle layers according to their depth. (deeper layers are more important if style_layers_weights > 1 (default = 1)) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features of the content image by feeding it into the network @TODO why put graph on cpu?, what is the high level idea of content_features? g = tf.Graph() with g.as_default(), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # compute style features of the content image by feeding it into the network for i in range(len(styles)): g = tf.Graph() with g.as_default(), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram # make stylized image using backpropogation # if the users doesn't specify a input image, start with noise # @TODO where does the number 0.256 come from? with tf.Graph().as_default(): initial = tf.random_normal(shape) * 0.256 image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss, we can adjust the weight of each CONTENT_LAYERS content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # We can specify different weight for different style images if style_blend_weights is None: # default is equal weights style_blend_weights = [1.0/len(style_images) for _ in style_images] else: total_blend_weight = sum(style_blend_weights) # normalization style_blend_weights = [weight/total_blend_weight for weight in style_blend_weights] # style loss style_loss = 0 # iterate to calculate style lose with multiple style images for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # total variation denoising, according to the paper # Mahendran, Aravindh, and Andrea Vedaldi. "Understanding deep image representations by inverting them." # Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition. 2015. tv_y_size = _tensor_size(image[:,1:,:,:]) tv_x_size = _tensor_size(image[:,:,1:,:]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) # yield ( # (None if last_step else i), # img_out # ) output_file = None if not last_step: if checkpoint_path: output_file = checkpoint_path % iteration else: output_file = output_path if output_file: imsave(output_file, image)
def main(): '''Search for similar images Search the style directory for images that closely resemble each image in the content directory. Save those images in an output directory folder corresponding to each content image, renamed as their matching rank number. ''' parser = build_parser() options = parser.parse_args() content_files = os.listdir(options.content_dir) content_images = [ read_img(os.path.join(options.content_dir, f)) for f in content_files ] # n_content by n_style matrix and list to store the best style images n_content = len(content_files) n_total = n_content * options.n_style best_style_score = np.float('inf') * np.ones((n_content, options.n_style)) best_style_file = np.array([['' for i in range(options.n_style)] for h in range(n_content)], dtype=object) vgg_weights, vgg_mean_pixel = vgg.load_net(options.network) content_features = [{} for _ in content_images] for i, c in enumerate(content_images): with tf.Graph().as_default(), tf.Session() as sess: image = tf.placeholder('float', shape=(1, ) + c.shape) net = vgg.net_preloaded(vgg_weights, image, 'max') content_pre = np.array([vgg.preprocess(c, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[i][layer] = net[layer].eval( feed_dict={image: content_pre}) final_style_score, final_style_file = search_dir( content_features, vgg_weights, vgg_mean_pixel, best_style_score, best_style_file, options.style_dir, options.recurse, options.n_search) if np.any(np.isinf(final_style_score)): inf_total = np.sum(np.isinf(final_style_score)) print('%d out of %d style images not found.' % (inf_total, n_total), 'Try rerunning with a smaller n-style.') raise sorted_files = final_style_file[np.indices( (n_content, options.n_style))[0], final_style_score.argsort()] format_str = '{0:0>%d}.{1}' % np.ceil(np.log10(n_total)) os.mkdir(options.output_dir) for i, f in enumerate(content_files): fname = ''.join(f.split('.')[:-1]) print('Copying style files for %s' % fname) os.mkdir(os.path.join(options.output_dir, fname)) for j in range(options.n_style): print(sorted_files[i, j]) img_ext = sorted_files[i, j].split('.')[-1] shutil.copy( sorted_files[i, j], os.path.join(options.output_dir, fname, format_str.format(j, img_ext)))
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, exp_sigma, mat_sigma, mat_rho, text_to_print, print_iterations=None, checkpoint_iterations=None, kernel=3, d=2, gamma_rho=1, gamma=1, rational_rho=1, alpha=1): tf.logging.set_verbosity(tf.logging.INFO) """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] 0 - dot product kernel 1 - exponential kernel 2 - matern kernel 3 - polynomial kernel """ shape = (1,) + content.shape style_shapes = [(1,) + style.shape for style in styles] content_features = {} style_features = [{} for _ in styles] vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # compute content features in feedforward mode g = tf.Graph() with g.as_default(), g.device('/cpu'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) # sqr = features.T*features.T # dim = features.shape if(kernel == 0): gram2 = np.matmul(features.T, features) / features.size elif(kernel == 1): gram2 = gramSquaredExp_np(features, exp_sigma) / features.size # exponential kernal elif(kernel == 2): gram2 = gramMatten_np(features, mat_sigma, v, mat_rho) / features.size # Mattern kernal elif(kernel == 3): print(d) gram2 = gramPoly_np(features, C=0, d=d) / features.size elif(kernel == 4): gram2 = gramGammaExp_np(features, gamma_rho, gamma) / features.size elif(kernel == 4): gram2 = gramRatioanlQuad_np(features, rational_rho, alpha) / features.size # print(features.shape,"diamention of feature\n") style_features[i][layer] = gram2 initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation g = tf.Graph() with g.as_default(), g.device('/gpu'): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) style_gram = style_features[i][style_layer] dim = feats.get_shape() # print(dim) sqr = tf.reduce_sum(tf.transpose(feats) * tf.transpose(feats), axis=1) if(kernel == 0): gram = (tf.matmul(tf.transpose(feats), feats)) / size elif(kernel == 1): gram = tf.exp(-1 * (tf.transpose(tf.ones([dim[1], dim[1]]) * sqr) + tf.ones([dim[1], dim[1]]) * sqr - 2 * tf.matmul(tf.transpose(feats), feats)) / 2 / (exp_sigma * exp_sigma)) / size # exponetial kernal elif(kernel == 2): # mattern kernal d2 = tf.nn.relu(tf.transpose(tf.ones([dim[1], dim[1]]) * sqr) + tf.ones([dim[1], dim[1]]) * sqr - 2 * tf.matmul(tf.transpose(feats), feats)) if(v == 0.5): gram = mat_sigma**2 * tf.exp(-1 * tf.sqrt(d2) / mat_rho) / size elif(v == 1.5): gram = mat_sigma**2 * (tf.ones([dim[1], dim[1]]) + tf.sqrt(3.0) * tf.sqrt(d2) / mat_rho) * tf.exp(-1 * tf.sqrt(3.0) * tf.sqrt(d2) / mat_rho) / size elif(v == 2.5): gram = mat_sigma**2 * (tf.ones([dim[1], dim[1]]) + tf.sqrt(5.0) * tf.sqrt(d2) / mat_rho + 5 * d2 / 3 / (mat_rho**2)) * tf.exp(-1 * tf.sqrt(5.0) * tf.sqrt(d2) / mat_rho) / size elif(kernel == 3): # polynomial kernal gram = (tf.matmul(tf.transpose(feats), feats))**d / size elif(kernel == 4): # gamma exponental kernal gram = tf.exp(-1 * (tf.sqrt(d2) / gamma_rho)**gamma) / size elif(kernel == 5): # gamma exponental kernal gram = (1 + (d2 / rational_rho**2 / 2 / alpha))**(-1 * alpha) / size style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:, 1:, :, :]) tv_x_size = _tensor_size(image[:, :, 1:, :]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:, 1:, :, :] - image[:, :shape[1] - 1, :, :]) / tv_y_size) + (tf.nn.l2_loss(image[:, :, 1:, :] - image[:, :, :shape[2] - 1, :]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup # train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss) train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(last_loss): new_loss = loss.eval() stderr.write('file ===> %s \n' % text_to_print) stderr.write(' content loss: %1.3e \t' % content_loss.eval()) stderr.write(' style loss: %1.3e \t' % style_loss.eval()) stderr.write(' tv loss: %1.3e \t' % tv_loss.eval()) stderr.write(' total loss: %1.3e \t' % new_loss) stderr.write(' loss difference: %1.3e \t\n' % (last_loss - new_loss)) return new_loss def save_progress(): dict = {"content loss": content_loss.eval(), "style loss": style_loss.eval(), "tv loss": tv_loss.eval(), "total loss": loss.eval()} return dict # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') new_loss = 0 # if (print_iterations and print_iterations != 0): # print_progress() for i in range(iterations): train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) new_loss = print_progress(new_loss) if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: dict = save_progress() this_loss = loss.eval() print(this_loss, "loss in each check point") if this_loss < best_loss: best_loss = this_loss best = image.eval() try: img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) except: print("uanlabe to result image due to given parameters") img_out = "no image" if preserve_colors and preserve_colors: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( (None if last_step else i), img_out, dict )
def _vggnet(data_path, input_image): weights, mean_pixel = vgg.load_net(data_path) image = vgg.preprocess(input_image, mean_pixel) net = vgg.net_preloaded(weights, image) return net
def stylize(network, initial, initial_noiseblend, content, styles, preserve_colors, iterations, content_weight, content_weight_blend, style_weight, style_layer_weight_exp, style_blend_weights, tv_weight, learning_rate, beta1, beta2, epsilon, pooling, print_iterations=None, checkpoint_iterations=None, vgg_weights=None, vgg_mean_pixel=None, # Added so that they are no reloaded every time content_features=None): # Added so that they are not recomputed every time """ Stylize images. This function yields tuples (iteration, image); `iteration` is None if this is the final image (the last iteration). Other tuples are yielded every `checkpoint_iterations` iterations. :rtype: iterator[tuple[int|None,image]] """ shape = (1,) + content.shape style_shapes = [(1,) + style.shape for style in styles] style_features = [{} for _ in styles] # Added option to have the net pre-loaded before calling the method if vgg_weights is None or vgg_mean_pixel is None: vgg_weights, vgg_mean_pixel = vgg.load_net(network) layer_weight = 1.0 style_layers_weights = {} for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] = layer_weight layer_weight *= style_layer_weight_exp # normalize style layer weights layer_weights_sum = 0 for style_layer in STYLE_LAYERS: layer_weights_sum += style_layers_weights[style_layer] for style_layer in STYLE_LAYERS: style_layers_weights[style_layer] /= layer_weights_sum # Jacob: These content features only need to be computed once, and can be reused for # each new style image. # compute content features in feedforward mode if content_features is None: content_features = {} g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, pooling) content_pre = np.array([vgg.preprocess(content, vgg_mean_pixel)]) for layer in CONTENT_LAYERS: content_features[layer] = net[layer].eval(feed_dict={image: content_pre}) # compute style features in feedforward mode for i in range(len(styles)): g = tf.Graph() with g.as_default(), g.device('/cpu:0'), tf.Session() as sess: image = tf.placeholder('float', shape=style_shapes[i]) net = vgg.net_preloaded(vgg_weights, image, pooling) style_pre = np.array([vgg.preprocess(styles[i], vgg_mean_pixel)]) for layer in STYLE_LAYERS: features = net[layer].eval(feed_dict={image: style_pre}) features = np.reshape(features, (-1, features.shape[3])) gram = np.matmul(features.T, features) / features.size style_features[i][layer] = gram initial_content_noise_coeff = 1.0 - initial_noiseblend # make stylized image using backpropogation with tf.Graph().as_default(): if initial is None: noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = tf.random_normal(shape) * 0.256 else: initial = np.array([vgg.preprocess(initial, vgg_mean_pixel)]) initial = initial.astype('float32') noise = np.random.normal(size=shape, scale=np.std(content) * 0.1) initial = (initial) * initial_content_noise_coeff + (tf.random_normal(shape) * 0.256) * (1.0 - initial_content_noise_coeff) image = tf.Variable(initial) net = vgg.net_preloaded(vgg_weights, image, pooling) # content loss content_layers_weights = {} content_layers_weights['relu4_2'] = content_weight_blend content_layers_weights['relu5_2'] = 1.0 - content_weight_blend content_loss = 0 content_losses = [] for content_layer in CONTENT_LAYERS: content_losses.append(content_layers_weights[content_layer] * content_weight * (2 * tf.nn.l2_loss( net[content_layer] - content_features[content_layer]) / content_features[content_layer].size)) content_loss += reduce(tf.add, content_losses) # style loss style_loss = 0 for i in range(len(styles)): style_losses = [] for style_layer in STYLE_LAYERS: layer = net[style_layer] _, height, width, number = map(lambda i: i.value, layer.get_shape()) size = height * width * number feats = tf.reshape(layer, (-1, number)) gram = tf.matmul(tf.transpose(feats), feats) / size style_gram = style_features[i][style_layer] style_losses.append(style_layers_weights[style_layer] * 2 * tf.nn.l2_loss(gram - style_gram) / style_gram.size) style_loss += style_weight * style_blend_weights[i] * reduce(tf.add, style_losses) # total variation denoising tv_y_size = _tensor_size(image[:,1:,:,:]) tv_x_size = _tensor_size(image[:,:,1:,:]) tv_loss = tv_weight * 2 * ( (tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:]) / tv_y_size) + (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / tv_x_size)) # overall loss loss = content_loss + style_loss + tv_loss # optimizer setup train_step = tf.train.AdamOptimizer(learning_rate, beta1, beta2, epsilon).minimize(loss) def print_progress(): stderr.write(' content loss: %g\n' % content_loss.eval()) stderr.write(' style loss: %g\n' % style_loss.eval()) stderr.write(' tv loss: %g\n' % tv_loss.eval()) stderr.write(' total loss: %g\n' % loss.eval()) # optimization best_loss = float('inf') best = None with tf.Session() as sess: sess.run(tf.global_variables_initializer()) stderr.write('Optimization started...\n') if (print_iterations and print_iterations != 0): print_progress() for i in range(iterations): stderr.write('Iteration %4d/%4d\n' % (i + 1, iterations)) train_step.run() last_step = (i == iterations - 1) if last_step or (print_iterations and i % print_iterations == 0): print_progress() if (checkpoint_iterations and i % checkpoint_iterations == 0) or last_step: this_loss = loss.eval() if this_loss < best_loss: best_loss = this_loss best = image.eval() img_out = vgg.unprocess(best.reshape(shape[1:]), vgg_mean_pixel) if preserve_colors and preserve_colors == True: original_image = np.clip(content, 0, 255) styled_image = np.clip(img_out, 0, 255) # Luminosity transfer steps: # 1. Convert stylized RGB->grayscale accoriding to Rec.601 luma (0.299, 0.587, 0.114) # 2. Convert stylized grayscale into YUV (YCbCr) # 3. Convert original image into YUV (YCbCr) # 4. Recombine (stylizedYUV.Y, originalYUV.U, originalYUV.V) # 5. Convert recombined image from YUV back to RGB # 1 styled_grayscale = rgb2gray(styled_image) styled_grayscale_rgb = gray2rgb(styled_grayscale) # 2 styled_grayscale_yuv = np.array(Image.fromarray(styled_grayscale_rgb.astype(np.uint8)).convert('YCbCr')) # 3 original_yuv = np.array(Image.fromarray(original_image.astype(np.uint8)).convert('YCbCr')) # 4 w, h, _ = original_image.shape combined_yuv = np.empty((w, h, 3), dtype=np.uint8) combined_yuv[..., 0] = styled_grayscale_yuv[..., 0] combined_yuv[..., 1] = original_yuv[..., 1] combined_yuv[..., 2] = original_yuv[..., 2] # 5 img_out = np.array(Image.fromarray(combined_yuv, 'YCbCr').convert('RGB')) yield ( (None if last_step else i), img_out )
#TODO: load VGG and extract feature VGG_PATH = 'imagenet-vgg-verydeep-19.mat' vgg_weights, vgg_mean_pixel = vgg.load_net(VGG_PATH) CONTENT_LAYERS = ('relu3_1', 'relu3_2', 'relu4_1', 'relu5_1', 'relu5_2') layer = 'relu5_2' input_image = cv2.imread( "E:\\image_cluster\\continue\\days_2_resize\\2_300001_2016-02-02_2016-02-03_1.png" ) feature_data = [] shape = (1, ) + input_image.shape g = tf.Graph() with g.as_default(), tf.Session() as sess: image = tf.placeholder('float', shape=shape) net = vgg.net_preloaded(vgg_weights, image, 'avg') for name in os.listdir(root): filename = os.path.join(root, name) input_image = cv2.imread(filename) # content_features = {} # for i in range(len(image_list)): # input_image=image_list[i] content_pre = np.array([vgg.preprocess(input_image, vgg_mean_pixel)]) content_features = net[layer].eval(feed_dict={image: content_pre})[0] feature_data.append(content_features.flatten()) #TODO: kmeans and save images with predict folder kmeans = KMeans(n_clusters=10, random_state=0).fit(feature_data) predict = kmeans.predict(feature_data)