def build(self, x, reuse=None):
""" TODO: define your model (2 conv layers and 2 fc layers?)
x: input image
logit: network output w/o softmax """
with tf.variable_scope('model', reuse=reuse):
W1 = tf.Variable(tf.random_normal([3, 3, 1, 32], stddev=0.01))
L1 = relu(conv2d(x, W1, strides=[1, 1, 1, 1], padding='SAME'))
L1 = max_pool(L1, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
W2 = tf.Variable(tf.random_normal([3, 3, 32, 64], stddev=0.01))
L2 = relu(conv2d(L1, W2, strides=[1, 1, 1, 1], padding='SAME'))
L2 = max_pool(L2, ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1], padding='SAME')
L2_flat = tf.reshape(L2, [-1, 7 * 7 * 64])
W3 = tf.get_variable("W3", shape=[7 * 7 * 64, 10],
initializer=xavier_initializer())
b = tf.Variable(tf.random_normal([10]))
logit = tf.matmul(L2_flat, W3) + b
return logit
def implement(self, x):
W = weight([
self.size[0], self.size[1],
x.get_shape().as_list()[3], self.features
])
b = bias([self.features])
return nn.conv2d(x, W, strides=self.strides, padding=self.padding) + b
def call(self, input):
masked_kernel = tf.math.multiply(self.mask, self.kernel)
x = nn.conv2d(input,
masked_kernel,
strides=[1, self.strides, self.strides, 1],
padding=self.padding)
x = nn.bias_add(x, self.bias)
return x
def build_net(sess):
in_len = 32
in_dep = 1
x_hold = tf.placeholder(tf.float32,shape=[None,in_dep*in_len*in_len])
y_hold = tf.placeholder(tf.float32,shape=[None,2])
keep_prob = tf.placeholder(tf.float32)
xt = tf.reshape(x_hold,[-1,in_len,in_len,in_dep])
#Layer 1 - 5x5 convolution
w1 = tfac.weight([5,5,in_dep,4])
b1 = tfac.bias([4])
c1 = nn.relu(nn.conv2d(xt,w1,strides=[1,2,2,1],padding='VALID')+b1)
o1 = c1
#Layer 2 - 3x3 convolution
w2 = tfac.weight([3,3,4,16])
b2 = tfac.bias([16])
c2 = nn.relu(nn.conv2d(o1,w2,strides=[1,2,2,1],padding='VALID')+b2)
o2 = c2
#Layer 3 - 3x3 convolution
w3 = tfac.weight([3,3,16,32])
b3 = tfac.bias([32])
c3 = nn.relu(nn.conv2d(o2,w3,strides=[1,1,1,1],padding='VALID')+b3)
o3 = c3
dim = 32 * 4*4
#Fully connected layer - 600 units
of = tf.reshape(o3,[-1,dim])
w4 = tfac.weight([dim,600])
b4 = tfac.bias([600])
o4 = nn.relu(tf.matmul(of,w4)+b4)
o4 = nn.dropout(o4, keep_prob)
#Output softmax layer - 2 units
w5 = tfac.weight([600,2])
b5 = tfac.bias([2])
y = nn.softmax(tf.matmul(o4,w5)+b5)
sess.run(tf.initialize_all_variables())
return y,x_hold,y_hold,keep_prob
def ConvReluMaxPool(X, weights, bias):
conv = nn.conv2d(X, weights, strides=[1, 1, 1, 1], padding="VALID", data_format="NHWC")
conv_bias = nn.bias_add(conv, bias, data_format="NHWC")
relu = nn.relu(conv_bias)
maxpool = nn.max_pool2d(relu, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding="VALID", data_format="NHWC")
return maxpool
def Convolution(X, weights, bias):
conv = nn.conv2d(X,
weights,
strides=[1, 1, 1, 1],
padding="VALID",
data_format="NHWC")
conv_bias = nn.bias_add(conv, bias, data_format="NHWC")
return conv_bias
def convolution_layer(data, nChannels, filter_size, nFilters):
kernelShape = [filter_size, filter_size, nChannels, nFilters]
weights = tensorflow.Variable(
tensorflow.truncated_normal(kernelShape, stddev=0.05))
biases = tensorflow.Variable(tensorflow.constant(0.05, shape=[nFilters]))
layer = nn.conv2d(data, weights, [1, 1, 1, 1], 'SAME')
layer = nn.bias_add(layer, biases)
layer = nn.max_pool(layer, [1, 2, 2, 1], [1, 2, 2, 1], 'SAME')
layer = nn.relu(layer)
return layer
def resnetBlock(X, weights1, bias1, weights2, bias2, mean1, variance1, offset1,
scale1, mean2, variance2, offset2, scale2):
conv1 = nn.conv2d(X,
weights1,
strides=[1, 1, 1, 1],
padding="VALID",
data_format="NCHW")
conv1_bias = nn.bias_add(conv1, bias1, data_format="NCHW")
bn1 = nn.batch_normalization(conv1_bias, mean1, variance1, offset1, scale1,
EPSILON)
relu1 = nn.relu(bn1)
conv2 = nn.conv2d(relu1,
weights2,
strides=[1, 1, 1, 1],
padding="SAME",
data_format="NCHW")
conv2_bias = nn.bias_add(conv2, bias2, data_format="NCHW")
bn2 = nn.batch_normalization(conv2_bias, mean2, variance2, offset2, scale2,
EPSILON)
return bn2
def vggBlock(X, weights1, bias1, weights2, bias2):
conv1 = nn.conv2d(X,
weights1,
strides=[1, 1, 1, 1],
padding="VALID",
data_format="NCHW")
conv1_bias = nn.bias_add(conv1, bias1, data_format="NCHW")
relu1 = nn.relu(conv1_bias)
conv2 = nn.conv2d(relu1,
weights2,
strides=[1, 1, 1, 1],
padding="SAME",
data_format="NCHW")
conv2_bias = nn.bias_add(conv2, bias2, data_format="NCHW")
relu2 = nn.relu(conv2_bias)
maxpool = nn.max_pool2d(relu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID",
data_format="NHWC")
return maxpool
def ResizeConvReluMaxPool(X, weights, bias):
resize = image.resize(X, [N + 2, N + 2])
conv = nn.conv2d(resize,
weights,
strides=[1, 1, 1, 1],
padding="VALID",
data_format="NHWC")
conv_bias = nn.bias_add(conv, bias, data_format="NHWC")
relu = nn.relu(conv_bias)
maxpool = nn.max_pool2d(relu,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding="VALID",
data_format="NHWC",
name="output")
return maxpool
def conv_layer(previous_layer, filters, kernel_size=(3,3),stride=2,relu=True,padding='SAME', norm='instance'):
#Get the output channel size from previous layer
channels = previous_layer.get_shape().as_list()[3]
#The shape of the output tensor for this convolutional layer
shape = [kernel_size[0], kernel_size[1],channels, filters]
#Create convolution layer
conv = conv2d(previous_layer, create_weights(shape), [1,stride,stride,1],padding=padding)
#If using instance norm..
if norm == 'instance':
normalised = tf.contrib.layers.instance_norm(conv)
#Otherwise, we use batch norm..
else:
mean,variance = tf.nn.moments(conv,[0,1,2])
normalised = batch_normalization(conv,mean,variance,None,None,0.0001)
#If we use ReLU, add ReLU layer.
if relu:
relu_layer = tf.nn.relu(normalised)
return relu_layer
return normalised
def _inception_layer(previous_layer, filters):
unfiltered_1_x_1_conv = nn.conv2d(previous_layer,
filters[0],
strides=(1, 1),
padding="SAME")
stacked_conv_1_x_1_conv_1 = nn.conv2d(previous_layer,
filters=filters[1],
strides=(1, 1),
padding="SAME")
stacked_conv_1_x_1_conv_2 = nn.conv2d(previous_layer,
filters=filters[2],
strides=(1, 1),
padding="SAME")
stacked_conv_3_x_3_conv = nn.conv2d(stacked_conv_1_x_1_conv_1,
filters=filters[3],
strides=(1, 1),
padding="SAME")
stacked_conv_5_x_5_conv = nn.conv2d(stacked_conv_1_x_1_conv_2,
filters=filters[4],
strides=(1, 1),
padding="SAME")
pooled_layer = nn.max_pool2d(previous_layer, [3, 3],
strides=(1, 1),
padding="SAME")
pooled_layer_conv = nn.conv2d(pooled_layer,
filters=filters[5],
strides=(1, 1),
padding="SAME")
return tf.concat([
unfiltered_1_x_1_conv, stacked_conv_3_x_3_conv,
stacked_conv_5_x_5_conv, pooled_layer_conv
],
axis=3)
def execute(self, inputs):
if not self.built:
raise (operationWithUnbuiltNode("execute"))
else:
myInput = concat(inputs, -1)
return conv2d(myInput, self.filter, self.strides, "VALID")
def Net(self, input, prob=0.0):
'''
Define network.
You can use init_weight() and init_bias() function to init weight matrix,
for example:
conv1_W = self.init_weight((3, 3, 1, 6))
conv1_b = self.init_bias(6)
'''
# Define the parameters
conv1_W = self.init_weight([11, 11, 3, 96])
conv1_b = self.init_bias(96)
conv2_W = self.init_weight([5, 5, 96, 256])
conv2_b = self.init_bias(256)
conv3_W = self.init_weight([3, 3, 256, 384])
conv3_b = self.init_bias(384)
conv4_W = self.init_weight([3, 3, 384, 384])
conv4_b = self.init_bias(384)
conv5_W = self.init_weight([3, 3, 384, 256])
conv5_b = self.init_bias(256)
fc1_W = self.init_weight([5 * 5 * 256, 4096])
fc1_b = self.init_bias(4096)
fc2_W = self.init_weight([4096, 4096])
fc2_b = self.init_bias(4096)
fc3_W = self.init_weight([4096, 10])
fc3_b = self.init_bias(10)
'''Define the architecture of the network'''
# N=(W-F+2P)/S+1
# Layer 1: Convolutional. Input = 224x224x3. Output = 54x54x96.
x = nn.conv2d(input, conv1_W, strides=[1, 4, 4, 1], padding='VALID') + conv1_b
x = nn.relu(x, name="conv_layer_01/relu")
# LRN. depth_radius = n/2 = 3, bias = k = 2. Following the paper.
x = nn.lrn(x, 3, bias=2.0, alpha=1e-4, beta=0.75, name="conv_layer_01/lrn1")
# Pooling. Input = 54x54x96. Output = 26x26x96.
x = nn.max_pool2d(x, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='VALID', name="conv_layer_01/pooling") # [batch, height, width, channels]
# Layer 2: Convolutional. Input = 26x26x96. Output = 26x26x256.
x = nn.conv2d(x, conv2_W, strides=[1, 1, 1, 1], padding='SAME') + conv2_b
x = nn.relu(x, name="conv_layer_02/relu")
# LRN. depth_radius = n/2 = 3, bias = k = 2. Following the paper.
x = nn.lrn(x, 3, bias=2.0, alpha=1e-4, beta=0.75, name="conv_layer_01/lrn1")
# Pooling. Input = 26x26x256. Output = 12x12x256.
x = nn.max_pool2d(x, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name="conv_layer_02/pooling")
# Layer 3: Convolutional. Input = 12x12x256. Output = 12x12x384.
x = nn.conv2d(x, conv3_W, strides=[1, 1, 1, 1], padding='SAME') + conv3_b
x = nn.relu(x, name="conv_layer_03/relu")
# Layer 4: Convolutional. Input = 12x12x384. Output = 12x12x384.
x = nn.conv2d(x, conv4_W, strides=[1, 1, 1, 1], padding='SAME') + conv4_b
x = nn.relu(x, name="conv_layer_04/relu")
# Layer 5: Convolutional. Input = 12x12x384. Output = 12x12x256.
x = nn.conv2d(x, conv5_W, strides=[1, 1, 1, 1], padding='SAME') + conv5_b
x = nn.relu(x, name="conv_layer_05/relu")
# Pooling. Input = 12x12x256. Output = 5x5x256.
x = nn.max_pool2d(x, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='VALID', name="conv_layer_05/pooling")
# Layer 6: Fully Connected. Input = 5x5x256=6400. Output = 4096.
x = flatten(x)
x = tf.matmul(x, fc1_W) + fc1_b
x = nn.relu(x, name="full_layer_01/relu")
# Dropout
x = nn.dropout(x, rate=prob)
# Layer 7: Fully Connected. Input = 4096. Output = 4096.
x = tf.matmul(x, fc2_W) + fc2_b
x = nn.relu(x, name="full_layer_02/relu")
# Dropout
x = nn.dropout(x, rate=prob)
# # Layer 8: Fully Connected. Input = 4096. Output = 10.
x = tf.add(tf.matmul(x, fc3_W), fc3_b, name="full_layer_03/linear")
logits = x # logits.shape = (batch_size, 10)
return logits
from scipy.misc import imread, imresize
from os import listdir
from os.path import splitext
from random import seed, shuffle
from time import time
from numpy import zeros
from tensorflow import Variable, truncated_normal, constant, nn
weights = lambda shape: Variable(truncated_normal(shape, stddev=0.1))
biases = lambda shape: Variable(constant(0.1, shape=shape))
conv2d = lambda x, W: nn.conv2d(x, W, strides=[1, 2, 2, 1], padding='SAME')
max_pool = lambda x: nn.max_pool(
x, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
def load_image(path, shape=False):
readed_img = imread(path)
if shape:
readed_img = imresize(readed_img, shape)
return readed_img
def extend_children(path, ftype=False):
allpaths = [path + '/' + child for child in listdir(path)]
if ftype != False:
# remember to include the period in ftype (ie .jpg)
# pass '' to include only folders
ret = []
for v in allpaths:
if splitext(v)[1] == ftype:
ret.append(v)
def conv2d(self, x, W):
return nn.conv2d(x, W, strides=[1, 2, 2, 1], padding='SAME')
def rgb2gray(rgb):
r, g, b = rgb[:, :, 0], rgb[:, :, 1], rgb[:, :, 2]
gray = 0.2989 * r + 0.5870 * g + 0.1140 * b
return gray
image = rgb2gray(image)
# In[39]:
sharpen = np.full((3, 3), -1 / 8)
sharpen[1, 1] = 1
horiz = np.zeros((3, 3))
horiz[0, :] = -1
horiz[2, :] = 1
vert = horiz.T
print(sharpen)
inp = image[None, :, :, None]
for kernel in [horiz, vert, sharpen]:
W = kernel[:, :, None, None]
out_t = nn.conv2d(inp, W, [1, 1, 1, 1], 'SAME')
out_np = out_t.eval(session=sess)
plt.imshow(out_np.squeeze(), cmap='gray')
plt.show()
# In[ ]:
def call(self, inputs, training=None):
conv = conv2d(inputs, self.w, self.strides, self.padding)
if self.use_bias:
return bias_add(conv, self.b)
return conv
size=w_shp),
dtype=input.dtype),
name='W')
# initialize shared variable for bias (1D tensor) with random values
# IMPORTANT: biases are usually initialized to zero. However in this
# particular application, we simply apply the convolutional layer to
# an image without learning the parameters. We therefore initialize
# them to random values to "simulate" learning.
b_shp = (2, )
b = tf.shared(numpy.asarray(rng.uniform(low=-.5, high=.5, size=b_shp),
dtype=input.dtype),
name='b')
# build symbolic expression that computes the convolution of input with filters in w
conv_out = conv2d(input, W)
# build symbolic expression to add bias and apply activation function, i.e. produce neural net layer output
# A few words on ``dimshuffle`` :
# ``dimshuffle`` is a powerful tool in reshaping a tensor;
# what it allows you to do is to shuffle dimension around
# but also to insert new ones along which the tensor will be
# broadcastable;
# dimshuffle('x', 2, 'x', 0, 1)
# This will work on 3d tensors with no broadcastable
# dimensions. The first dimension will be broadcastable,
# then we will have the third dimension of the input tensor as
# the second of the resulting tensor, etc. If the tensor has
# shape (20, 30, 40), the resulting tensor will have dimensions
# (1, 40, 1, 20, 30). (AxBxC tensor is mapped to 1xCx1xAxB tensor)
# More examples:
def u_net_2_inp_prep(self, model_2_output, model_1_output):
model_3_inp_1 = concat([model_2_output[0], model_1_output[0]])
model_3_inp_2 = concat([model_2_output[1], model_1_output[1]])
model_3_inp_1_final = conv2d(model_3_inp_1)
model_3_inp_2_final = conv2d(model_3_inp_2)
return concat([model_3_inp_1_final, model_3_inp_2_final, model_1_output[2], model_1_output[3], model_1_output[4]])
def __init__(self, h_size, env, name, LEARNING_RATE, n_step):
# The network recieves a frame from the game, flattened into an array.
# It then resizes it and processes it through four convolutional layers.
WINDOW_SIZE = env.win_size
CONV_FILTER_SIZE_X = [3, 3, 3, 3]
CONV_FILTER_SIZE_Y = [3, 3, 3, 3]
CONV_STRIDE_X = [3, 1, 1, 3]
CONV_STRIDE_Y = [3, 1, 1, 3]
CONV_LAYER_NUM = 4
CONV_FILTER_NUM = [8, 32, 32, 64]
IMAGE_SIZE = [2 * (WINDOW_SIZE + 2), 8, 3]
self.scalarInput = tf.placeholder(
shape=[None, IMAGE_SIZE[0] * IMAGE_SIZE[1] * IMAGE_SIZE[2]],
dtype=tf.float32)
self.imageIn = tf.reshape(
self.scalarInput,
shape=[-1, IMAGE_SIZE[0], IMAGE_SIZE[1], IMAGE_SIZE[2]])
depthwise_filter1 = tf.get_variable(shape=(CONV_FILTER_SIZE_X[0],
CONV_FILTER_SIZE_Y[0], 3,
1),
name=name + "_depthwise_filter1")
pointwise_filter1 = tf.get_variable(
shape=[1, 1, 3, CONV_FILTER_NUM[0]],
name=name + "_pointwise_filter1")
self.conv1 = nn.separable_conv2d(
self.imageIn,
depthwise_filter1,
pointwise_filter1,
strides=[1, CONV_STRIDE_X[0], CONV_STRIDE_Y[0], 1],
padding='SAME')
print(np.shape(self.conv1))
self.relu1 = nn.relu(self.conv1, name=name + "_relu1")
print(np.shape(self.relu1))
depthwise_filter2 = tf.get_variable(shape=(CONV_FILTER_SIZE_X[1],
CONV_FILTER_SIZE_Y[1],
CONV_FILTER_NUM[0], 1),
name=name + "_depthwise_filter2")
pointwise_filter2 = tf.get_variable(
shape=[1, 1, CONV_FILTER_NUM[0], CONV_FILTER_NUM[1]],
name=name + "_pointwise_filter2")
self.conv2 = nn.separable_conv2d(
self.relu1,
depthwise_filter2,
pointwise_filter2,
strides=[1, CONV_STRIDE_X[1], CONV_STRIDE_Y[1], 1],
padding='SAME')
print(np.shape(self.conv2))
self.relu2 = nn.relu(self.conv2, name=name + "_relu2")
print(np.shape(self.relu2))
depthwise_filter3 = tf.get_variable(shape=(CONV_FILTER_SIZE_X[2],
CONV_FILTER_SIZE_Y[2],
CONV_FILTER_NUM[1], 1),
name=name + "_depthwise_filter3")
pointwise_filter3 = tf.get_variable(
shape=[1, 1, CONV_FILTER_NUM[1], CONV_FILTER_NUM[2]],
name=name + "_pointwise_filter3")
self.conv3 = nn.separable_conv2d(
self.relu2,
depthwise_filter3,
pointwise_filter3,
strides=[1, CONV_STRIDE_X[2], CONV_STRIDE_Y[2], 1],
padding='SAME')
print(np.shape(self.conv3))
self.relu3 = nn.relu(self.conv3, name=name + "_relu3")
print(np.shape(self.relu3))
self.maxpool1 = nn.max_pool(self.relu3,
ksize=[1, 3, 1, 1],
strides=[1, 3, 1, 1],
padding='VALID')
print(np.shape(self.maxpool1))
if np.ceil(np.floor(np.ceil(2 * (WINDOW_SIZE + 2) / 3) / 3) / 3) >= 2:
conv_filter4 = tf.get_variable(
shape=(CONV_FILTER_SIZE_X[3], CONV_FILTER_SIZE_Y[3],
CONV_FILTER_NUM[2], CONV_FILTER_NUM[3]),
name=name + "_conv_filter4")
self.conv4 = nn.conv2d(
self.maxpool1,
conv_filter4,
strides=[1, CONV_STRIDE_X[3], CONV_STRIDE_Y[3], 1],
padding='SAME')
print(np.shape(self.conv4))
self.relu4 = nn.relu(self.conv4, name=name + "_relu4")
print(np.shape(self.relu4))
self.maxpool2 = nn.max_pool(self.relu4,
ksize=[1, 2, 1, 1],
strides=[1, 2, 1, 1],
padding='VALID')
LAST_CONV_FILTER = [
np.floor(
np.ceil(
np.floor(np.ceil(2 * (WINDOW_SIZE + 2) / 3) / 3) / 3) /
2), 1
]
conv_filter5 = tf.get_variable(shape=(LAST_CONV_FILTER[0],
LAST_CONV_FILTER[1],
CONV_FILTER_NUM[3], h_size),
name=name + "_conv_filter5")
self.conv5 = nn.conv2d(
self.maxpool2,
conv_filter5,
strides=[1, CONV_STRIDE_X[3], CONV_STRIDE_Y[3], 1],
padding='VALID')
print(np.shape(self.maxpool2))
else:
LAST_CONV_FILTER = [
np.floor(np.ceil(2 * (WINDOW_SIZE + 2) / 3) / 3), 3
]
conv_filter5 = tf.get_variable(shape=(LAST_CONV_FILTER[0],
LAST_CONV_FILTER[1],
CONV_FILTER_NUM[2], h_size),
name=name + "_conv_filter5")
self.conv5 = nn.conv2d(
self.maxpool1,
conv_filter5,
strides=[1, CONV_STRIDE_X[3], CONV_STRIDE_Y[3], 1],
padding='VALID')
print(np.shape(self.conv5))
self.relu5 = nn.relu(self.conv5, name=name + "_relu5")
print(np.shape(self.relu5))
# We take the output from the final convolutional layer and split it into separate advantage and value streams.
self.streamAC, self.streamVC = tf.split(self.relu5, 2, 3)
self.streamA = slim.flatten(self.streamAC)
self.streamV = slim.flatten(self.streamVC)
xavier_init = tf.contrib.layers.xavier_initializer()
self.AW = tf.Variable(xavier_init([h_size // 2, env.actions]))
self.VW = tf.Variable(xavier_init([h_size // 2, 1]))
print(self.conv5)
print(self.streamA)
print(self.AW)
self.Advantage = tf.matmul(self.streamA, self.AW)
self.Value = tf.matmul(self.streamV, self.VW)
# Then combine them together to get our final Q-values.
self.Qout = self.Value + tf.subtract(
self.Advantage,
tf.reduce_mean(self.Advantage, axis=1, keep_dims=True))
self.predict = tf.argmax(self.Qout, 1)
# Below we obtain the loss by taking the sum of squares difference between the target and prediction Q values.
self.targetQ = tf.placeholder(shape=[None], dtype=tf.float32)
self.actions = tf.placeholder(shape=[None], dtype=tf.int32)
self.actions_onehot = tf.one_hot(self.actions,
env.actions,
dtype=tf.float32)
self.Q = tf.reduce_sum(tf.multiply(self.Qout, self.actions_onehot),
axis=1)
self.td_error = tf.square(self.targetQ - self.Q)
self.loss = tf.reduce_mean(self.td_error)
self.trainer = tf.train.AdamOptimizer(learning_rate=LEARNING_RATE)
self.updateModel = self.trainer.minimize(self.loss)
def build_model(self):
with tf.name_scope('Input'):
content_img = tf.constant(self.content_img_value,
name='content_image')
style_img = tf.constant(self.style_img_value, name='style_image')
self.magical_img = tf.Variable(initial_value=content_img,
name='magical_image')
input_tensor = tf.concat([
tf.expand_dims(content_img, axis=0),
tf.expand_dims(style_img, axis=0),
tf.expand_dims(self.magical_img, axis=0)
],
axis=0)
with tf.name_scope('conv1'):
conv1_1 = relu(bias_add(
conv2d(input_tensor,
tf.constant(self.vgg16_weights['block1_conv1'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block1_conv1'][1]),
name='conv1_1')
conv1_2 = relu(bias_add(
conv2d(conv1_1,
tf.constant(self.vgg16_weights['block1_conv2'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block1_conv2'][1]),
name='conv1_2')
pool1 = max_pool(conv1_2, [1, 2, 2, 1], [1, 2, 2, 1],
'SAME',
name='pool1')
with tf.name_scope('conv2'):
conv2_1 = relu(bias_add(
conv2d(pool1,
tf.constant(self.vgg16_weights['block2_conv1'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block2_conv1'][1]),
name='conv2_1')
conv2_2 = relu(bias_add(
conv2d(conv2_1,
tf.constant(self.vgg16_weights['block2_conv2'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block2_conv2'][1]),
name='conv2_2')
pool2 = max_pool(conv2_2, [1, 2, 2, 1], [1, 2, 2, 1],
'SAME',
name='pool2')
with tf.name_scope('conv3'):
conv3_1 = relu(bias_add(
conv2d(pool2,
tf.constant(self.vgg16_weights['block3_conv1'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block3_conv1'][1]),
name='conv3_1')
conv3_2 = relu(bias_add(
conv2d(conv3_1,
tf.constant(self.vgg16_weights['block3_conv2'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block3_conv2'][1]),
name='conv3_2')
conv3_3 = relu(bias_add(
conv2d(conv3_2,
tf.constant(self.vgg16_weights['block3_conv3'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block3_conv3'][1]),
name='conv3_3')
pool3 = max_pool(conv3_3, [1, 2, 2, 1], [1, 2, 2, 1],
'SAME',
name='pool3')
with tf.name_scope('conv4'):
conv4_1 = relu(bias_add(
conv2d(pool3,
tf.constant(self.vgg16_weights['block4_conv1'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block4_conv1'][1]),
name='conv4_1')
conv4_2 = relu(bias_add(
conv2d(conv4_1,
tf.constant(self.vgg16_weights['block4_conv2'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block4_conv2'][1]),
name='conv4_2')
conv4_3 = relu(bias_add(
conv2d(conv4_2,
tf.constant(self.vgg16_weights['block4_conv3'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block4_conv3'][1]),
name='conv4_3')
pool4 = max_pool(conv4_3, [1, 2, 2, 1], [1, 2, 2, 1],
'SAME',
name='pool4')
with tf.name_scope('conv5'):
conv5_1 = relu(bias_add(
conv2d(pool4,
tf.constant(self.vgg16_weights['block5_conv1'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block5_conv1'][1]),
name='conv5_1')
conv5_2 = relu(bias_add(
conv2d(conv5_1,
tf.constant(self.vgg16_weights['block5_conv2'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block5_conv2'][1]),
name='conv5_2')
conv5_3 = relu(bias_add(
conv2d(conv5_2,
tf.constant(self.vgg16_weights['block5_conv3'][0]),
strides=[1, 1, 1, 1],
padding='SAME'), self.vgg16_weights['block5_conv3'][1]),
name='conv5_3')
pool5 = max_pool(conv5_3, [1, 2, 2, 1], [1, 2, 2, 1],
'SAME',
name='pool5')
c_loss = self.content_loss([input_tensor])
s_loss = self.style_loss([conv1_1, conv2_1, conv3_1, conv4_1, conv5_1])
self.loss = self.content_weight * c_loss + self.style_weight * s_loss
self.train_op = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(
self.loss)
if self.graph_write:
writer = tf.summary.FileWriter('logs',
graph=tf.get_default_graph())
writer.flush()
writer.close()
def call(self, inputs, training=None, **kwargs):
# get offset, shape [batch_size, out_h, out_w, filter_h, * filter_w * channel_out * 2]
offset = nn.conv2d(inputs,
filter=self.offset_layer_kernel,
strides=[1, *self.strides, 1],
padding=self.padding.upper(),
dilations=[1, *self.dilation_rate, 1])
# shape of offset: n_batch, feature_map_x, feature_map_y, offset_num*2
offset += self.offset_layer_bias
# add padding if needed
inputs = self._pad_input(inputs)
# some length
batch_size = K.shape(offset)[0]
channel_in = inputs.get_shape().as_list()[-1]
in_h, in_w = inputs.get_shape().as_list()[1:
3] # input feature map size
out_h, out_w = offset.get_shape().as_list()[
1:3] # output feature map size
filter_h, filter_w = self.kernel_size
# get x, y axis offset
offset = tf.reshape(offset, [batch_size, out_h, out_w, -1, 2])
y_off, x_off = offset[:, :, :, :, 0], offset[:, :, :, :, 1]
# input feature map gird coordinates
y, x = self._get_conv_indices(
[in_h, in_w]) # [1, out_h, out_w, filter_h*filter_w]
y, x = [tf.expand_dims(i, axis=-1) for i in [y, x]]
y, x = [
tf.tile(i, [batch_size, 1, 1, 1, self.num_deformable_group])
for i in [y, x]
] # make batch_dim and filter_dim equal to kernel
y, x = [
tf.reshape(i, [
K.shape(i)[0], *i.shape[1:3],
filter_h * filter_w * self.num_deformable_group
]) for i in [y, x]
]
y, x = [tf.to_float(i) for i in [y, x]]
# print(y, x)
# add offset
y, x = y + y_off, x + x_off
y = tf.clip_by_value(y, 0, in_h - 1)
x = tf.clip_by_value(x, 0, in_w - 1)
# get four coordinates of points around (x, y)
y0, x0 = [tf.to_int32(tf.floor(i)) for i in [y, x]]
y1, x1 = y0 + 1, x0 + 1
# clip
y0, y1 = [tf.clip_by_value(i, 0, in_h - 1) for i in [y0, y1]]
x0, x1 = [tf.clip_by_value(i, 0, in_w - 1) for i in [x0, x1]]
# get pixel values
indices = [[y0, x0], [y0, x1], [y1, x0], [y1, x1]]
p0, p1, p2, p3 = [
self._get_pixel_values_at_point(inputs, i) for i in indices
]
# cast to float
x0, x1, y0, y1 = [tf.to_float(i) for i in [x0, x1, y0, y1]]
# weights
w0 = (y1 - y) * (x1 - x)
w1 = (y1 - y) * (x - x0)
w2 = (y - y0) * (x1 - x)
w3 = (y - y0) * (x - x0)
# expand dim for broadcast
w0, w1, w2, w3 = [tf.expand_dims(i, axis=-1) for i in [w0, w1, w2, w3]]
# bilinear interpolation
pixels = tf.add_n([w0 * p0, w1 * p1, w2 * p2, w3 * p3])
# reshape the "big" feature map
pixels = tf.reshape(pixels, [
batch_size, out_h, out_w, filter_h, filter_w,
self.num_deformable_group, channel_in
])
pixels = tf.transpose(pixels, [0, 1, 3, 2, 4, 5, 6])
pixels = tf.reshape(pixels, [
batch_size, out_h * filter_h, out_w * filter_w,
self.num_deformable_group, channel_in
])
# copy channels to same group
feat_in_group = self.filters // self.num_deformable_group
pixels = tf.tile(pixels, [1, 1, 1, 1, feat_in_group])
#print("a", pixels)
#print(channel_in)
pixels = tf.reshape(
pixels, [batch_size, out_h * filter_h, out_w * filter_w, -1])
#print("b", pixels)
# depth-wise conv
#print(pixels)
#print(self.kernel)
out = tf.nn.depthwise_conv2d(pixels, self.kernel,
[1, filter_h, filter_w, 1], 'VALID')
# add the output feature maps in the same group
out = tf.reshape(out, [-1, out_h, out_w, self.filters, channel_in])
out = tf.reduce_sum(out, axis=-1)
if self.use_bias:
out += self.bias
return self.activation(out)
def _convolutional_layer(layer_input, layer_filter, stride):
return nn.conv2d(layer_input,
filters=layer_filter,
strides=stride,
padding="SAME")
