def deconv2d(input_,
output_shape,
k_h=5,
k_w=5,
d_h=2,
d_w=2,
stddev=0.02,
name='deconv2d',
with_w=False):
with tensorflow.variable_scope(name):
w = tensorflow.get_variable(
'w', [k_h, k_w, output_shape[-1],
input_.get_shape()[-1]],
initializer=tensorflow.random_normal_initializer(stddev=stddev))
try:
deconv = tensorflow.nn.conv2d_transpose(input_,
w,
output_shape=output_shape,
strides=[1, d_h, d_w, 1])
except AttributeError:
deconv = tensorflow.nn.deconv2d(input_,
w,
output_shape=output_shape,
strides=[1, d_h, d_w, 1])
biases = tensorflow.get_varaible(
'biases', [output_shape[-1]],
initializer=tensorflow.constant_initializer(0.0))
deconv = tensorflow.reshape(tensorflow.nn.bias_add(deconv, biases),
deconv.get_shape())
if with_w:
return deconv, w, biases
else:
return deconv
def build_generator(self, x_input, bn_train):
temp_vec = x_input
temp_dim = self.random_dim
with tensorflow.variable_scope(
'generator',
regularizer=tensorflow.contrib.layers.l2_regularizer(
self.l2scale)):
for i, gen_dim in enumerate(self.generator_dims[:-1]):
w = tensorflow.get_variable('w_' + str(i),
shape=[temp_dim, gen_dim])
h = tensorflow.matmul(temp_vec, w)
h2 = tensorflow.contrib.layers.batch_norm(
h,
decay=self.bn_decay,
scale=True,
is_training=bn_train,
updates_collections=None)
h3 = self.generator_activation(h2)
temp_vec = h3 + temp_vec
temp_dim = temp_dim
w = tensorflow.get_varaible(
'w' + str(i), shape=[temp_dim, self.generator_dims[-1]])
h = tensorflow.matmul(temp_vec, w)
h2 = tensorflow.contrib.layers.batch_norm(h,
decay=self.bn_decay,
scale=True,
is_training=bn_train,
updates_collections=None)
if self.data_type == 'binary':
h3 = tensorflow.nn.tanh(h2)
else:
h3 = tensorflow.nn.relu(h2)
output = h3 + temp_vec
return output
def __init__(self,
config,
name_scope,
forward_only=False,
num_sample=256,
dtype=tf.float32):
#self.scope_name = scope_name
# with tf.variable_scope(self.scope_name):
source_vocab_size = config.vocab_size
target_vocab_size = config.vocab_size
emb_dim = config.emb_dim
self.buckets = config.buckets
self.learning_rate = tf.Variable(float(config.learning_rate),
trainable=False,
dtype=dtype)
self.learning_rate_decay_op = self.learning_rate.assign(
self.learning_rate * config.learning_rate_decay_factor)
self.global_step = tf.Variable(0, trainable=False)
self.batch_size = config.batch_size
self.num_layers = config.num_layers
self.max_gradient_norm = config.max_gradient_norm
self.mc_search = tf.placeholder(tf.bool, name="mc_search")
self.forward_only = tf.placeholder(tf.bool, name="forward_only")
self.up_reward = tf.placeholder(tf.bool, name="up_reward")
self.reward_bias = tf.get_varaible("reward_bias", [1],
dtype=tf.float32)
output_projection = None
softmax_loss_function = None
if num_samples > 0 and num_samples < target_vocab_size:
w_t = tf.get_varaible("proj_w", [target_vocab_size, emb_dim],
dtype=dtype)
w = tf.transpose(w_t)
b = tf.get_varaible("proj_b", [target_vocab_size], dtype=dtype)
output_projection = (w, b)
def sampled_loss(inputs, labels):
labels = tf.reshape(labels, [-1, 1])
# we need to compute the sampled_softmax_loss using 32bit floats to
# avoid numberical instabilities.
local_w_t = tf.cast(w_t, tf.float32)
local_b = tf.cast(b, tf.float32)
local_inputs = tf.cast(inputs, tf.float32)
return tf.cast(
tf.nn.sampled_softmax_loss(local_w_t, local_b, labels,
local_inputs, num_samples,
target_vocab_size), dtype)
softmax_loss_function = sampled_loss
# Create the internal multi-layer cell for our learning_rate.
single_cell = tf.contrrib.rnn.GRUCell(emb_dim)
cell = single_cell
if self.num_layers > 1:
cell = tf.contrib.rnn.MultiRNNCell([single_cell] * self.num_layers)
# The seq2seq function: we use embedding for the input and attention.
def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):
return rl_seq2seq.embedding_attention_seq2seq(
encoder_inputs,
decoder_inputs,
cell,
num_encoder_symbols=source_vocab_size,
num_decoder_symbols=target_vocab_size,
embedding_size=emb_dim,
output_projection=output_projection,
feed_previous=do_decode,
mc_search=self.mc_search,
dtype=dtype)
# Feeds for inputs.
self.encoder_inputs = []
self.decoder_inputs = []
self.target_weights = []
for i in xrange(
self.buckets[-1][0]): # Last bucket is the biggest one.
self.encoder_inputs.append(
tf.placeholder(tf.int32,
shape=[None],
name="encoder{0}".format(i)))
for i in xrange(self.buckets[-1][1] + 1):
self.decoder_inputs.append(
tf.placeholder(tf.int32,
shape=[None],
name="decoder{0}".format(i)))
self.target_weights.append(
tf.placeholder(dtype, shape=[None],
name="weight{0}".format(i)))
self.reward = [
tf.placeholder(tf.float32, name="reward_%i" % i)
for i in range(len(self.buckets))
]
#Our targets are decoder inputs shifted by one.
targets = [
self.deocder_inputs[i + 1]
for i in xrange(len(self.decoder_inputs) - 1)
]
self.outputs, self.losses, self.encoder_state = rl_seq2seq.model_with_buchkets(
self.encoder_inputs,
self.decoder_inputs,
targets,
self.target_weights,
self.buckets,
source_vocab_size,
self.batch_size,
lambda x, y: seq2seq2seq_f(
x, y, tf.where(self.forward_only, True, False)),
output_projection=output_projection,
softmax_loss_function=softmax_loss_function)
for b in xrange(len(self.buckets)):
self.outputs[b] = [
tf.cond(
self.forward_only, lambda: tf.matmul(
output, output_projection[0]) + output_projection[1],
lambda: output) for output in self.outputs[b]
]
if not forward_only:
with tf.name_scope("gradient_descent"):
self.gradient_norms = []
self.updates = []
self.aj_losses = []
self.gen_params = [
p for p in tf.trainable_variables() if name_scope in p.name
]
#opt = tf.train.GradientDescentOptimizer(self.learning_rate)
for b in xrange(len(self.buckets)):
R = tf.subtract(self.reward[b], self.reward_bias)
# self.reward[b] = self.reward[b] - reward_bias
adjusted_loss = tf.cond(
self.up_reward,
lambda: tf.multiply(self.losses[b], self.reward[b]),
lambda: self.losses[b])
# adjusted_loss = tf.cond(self.up_reward,
# lambda: tf.multiply(self.losses[b], R),
# lambda: self.losses[b])
self.aj_losses.append(adjusted_loss)
gradients = tf.gradients(adjusted_loss, self.gen_params)
clipped_gradients, norm = tf.clip_by_global_norm(
gradients, self.max_gradient_norm)
self.gradient_norms.append(norm)
self.updates.append(
opt.apply_gradients(zip(clipped_gradients,
self.gen_params),
global_step=self.global_step))
self.gen_variables = [
k for k in tf.gloabl_variable() if name_scope in k.name
]
self.saver = tf.train.Saver(self.gen_variables)
def layer_op(self, image, conditioning, is_training):
conditioning = tensorflow.image.resize_images(conditioning, [160, 120])
batch_size = image.get_shape().as_list()[0]
w_init = tensorflow.random_normal_initializer(0.0, 0.02)
b_init = tensorflow.random_normal_initializer(0.0, 0.02)
ch = [32, 64, 128, 256, 512, 1024]
def leaky_relu(x, alpha=0.2):
with tensorflow.name_scope('leaky_relu'):
return 0.5 * (1 + alpha) * x + 0.5 * (1 - alpha) * abs(x)
def down(ch, x):
with tensorflow.name_scope('downsample'):
x_ch = x.shape.as_list()[-1]
conv_kernel = tensorflow.get_variable('w',
shape=(x.shape[1],
x.shape[2], 3),
initializer=w_init,
regularizer=None)
c = tensorflow.nn.convolution(input=x,
filter=conv_kernel,
strides=2,
name='conv')
c = tensorflow.contrib.layers.batch_norm(c)
c = leaky_relu(c)
return c
def convr(ch, x):
conv_kernel = tensorflow.get_variable('w',
shape=(x.shape[1],
x.shape[2], 3),
initializer=w_init,
regularizer=None)
c = tensorflow.nn.convolution(input=x,
filter=conv_kernel,
strides=2,
name='conv')
return leaky_relu(tensorflow.contrib.layers.batch_norm(c))
def conv(ch, x, s):
conv_kernel = tensorflow.get_variable('w',
shape=(x.shape[1],
x.shape[2], 3),
initializer=w_init,
regularizer=None)
c = tensorflow.nn.convolution(input=x,
filter=conv_kernel,
strides=2,
name='conv')
return leaky_relu(tensorflow.contrib.layers.batch_norm(c) + s)
def down_blocks(ch, x):
with tensorflow.name_scope('down_resnet'):
s = down(ch, x)
r = convr(ch, s)
return conv(ch, r, s)
if not conditioning is None:
image = tensorflow.concat([image, conditioning], axis=1)
with tensorflow.name_scope('feature'):
conv_kernel = tensorflow.get_variable('w',
shape=(image.shape[1],
image.shape[2], 5),
initializer=w_init,
regularizer=None)
d_h1s = tensorflow.nn.convolution(input=image,
filters=conv_kernel,
strides=2,
name='conv')
d_h1s = leaky_relu(d_h1s)
d_h1r = convr(ch[0], d_h1s)
d_h1 = conv(ch[0], d_h1r, d_h1s)
d_h2 = down_blocks(ch[1], d_h1)
d_h3 = down_blocks(ch[2], d_h2)
d_h4 = down_blocks(ch[3], d_h3)
d_h5 = down_blocks(ch[4], d_h4)
d_h6 = down_blocks(ch[5], d_h5)
with tensorflow.name_scope('fc'):
d_hf = tensorflow.reshape(d_h6, [batch_size, -1])
d_nf_o = numpy.prod(d_hf.get_shape().as_list()[1:])
D_wo = tensorflow.get_varaible('D_Wo',
shape=[d_nf_o, 1],
initializer=w_init)
D_bo = tensorflow.get_variable('D_bo',
shape=[1],
initializer=b_init)
d_logit = tensorflow.matmul(d_hf, D_wo) + D_bo
return d_logit
def _build_cin(self, emb_out, use_resblock=False, split_connect=True, reduce_filter_complexity=False, add_bias=False):
# hk: field_size of next layer
h0 = self._field_size
hk = h0
cin_x0 = tf.reshape(emb_out, shape=[-1,self._field_size, self.embedding_size])
cin_xk = cin_x0 #cin input of next layer
cin_outs = []
cin_out_size = 0
cin_x0_split = tf.split(cin_x0,self.embedding_size*[1],2) # k# None*F*1
with tf.variable_scope('cin',initializer=tf.truncated_normal_initializer(stddev=0.1)) as scope:
for index, next_hk in enumerate(self.cross_layer_sizes):
cin_xk_split = tf.split(cin_xk, self.embedding_size,2) # k# None*hk*1
dots = tf.matmul(cin_x0_split, cin_xk_split, transpose_b=True) # k*None*F*hk
dots = tf.reshape(dots, shape=[self.embedding_size,-1, h0*hk]) # k*None* (F*hk)
dots = tf.transpose(dots, perm=[1,0,2]) # None * k *(F*hk)
if reduce_filter_complexity:
latent_dim = self.config.get('cin_filter_latent_dim',2)
filter0 = tf.get_variable('filter0_'+str(index),
shape=[1,next_hk, h0,latent_dim],dtype=tf.float32)
filter1 = tf.get_variable('filter1_'+str(index),
shape=[1, next_hk,latent_dim, hk],dtype=tf.float32)
filters = tf.matmul(filter0, filter1) # 1*next_hk*F*hk
filters = tf.reshape(filters,shape=[1,next_hk, h0*hk])
filters = tf.transpose(filters, perm=[0,2,1]) # 1*(h0*hk)*next_hk
else:
filters = tf.get_variable('filter_'+str(index),
shape=[1,h0*hk,next_hk],dtype=tf.float32)
# None * k *(F*hk) conv 1*(h0*hk)*next_hk = None*k*next_hk
layer_out = tf.nn.conv1d(dots,filters=filters,stride=1,padding='VALID') # None*k*next_hk
if add_bias:
bias = tf.get_variable('filter_bias'+str(index),
shape=[next_hk],dtype= tf.float32, initializer=tf.zeros_initializer())
layer_out = tf.nn.bias_add(layer_out, bias)
activate = self.config.get('cin_activate',tf.nn.relu)
layer_out = activate(layer_out)
layer_out = tf.transpose(layer_out, perm=[0,2,1]) #None*next_hk*k
if split_connect:
if index != len(self.cross_layer_sizes)-1:
cin_xk, cin_out = tf.split(layer_out, 2*[int(next_hk/2)],1)
cin_out_size += int(next_hk/2)
else:
cin_xk = 0
cin_out = layer_out
cin_out_size += next_hk
hk = int(next_hk/2)
else:
cin_xk = layer_out
cin_out = layer_out
cin_out_size += next_hk
cin_outs.append(cin_out)
result = tf.concat(cin_outs,axis=1) # None*cin_out_size * k
result = tf.reduce_sum(result, axis=-1) # None * cin_out_size
if use_resblock: # concat instead of add
hidden_size = self.config.get('cin_resblock_hidden_size',32)
block_w = tf.get_variable('resblock_hidden_w',shape=[cin_out_size, hidden_size],dtype=tf.float32)
block_b = tf.get_variable('resblock_hidden_b',shape=[hidden_size],dtyep=tf.float32,initializer=tf.zeros_initializer())
hidden_input = tf.nn.xw_plus_b(result, block_w, block_b)
activate = tf.config.get('cin_resblock_hidden_activate',tf.nn.relu)
hidden_out = activate(hidden_input) #None* hidden_size
merge_w = tf.get_variable('resblock_merge_w', shape=[hidden_size+cin_out_size,1],dtype=tf.float32)
merge_b = tf.get_varaible('resblock_merge_b',shape=[1],dtype=tf.float32,initializer=tf.zeros_initializer())
merge_input= tf.concat([hidden_out, result], axis=1) # None*(hidden_size+cin_out_size)
xdeep_out = tf.nn.xw_plus_b(merge_input,merge_w, merge_b)
else:
w = tf.get_variable('cin_w',shape=[cin_out_size,1],dtype=tf.float32)
b = tf.get_variable('cin_b',shape=[1],dtype=tf.float32, initializer=tf.zeros_initializer())
xdeep_out = tf.nn.xw_plus_b(result, w,b)
return xdeep_out # None * 1
# Define how many neurons we want in each layer of our neural network
layer_1_nodes = 50
layer_2_nodes = 100
layer_3_nodes = 50
# Section One: Define the layers of the neural network itself
# Input Layer
with tf.variable_scope('input'):
X = tf.placeholder(tf.float32, name='X', shape=(None, number_of_inputs))
# Layer 1
with tf.variable_scope('layer_1'):
weights = tf.get_varaible(
name='weights1',
shape=[number_of_inputs, layer_1_nodes],
initializer=tf.contrib.layers.xavier_intializer())
biases = tf.get_variable(name='biases1',
shape=[layer_1_nodes],
initializer=tf.zeros_initializer())
layer_1_output = tf.nn.relu(tf.matmul(X, weights) + biases)
# Layer 2
with tf.variable_scope('layer_2'):
weights = tf.get_varaible(
name='weights2',
shape=[layer_1_nodes, layer_2_nodes],
initializer=tf.contrib.layers.xavier_intializer())
biases = tf.get_variable(name='biases2',
shape=[layer_2_nodes],
initializer=tf.zeros_initializer())