def discriminator(self, image, t_text_embedding, reuse=False): if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu(ops.conv2d(image, self.options['df_dim'], name = 'd_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, self.options['df_dim']*2, name = 'd_h1_conv'))) #16 h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, self.options['df_dim']*4, name = 'd_h2_conv'))) #8 h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, self.options['df_dim']*8, name = 'd_h3_conv'))) #4 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1,4,4,1], name='tiled_embeddings') h3_concat = tf.concat( 3, [h3, tiled_embeddings], name='h3_concat') h3_new = ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,1,1,1, name = 'd_h3_conv_new'))) #4 h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4), h4
def discriminator(self, image, t_text_embedding):
update_collection=tf.GraphKeys.UPDATE_OPS
with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE):
h0 = ops.lrelu(ops.conv2d_sn(image, self.options['df_dim'], spectral_normed=True, update_collection=update_collection, name = 'd_h0_conv')) #32
h1 = ops.lrelu( self.d_bn1(ops.conv2d_sn(h0, self.options['df_dim']*2, spectral_normed=True, update_collection=update_collection, name = 'd_h1_conv'))) #16
h2 = ops.lrelu( self.d_bn2(ops.conv2d_sn(h1, self.options['df_dim']*4, spectral_normed=True, update_collection=update_collection, name = 'd_h2_conv'))) #8
h3 = ops.lrelu( self.d_bn3(ops.conv2d_sn(h2, self.options['df_dim']*8, spectral_normed=True, update_collection=update_collection, name = 'd_h3_conv'))) #4
h3_new = ops.lrelu( self.d_bn4(ops.conv2d_sn(h3, self.options['df_dim']*8, 1,1,1,1, spectral_normed=True, update_collection=update_collection, name = 'd_h3_conv_new'))) #4
h3_new = tf.reshape(h3_new, [self.options['batch_size'], -1])
image_embedding = ops.linear(h3_new, self.options['t_dim'], 'd_h3_embedding')
# Auxiliary classifier
class_logit = ops.linear(image_embedding, self.options['num_class'], 'd_image_classifier')
# Embedding matrix of condition
reduced_text_embeddings = ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')
# Scalar output function
h4 = ops.linear(image_embedding, 1, 'd_scalar_output')
discriminator_output_logit = tf.reduce_sum(tf.multiply(reduced_text_embeddings, image_embedding), 1, keepdims=True) + h4
return discriminator_output_logit, discriminator_output_logit, class_logit
def generator(self, t_z, t_text_embedding): s = self.options['image_size'] s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding') ) z_concat = tf.concat(1, [t_z, reduced_text_embedding]) z_ = ops.linear(z_concat, self.options['gf_dim']*8*s16*s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0)) h1 = ops.deconv2d(h0, [self.options['batch_size'], s8, s8, self.options['gf_dim']*4], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1)) h2 = ops.deconv2d(h1, [self.options['batch_size'], s4, s4, self.options['gf_dim']*2], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2)) h3 = ops.deconv2d(h2, [self.options['batch_size'], s2, s2, self.options['gf_dim']*1], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') return (tf.tanh(h4)/2. + 0.5)
def generator(self, t_z, t_text_embedding): s = self.options['image_size'] s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding') ) z_concat = tf.concat([t_z, reduced_text_embedding], 1) z_ = ops.linear(z_concat, self.options['gf_dim']*8*s16*s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0)) h1 = ops.deconv2d(h0, [self.options['batch_size'], s8, s8, self.options['gf_dim']*4], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1)) h2 = ops.deconv2d(h1, [self.options['batch_size'], s4, s4, self.options['gf_dim']*2], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2)) h3 = ops.deconv2d(h2, [self.options['batch_size'], s2, s2, self.options['gf_dim']*1], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') return (tf.tanh(h4)/2. + 0.5)
def sampler(self, t_z, t_text_embedding):
tf.get_variable_scope().reuse_variables()
s = self.options['image_size']
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
reduced_text_embedding = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding'))
#z_concat = tf.concat(1, [t_z, reduced_text_embedding])
z_concat = t_text_embedding
z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16,
'g_h0_lin')
h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8])
h0 = tf.nn.relu(self.g_bn0(h0, train=False))
h1 = ops.deconv2d(
h0,
[self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4],
name='g_h1')
h1 = tf.nn.relu(self.g_bn1(h1, train=False))
h2 = ops.deconv2d(
h1,
[self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2],
name='g_h2')
h2 = tf.nn.relu(self.g_bn2(h2, train=False))
h3 = ops.deconv2d(
h2,
[self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1],
name='g_h3')
h3 = tf.nn.relu(self.g_bn3(h3, train=False))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3],
name='g_h4')
return (tf.tanh(h4) / 2. + 0.5)
def discriminator(self, image, t_text_embedding, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
h0 = ops.lrelu(
ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) #32
h1 = ops.lrelu(
self.d_bn1(
ops.conv2d(h0, self.options['df_dim'] * 2,
name='d_h1_conv'))) #16
h2 = ops.lrelu(
self.d_bn2(
ops.conv2d(h1, self.options['df_dim'] * 4,
name='d_h2_conv'))) #8
h3 = ops.lrelu(
self.d_bn3(
ops.conv2d(h2, self.options['df_dim'] * 8,
name='d_h3_conv'))) #4
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings =\
ops.lrelu(ops.linear(t_text_embedding,
self.options['t_dim'], 'd_embedding'))
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1)
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2)
tiled_embeddings =\
tf.tile(reduced_text_embeddings, [1,4,4,1], name='tiled_embeddings')
h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_new =\
ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,
1, 1, 1, name='d_h3_conv_new'))) #4
h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]),
1, 'd_h3_lin')
return tf.nn.sigmoid(h4), h4
def discriminator(self, image, reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
h0 = ops.lrelu( ops.conv2d(image, 64,
name = 'd_h0_conv')) #32
h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, 128,
name = 'd_h1_conv'))) #32
h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, 256,
name = 'd_h2_conv'))) #32
h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, 512,
name = 'd_h3_conv'))) #32
h4 = ops.linear(tf.reshape(h3, [self.options['batch_size'], -1]), 1, 'd_h3_lin')
return tf.nn.sigmoid(h4), h4
def discriminator(self, image, t_text_embedding):
with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE):
h0 = ops.lrelu(ops.conv2d(image, self.options['df_dim'], name = 'd_h0_conv')) #32
h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, self.options['df_dim']*2, name = 'd_h1_conv'))) #16
h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, self.options['df_dim']*4, name = 'd_h2_conv'))) #8
h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, self.options['df_dim']*8, name = 'd_h3_conv'))) #4
h4 = ops.linear(tf.reshape(h3, [self.options['batch_size'], -1]), 1, 'd_h3_lin')
# ADD TEXT EMBEDDING TO THE NETWORK
#reduced_text_embeddings = ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding'))
#reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,1)
#reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,2)
#tiled_embeddings = tf.tile(reduced_text_embeddings, [1,4,4,1], name='tiled_embeddings')
#h3_concat = tf.concat( 3, [h3, tiled_embeddings], name='h3_concat')
#h3_new = ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,1,1,1, name = 'd_h3_conv_new'))) #4
#h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin')
return h4, h4
def decoder(self, code):
hidden = ops.linear(code, self.options['caption_vector_length'], 'g_dec_h1')
reconstruct_code = ops.linear(code, self.options['caption_vector_length'], 'g_dec_h2')
return reconstruct_code
def discriminator(self, image, t_text_embedding):
update_collection = tf.GraphKeys.UPDATE_OPS
with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE):
h0 = ops.lrelu(
ops.conv2d_sn(image,
self.options['df_dim'],
spectral_normed=True,
update_collection=update_collection,
name='d_h0_conv')) #32
h1 = ops.lrelu(
self.d_bn1(
ops.conv2d_sn(h0,
self.options['df_dim'] * 2,
spectral_normed=True,
update_collection=update_collection,
name='d_h1_conv'))) #16
h2 = ops.lrelu(
self.d_bn2(
ops.conv2d_sn(h1,
self.options['df_dim'] * 4,
spectral_normed=True,
update_collection=update_collection,
name='d_h2_conv'))) #8
h3 = ops.lrelu(
self.d_bn3(
ops.conv2d_sn(h2,
self.options['df_dim'] * 8,
spectral_normed=True,
update_collection=update_collection,
name='d_h3_conv'))) #4
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'],
'd_embedding'))
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,
1)
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,
2)
tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1],
name='tiled_embeddings')
h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_new = ops.lrelu(
self.d_bn4(
ops.conv2d_sn(h3_concat,
self.options['df_dim'] * 8,
1,
1,
1,
1,
spectral_normed=True,
update_collection=update_collection,
name='d_h3_conv_new'))) #4
h4 = ops.linear(
tf.reshape(h3_new, [self.options['batch_size'], -1]), 1,
'd_h3_lin')
# Auxiliary classifier
class_logit = ops.linear(
tf.reshape(h3_new, [self.options['batch_size'], -1]),
self.options['num_class'], 'd_image_classifier')
return h4, h4, class_logit
def discriminator(self, image, t_text_embedding, reuse=False):
if reuse:
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
h0 = ops.lrelu(
ops.conv2d(image, self.options['df_dim'],
name='d_h0_conv')) #32,48
h1 = ops.lrelu(
self.d_bn1(
ops.conv2d(h0,
self.options['df_dim'] * 2,
name='d_h1_conv'))) #16,24
h2 = ops.lrelu(
self.d_bn2(
ops.conv2d(h1,
self.options['df_dim'] * 4,
name='d_h2_conv'))) #8,12
h3 = ops.lrelu(
self.d_bn3(
ops.conv2d(h2,
self.options['df_dim'] * 8,
name='d_h3_conv'))) #4,6
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'],
'd_embedding'))
reduced_text_embeddings = tf.expand_dims(
reduced_text_embeddings, 1)
reduced_text_embeddings = tf.expand_dims(
reduced_text_embeddings, 2)
tiled_embeddings = tf.tile(reduced_text_embeddings,
[1, 4, 4, 1],
name='tiled_embeddings')
# h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_concat = tf.concat(values=[h3, tiled_embeddings],
axis=3,
name='h3_concat')
h3_new = ops.lrelu(
self.d_bn4(
ops.conv2d(h3_concat,
self.options['df_dim'] * 8,
1,
1,
1,
1,
name='d_h3_conv_new'))) #4
h4 = ops.linear(
tf.reshape(h3_new, [self.options['batch_size'], -1]), 1,
'd_h3_lin')
# return tf.nn.sigmoid(h4), h4
else:
h0 = ops.lrelu(
ops.conv2d(image, self.options['df_dim'],
name='d_h0_conv')) #32,48
h1 = ops.lrelu(
self.d_bn1(
ops.conv2d(h0,
self.options['df_dim'] * 2,
name='d_h1_conv'))) #16,24
h2 = ops.lrelu(
self.d_bn2(
ops.conv2d(h1,
self.options['df_dim'] * 4,
name='d_h2_conv'))) #8,12
h3 = ops.lrelu(
self.d_bn3(
ops.conv2d(h2,
self.options['df_dim'] * 8,
name='d_h3_conv'))) #4,6
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'],
'd_embedding'))
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,
1)
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,
2)
tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1],
name='tiled_embeddings')
# h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_concat = tf.concat(values=[h3, tiled_embeddings],
axis=3,
name='h3_concat')
h3_new = ops.lrelu(
self.d_bn4(
ops.conv2d(h3_concat,
self.options['df_dim'] * 8,
1,
1,
1,
1,
name='d_h3_conv_new'))) #4
h4 = ops.linear(
tf.reshape(h3_new, [self.options['batch_size'], -1]), 1,
'd_h3_lin')
# return tf.nn.sigmoid(h4), h4
return tf.nn.sigmoid(h4), h4
# if reuse:
# tf.get_variable_scope().reuse_variables()
# h0 = ops.lrelu(ops.conv2d(image, self.options['df_dim'], name = 'd_h0_conv')) #32,48
# h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, self.options['df_dim']*2, name = 'd_h1_conv'))) #16,24
# h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, self.options['df_dim']*4, name = 'd_h2_conv'))) #8,12
# h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, self.options['df_dim']*8, name = 'd_h3_conv'))) #4,6
# # ADD TEXT EMBEDDING TO THE NETWORK
# reduced_text_embeddings = ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding'))
# reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,1)
# reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,2)
# tiled_embeddings = tf.tile(reduced_text_embeddings, [1,6,6,1], name='tiled_embeddings')
# # h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
# h3_concat = tf.concat(values=[h3, tiled_embeddings], axis=3, name='h3_concat')
# h3_new = ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,1,1,1, name = 'd_h3_conv_new'))) #4
# h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin')
# return tf.nn.sigmoid(h4), h4
def discriminator(self, image, t_text_embedding):
with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE):
if self.options['vgg']:
h0 = ops.lrelu(
ops.conv2d(image, 3, stride=1, name='d_h0a_conv'))
h0 = ops.lrelu(ops.conv2d(h0, 3, stride=2, name='d_h0_conv'))
h1 = ops.lrelu(
self.d_bn1a(
ops.conv2d(h0,
self.options['df_dim'] * 2,
stride=1,
name='d_h1a_conv')))
h1 = ops.lrelu(
self.d_bn1(
ops.conv2d(h0,
self.options['df_dim'] * 2,
stride=2,
name='d_h1_conv')))
h2 = ops.lrelu(
self.d_bn2a(
ops.conv2d(h1,
self.options['df_dim'] * 4,
stride=1,
name='d_h2a_conv')))
h2 = ops.lrelu(
self.d_bn2(
ops.conv2d(h2,
self.options['df_dim'] * 4,
stride=2,
name='d_h2_conv')))
h3 = ops.lrelu(
self.d_bn3a(
ops.conv2d(h2,
self.options['df_dim'] * 8,
stride=1,
name='d_h3a_conv')))
h3 = ops.lrelu(
self.d_bn3(
ops.conv2d(h3,
self.options['df_dim'] * 8,
stride=2,
name='d_h3_conv')))
else:
if self.options['extra_64']:
image = ops.lrelu(
ops.conv2d(image, 3, stride=1, name='d_h0a_conv'))
h0 = ops.lrelu(
ops.conv2d(image, self.options['df_dim'],
name='d_h0_conv')) # 32
if self.options['extra_32']:
h0 = ops.lrelu(
self.d_bn1a(
ops.conv2d(h0,
self.options['df_dim'],
stride=1,
name='d_h1a_conv')))
h1 = ops.lrelu(
self.d_bn1(
ops.conv2d(h0,
self.options['df_dim'] * 2,
name='d_h1_conv'))) # 16
h2 = ops.lrelu(
self.d_bn2(
ops.conv2d(h1,
self.options['df_dim'] * 4,
name='d_h2_conv'))) # 8
h3 = ops.lrelu(
self.d_bn3(
ops.conv2d(h2,
self.options['df_dim'] * 8,
name='d_h3_conv'))) # 4
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'],
'd_embedding'))
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,
1)
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,
2)
tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1],
name='tiled_embeddings')
h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_new = ops.lrelu(
self.d_bn4(
ops.conv2d(h3_concat,
self.options['df_dim'] * 8,
1,
1,
1,
1,
name='d_h3_conv_new'))) # 4
h4 = ops.linear(
tf.reshape(h3_new, [self.options['batch_size'], -1]), 1,
'd_h3_lin')
return tf.nn.sigmoid(h4), h4
def generator(self, t_z, t_text_embedding):
s = self.options['image_size'] #64 x 64
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
reduced_text_embedding = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'],
'g_embedding')) #self.options['t_dim', 256]
z_concat = tf.concat([t_z, reduced_text_embedding],
1) #t_z is batch_size, z_dim, which is 100
z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16,
'g_h0_lin')
h0 = tf.reshape(z_, [
-1, s16, s16, self.options['gf_dim'] * 8
]) #[-1, 4, 4, 64 * 8] gf_dim is number of filters in the first layer
h0 = tf.nn.relu(self.g_bn0(h0))
if self.options['vgg']:
h1 = ops.deconv2d(h0, [
self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4
],
stride=2,
name='g_h1') #8
h1 = tf.nn.relu(self.g_bn1(h1))
h1 = ops.deconv2d(h1, [
self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4
],
stride=1,
name='g_h1b')
h1 = tf.nn.relu(self.g_bn1b(h1))
h2 = ops.deconv2d(h1, [
self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2
],
stride=2,
name='g_h2') #16
h2 = tf.nn.relu(self.g_bn2(h2))
h2 = ops.deconv2d(h2, [
self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2
],
stride=1,
name='g_h2b')
h2 = tf.nn.relu(self.g_bn2b(h2))
h3 = ops.deconv2d(h2, [
self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1
],
stride=2,
name='g_h3') #32
h3 = tf.nn.relu(self.g_bn3(h3))
h3 = ops.deconv2d(h3, [
self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1
],
stride=1,
name='g_h3b')
h3 = tf.nn.relu(self.g_bn3b(h3))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3],
stride=2,
name='g_h4') #64
h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3],
stride=1,
name='g_h4b')
return (tf.tanh(h4) / 2. + 0.5)
else:
h1 = ops.deconv2d(h0, [
self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4
],
name='g_h1')
h1 = tf.nn.relu(self.g_bn1(h1))
h2 = ops.deconv2d(h1, [
self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2
],
name='g_h2')
h2 = tf.nn.relu(self.g_bn2(h2))
h3 = ops.deconv2d(h2, [
self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1
],
name='g_h3')
h3 = tf.nn.relu(self.g_bn3(h3))
if self.options['extra_32']:
h3 = ops.deconv2d(h3, [
self.options['batch_size'], s2, s2,
self.options['gf_dim'] * 1
],
stride=1,
name='g_h3b')
h3 = tf.nn.relu(self.g_bn3b(h3))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3],
name='g_h4')
if self.options['extra_64']:
h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3],
stride=1,
name='g_h4b')
return (tf.tanh(h4) / 2. + 0.5)
def sampler(self, t_z, t_text_embedding):
tf.get_variable_scope().reuse_variables()
s = self.options['image_size']
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
reduced_text_embedding = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding'))
z_concat = tf.concat([t_z, reduced_text_embedding], 1)
z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16,
'g_h0_lin')
h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8])
h0 = tf.nn.relu(self.g_bn0(h0, train=False))
if self.options['vgg']:
h1 = ops.deconv2d(h0, [
self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4
],
stride=2,
name='g_h1') #8
h1 = tf.nn.relu(self.g_bn1(h1, train=False))
h1 = ops.deconv2d(h1, [
self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4
],
stride=1,
name='g_h1b')
h1 = tf.nn.relu(self.g_bn1b(h1, train=False))
h2 = ops.deconv2d(h1, [
self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2
],
stride=2,
name='g_h2') #16
h2 = tf.nn.relu(self.g_bn2(h2, train=False))
h2 = ops.deconv2d(h2, [
self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2
],
stride=1,
name='g_h2b')
h2 = tf.nn.relu(self.g_bn2b(h2, train=False))
h3 = ops.deconv2d(h2, [
self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1
],
stride=2,
name='g_h3') #32
h3 = tf.nn.relu(self.g_bn3(h3, train=False))
h3 = ops.deconv2d(h3, [
self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1
],
stride=1,
name='g_h3b')
h3 = tf.nn.relu(self.g_bn3b(h3, train=False))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3],
stride=2,
name='g_h4') #64
h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3],
stride=1,
name='g_h4b')
return (tf.tanh(h4) / 2. + 0.5)
else:
h1 = ops.deconv2d(h0, [
self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4
],
name='g_h1')
h1 = tf.nn.relu(self.g_bn1(h1, train=False))
h2 = ops.deconv2d(h1, [
self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2
],
name='g_h2')
h2 = tf.nn.relu(self.g_bn2(h2, train=False))
h3 = ops.deconv2d(h2, [
self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1
],
name='g_h3')
h3 = tf.nn.relu(self.g_bn3(h3, train=False))
if self.options['extra_32']:
h3 = ops.deconv2d(h3, [
self.options['batch_size'], s2, s2,
self.options['gf_dim'] * 1
],
stride=1,
name='g_h3b')
h3 = tf.nn.relu(self.g_bn3b(h3, train=False))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3],
name='g_h4')
if self.options['extra_64']:
h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3],
stride=1,
name='g_h4b')
return (tf.tanh(h4) / 2. + 0.5)
def discriminator(self,
image,
t_text_embedding,
n_classes,
t_training,
reuse=False):
if reuse:
tf.get_variable_scope().reuse_variables()
h0 = ops.lrelu(
ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) # 64
h1 = ops.lrelu(
slim.batch_norm(ops.conv2d(h0,
self.options['df_dim'] * 2,
name='d_h1_conv'),
reuse=reuse,
is_training=t_training,
scope='d_bn1')) # 32
h2 = ops.lrelu(
slim.batch_norm(ops.conv2d(h1,
self.options['df_dim'] * 4,
name='d_h2_conv'),
reuse=reuse,
is_training=t_training,
scope='d_bn2')) # 16
h3 = ops.lrelu(
slim.batch_norm(ops.conv2d(h2,
self.options['df_dim'] * 8,
name='d_h3_conv'),
reuse=reuse,
is_training=t_training,
scope='d_bn3')) # 8
h3_shape = h3.get_shape().as_list()
# ADD TEXT EMBEDDING TO THE NETWORK
reduced_text_embeddings = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding'))
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1)
reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2)
tiled_embeddings = tf.tile(reduced_text_embeddings,
[1, h3_shape[1], h3_shape[1], 1],
name='tiled_embeddings')
h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat')
h3_new = ops.lrelu(
slim.batch_norm(ops.conv2d(h3_concat,
self.options['df_dim'] * 8,
1,
1,
1,
1,
name='d_h3_conv_new'),
reuse=reuse,
is_training=t_training,
scope='d_bn4')) # 4
h3_flat = tf.reshape(h3_new, [self.options['batch_size'], -1])
h4 = ops.linear(h3_flat, 1, 'd_h4_lin_rw')
h4_aux = ops.linear(h3_flat, n_classes, 'd_h4_lin_ac')
return tf.nn.sigmoid(h4), h4, tf.nn.sigmoid(h4_aux), h4_aux
def generator(self, word2vec, reuse=False):
with tf.variable_scope("generator") as scope:
if reuse:
scope.reuse_variables()
filter_sizes = [3, 4, 5]
embedding_size = 400
num_filters = 800
sequence_length = 15
embedded_chars_expanded = tf.expand_dims(word2vec, -1) # ?xlengthxfeaturex1
# Create a convolution + maxpool layer for each filter size
pooled_outputs = []
for i, filter_size in enumerate(filter_sizes):
#with tf.name_scope("conv-maxpool-%s" % filter_size):
# Convolution Layer
filter_shape = [filter_size, embedding_size, 1, num_filters]
# W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W")
# b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b")
if i == 0:
h = tf.nn.relu(self.gbn0(
ops.conv2dv1(embedded_chars_expanded, num_filters, filter_shape, name='g_h%d_conv' % i))) # 16
elif i == 1:
h = tf.nn.relu(self.gbn1(
ops.conv2dv1(embedded_chars_expanded, num_filters, filter_shape, name='g_h%d_conv' % i))) # 16
else:
h = tf.nn.relu(self.gbn2(
ops.conv2dv1(embedded_chars_expanded, num_filters, filter_shape, name='g_h%d_conv' % i))) # 16
# conv = tf.nn.conv2d(
# self.embedded_chars_expanded,
# W,
# strides=[1, 1, 1, 1],
# padding="VALID",
# name="conv")
# Apply nonlinearity
#h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu")
# Maxpooling over the outputs
pooled = tf.nn.max_pool(
h,
ksize=[1, sequence_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID',
name="pool")
pooled_outputs.append(pooled)
# Combine all the pooled features
num_filters_total = num_filters * len(filter_sizes)
h_pool = tf.concat(pooled_outputs, 3)
h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total])
s = self.options['image_size']
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
reduced_text_embedding = ops.lrelu(ops.linear(h_pool_flat, self.options['t_dim'], 'g_embedding'))
z_ = ops.linear(reduced_text_embedding, self.options['gf_dim'] * 8 * s16 * s16, 'g_h0_lin')
h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8])
h0 = tf.nn.relu(self.g_bn0(h0))
h1 = ops.deconv2d(h0, [self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4], name='g_h1')
h1 = tf.nn.relu(self.g_bn1(h1))
h2 = ops.deconv2d(h1, [self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2], name='g_h2')
h2 = tf.nn.relu(self.g_bn2(h2))
h3 = ops.deconv2d(h2, [self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1], name='g_h3')
h3 = tf.nn.relu(self.g_bn3(h3))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4')
return (tf.tanh(h4) / 2. + 0.5), h_pool_flat
def __init__(self, size_obs, size_act, net_struct = [100, 100, 100, 100], name='dbg'):
self.tensorboardpath = 'tensorboards/' + name
self.train_writer = tf.summary.FileWriter(self.tensorboardpath)
self.ModelPath = 'Models/Imitation' + name
self.mse_train = []
self.mse_val = []
self.last_epoch = 0
size_inpt = 200
self.obs = tf.placeholder(tf.float32, shape=(None, size_obs))
self.ret = tf.placeholder(tf.float32, shape=(None))
act_trn = self.obs
act_tst = self.obs
prev_layer_size = size_obs
#Hidden layers
self.l2_reg = 1e-8
self.Q_lr = tf.placeholder(tf.float32, shape=(None))
self.lr = tf.placeholder(tf.float32, shape=(None))
if 1:
for idx, l in enumerate(net_struct):
act_trn, act_tst = ops.cascade_bn_relu_trn_tst(
act_trn, prev_layer_size, l, name='layer' + str(idx), input_tst = act_tst)
prev_layer_size += l
w = tf.Variable(tf.random_uniform([prev_layer_size, size_act],minval = -1., maxval = 1.), name='net_output_w') * 1e-3
b = tf.Variable(tf.random_uniform([size_act],minval = -1., maxval = 1.), name='net_output_bias') * 1e-3
else:
for idx, l in enumerate(net_struct):
act_trn = ops.linear(act_trn, l, 'layer' + str(idx))
w = tf.Variable(tf.random_uniform([l, size_act],minval = -1., maxval = 1.), name='net_output_w') * 1e-2
b = tf.Variable(tf.random_uniform([size_act],minval = -1., maxval = 1.), name='net_output_bias') * 1e-2
self.yhat = tf.reshape(tf.matmul(act_trn, w) + b, [-1, size_act])
self.yhat_tst = tf.reshape(tf.matmul(act_tst, w) + b, [-1, size_act])
self.obs_act = tf.concat((self.obs, self.yhat),1)
self.Q = Q(size_obs + size_act, tf.stop_gradient(self.obs_act))
self.act = tf.placeholder(tf.float32, shape=(None))
self.l2_loss = tf.reduce_mean(tf.square(self.yhat - self.act))
self.adv_loss = tf.reduce_mean(tf.square(self.yhat_tst - self.act))
#-1*tf.gather_nd(output_tst, self.y_raw, axis=1)output_tst[list(np.arange(bs)),self.y_raw]
self.advers = tf.gradients(self.l2_loss, self.obs)
t_vars = tf.trainable_variables()
net_vars = [var for var in t_vars if 'net_' in var.name]
self.reg_loss = tf.reduce_sum([tf.reduce_sum(tf.square(var)) for var in net_vars])*self.l2_reg
optimizer = tf.train.AdamOptimizer(learning_rate=self.lr)
gvs = optimizer.compute_gradients(self.l2_loss + self.reg_loss - self.Q.yhat * self.Q_lr + self.Q.l2_loss)
self.grad_norm = tf.reduce_mean([tf.reduce_mean(grad) for grad, var in gvs if grad is not None])
clip_norm = 100
clip_single = 1
capped_gvs = [(tf.clip_by_value(grad, -1*clip_single,clip_single), var) for grad, var in gvs if grad is not None]
capped_gvs = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in capped_gvs if grad is not None]
self.optimizer = optimizer.apply_gradients(capped_gvs)
#self.optimizer = tf.train.AdamOptimizer(self.lr).minimize(self.l2_loss)
self.cur_Q_lr = 0
self.session = tf.Session()
self.session.run(tf.global_variables_initializer())
self.Saver = tf.train.Saver()
