def discriminator(self, image, t_text_embedding, reuse=False): if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu(ops.conv2d(image, self.options['df_dim'], name = 'd_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, self.options['df_dim']*2, name = 'd_h1_conv'))) #16 h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, self.options['df_dim']*4, name = 'd_h2_conv'))) #8 h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, self.options['df_dim']*8, name = 'd_h3_conv'))) #4 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1,4,4,1], name='tiled_embeddings') h3_concat = tf.concat( 3, [h3, tiled_embeddings], name='h3_concat') h3_new = ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,1,1,1, name = 'd_h3_conv_new'))) #4 h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4), h4
def discriminator(self, image, t_text_embedding): update_collection=tf.GraphKeys.UPDATE_OPS with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE): h0 = ops.lrelu(ops.conv2d_sn(image, self.options['df_dim'], spectral_normed=True, update_collection=update_collection, name = 'd_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1(ops.conv2d_sn(h0, self.options['df_dim']*2, spectral_normed=True, update_collection=update_collection, name = 'd_h1_conv'))) #16 h2 = ops.lrelu( self.d_bn2(ops.conv2d_sn(h1, self.options['df_dim']*4, spectral_normed=True, update_collection=update_collection, name = 'd_h2_conv'))) #8 h3 = ops.lrelu( self.d_bn3(ops.conv2d_sn(h2, self.options['df_dim']*8, spectral_normed=True, update_collection=update_collection, name = 'd_h3_conv'))) #4 h3_new = ops.lrelu( self.d_bn4(ops.conv2d_sn(h3, self.options['df_dim']*8, 1,1,1,1, spectral_normed=True, update_collection=update_collection, name = 'd_h3_conv_new'))) #4 h3_new = tf.reshape(h3_new, [self.options['batch_size'], -1]) image_embedding = ops.linear(h3_new, self.options['t_dim'], 'd_h3_embedding') # Auxiliary classifier class_logit = ops.linear(image_embedding, self.options['num_class'], 'd_image_classifier') # Embedding matrix of condition reduced_text_embeddings = ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding') # Scalar output function h4 = ops.linear(image_embedding, 1, 'd_scalar_output') discriminator_output_logit = tf.reduce_sum(tf.multiply(reduced_text_embeddings, image_embedding), 1, keepdims=True) + h4 return discriminator_output_logit, discriminator_output_logit, class_logit
def generator(self, t_z, t_text_embedding): s = self.options['image_size'] s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding') ) z_concat = tf.concat(1, [t_z, reduced_text_embedding]) z_ = ops.linear(z_concat, self.options['gf_dim']*8*s16*s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0)) h1 = ops.deconv2d(h0, [self.options['batch_size'], s8, s8, self.options['gf_dim']*4], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1)) h2 = ops.deconv2d(h1, [self.options['batch_size'], s4, s4, self.options['gf_dim']*2], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2)) h3 = ops.deconv2d(h2, [self.options['batch_size'], s2, s2, self.options['gf_dim']*1], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') return (tf.tanh(h4)/2. + 0.5)
def generator(self, t_z, t_text_embedding): s = self.options['image_size'] s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding') ) z_concat = tf.concat([t_z, reduced_text_embedding], 1) z_ = ops.linear(z_concat, self.options['gf_dim']*8*s16*s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0)) h1 = ops.deconv2d(h0, [self.options['batch_size'], s8, s8, self.options['gf_dim']*4], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1)) h2 = ops.deconv2d(h1, [self.options['batch_size'], s4, s4, self.options['gf_dim']*2], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2)) h3 = ops.deconv2d(h2, [self.options['batch_size'], s2, s2, self.options['gf_dim']*1], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') return (tf.tanh(h4)/2. + 0.5)
def sampler(self, t_z, t_text_embedding): tf.get_variable_scope().reuse_variables() s = self.options['image_size'] s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding')) #z_concat = tf.concat(1, [t_z, reduced_text_embedding]) z_concat = t_text_embedding z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0, train=False)) h1 = ops.deconv2d( h0, [self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1, train=False)) h2 = ops.deconv2d( h1, [self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2, train=False)) h3 = ops.deconv2d( h2, [self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3, train=False)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') return (tf.tanh(h4) / 2. + 0.5)
def discriminator(self, image, t_text_embedding, reuse=False): if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu( ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1( ops.conv2d(h0, self.options['df_dim'] * 2, name='d_h1_conv'))) #16 h2 = ops.lrelu( self.d_bn2( ops.conv2d(h1, self.options['df_dim'] * 4, name='d_h2_conv'))) #8 h3 = ops.lrelu( self.d_bn3( ops.conv2d(h2, self.options['df_dim'] * 8, name='d_h3_conv'))) #4 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings =\ ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2) tiled_embeddings =\ tf.tile(reduced_text_embeddings, [1,4,4,1], name='tiled_embeddings') h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_new =\ ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1, 1, 1, 1, name='d_h3_conv_new'))) #4 h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4), h4
def discriminator(self, image, reuse=False): if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu( ops.conv2d(image, 64, name = 'd_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, 128, name = 'd_h1_conv'))) #32 h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, 256, name = 'd_h2_conv'))) #32 h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, 512, name = 'd_h3_conv'))) #32 h4 = ops.linear(tf.reshape(h3, [self.options['batch_size'], -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4), h4
def discriminator(self, image, t_text_embedding): with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE): h0 = ops.lrelu(ops.conv2d(image, self.options['df_dim'], name = 'd_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, self.options['df_dim']*2, name = 'd_h1_conv'))) #16 h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, self.options['df_dim']*4, name = 'd_h2_conv'))) #8 h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, self.options['df_dim']*8, name = 'd_h3_conv'))) #4 h4 = ops.linear(tf.reshape(h3, [self.options['batch_size'], -1]), 1, 'd_h3_lin') # ADD TEXT EMBEDDING TO THE NETWORK #reduced_text_embeddings = ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) #reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,1) #reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,2) #tiled_embeddings = tf.tile(reduced_text_embeddings, [1,4,4,1], name='tiled_embeddings') #h3_concat = tf.concat( 3, [h3, tiled_embeddings], name='h3_concat') #h3_new = ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,1,1,1, name = 'd_h3_conv_new'))) #4 #h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') return h4, h4
def decoder(self, code): hidden = ops.linear(code, self.options['caption_vector_length'], 'g_dec_h1') reconstruct_code = ops.linear(code, self.options['caption_vector_length'], 'g_dec_h2') return reconstruct_code
def discriminator(self, image, t_text_embedding): update_collection = tf.GraphKeys.UPDATE_OPS with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE): h0 = ops.lrelu( ops.conv2d_sn(image, self.options['df_dim'], spectral_normed=True, update_collection=update_collection, name='d_h0_conv')) #32 h1 = ops.lrelu( self.d_bn1( ops.conv2d_sn(h0, self.options['df_dim'] * 2, spectral_normed=True, update_collection=update_collection, name='d_h1_conv'))) #16 h2 = ops.lrelu( self.d_bn2( ops.conv2d_sn(h1, self.options['df_dim'] * 4, spectral_normed=True, update_collection=update_collection, name='d_h2_conv'))) #8 h3 = ops.lrelu( self.d_bn3( ops.conv2d_sn(h2, self.options['df_dim'] * 8, spectral_normed=True, update_collection=update_collection, name='d_h3_conv'))) #4 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1], name='tiled_embeddings') h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_new = ops.lrelu( self.d_bn4( ops.conv2d_sn(h3_concat, self.options['df_dim'] * 8, 1, 1, 1, 1, spectral_normed=True, update_collection=update_collection, name='d_h3_conv_new'))) #4 h4 = ops.linear( tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') # Auxiliary classifier class_logit = ops.linear( tf.reshape(h3_new, [self.options['batch_size'], -1]), self.options['num_class'], 'd_image_classifier') return h4, h4, class_logit
def discriminator(self, image, t_text_embedding, reuse=False): if reuse: with tf.variable_scope(tf.get_variable_scope(), reuse=True): h0 = ops.lrelu( ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) #32,48 h1 = ops.lrelu( self.d_bn1( ops.conv2d(h0, self.options['df_dim'] * 2, name='d_h1_conv'))) #16,24 h2 = ops.lrelu( self.d_bn2( ops.conv2d(h1, self.options['df_dim'] * 4, name='d_h2_conv'))) #8,12 h3 = ops.lrelu( self.d_bn3( ops.conv2d(h2, self.options['df_dim'] * 8, name='d_h3_conv'))) #4,6 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims( reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims( reduced_text_embeddings, 2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1], name='tiled_embeddings') # h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_concat = tf.concat(values=[h3, tiled_embeddings], axis=3, name='h3_concat') h3_new = ops.lrelu( self.d_bn4( ops.conv2d(h3_concat, self.options['df_dim'] * 8, 1, 1, 1, 1, name='d_h3_conv_new'))) #4 h4 = ops.linear( tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') # return tf.nn.sigmoid(h4), h4 else: h0 = ops.lrelu( ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) #32,48 h1 = ops.lrelu( self.d_bn1( ops.conv2d(h0, self.options['df_dim'] * 2, name='d_h1_conv'))) #16,24 h2 = ops.lrelu( self.d_bn2( ops.conv2d(h1, self.options['df_dim'] * 4, name='d_h2_conv'))) #8,12 h3 = ops.lrelu( self.d_bn3( ops.conv2d(h2, self.options['df_dim'] * 8, name='d_h3_conv'))) #4,6 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1], name='tiled_embeddings') # h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_concat = tf.concat(values=[h3, tiled_embeddings], axis=3, name='h3_concat') h3_new = ops.lrelu( self.d_bn4( ops.conv2d(h3_concat, self.options['df_dim'] * 8, 1, 1, 1, 1, name='d_h3_conv_new'))) #4 h4 = ops.linear( tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') # return tf.nn.sigmoid(h4), h4 return tf.nn.sigmoid(h4), h4 # if reuse: # tf.get_variable_scope().reuse_variables() # h0 = ops.lrelu(ops.conv2d(image, self.options['df_dim'], name = 'd_h0_conv')) #32,48 # h1 = ops.lrelu( self.d_bn1(ops.conv2d(h0, self.options['df_dim']*2, name = 'd_h1_conv'))) #16,24 # h2 = ops.lrelu( self.d_bn2(ops.conv2d(h1, self.options['df_dim']*4, name = 'd_h2_conv'))) #8,12 # h3 = ops.lrelu( self.d_bn3(ops.conv2d(h2, self.options['df_dim']*8, name = 'd_h3_conv'))) #4,6 # # ADD TEXT EMBEDDING TO THE NETWORK # reduced_text_embeddings = ops.lrelu(ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) # reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,1) # reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings,2) # tiled_embeddings = tf.tile(reduced_text_embeddings, [1,6,6,1], name='tiled_embeddings') # # h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') # h3_concat = tf.concat(values=[h3, tiled_embeddings], axis=3, name='h3_concat') # h3_new = ops.lrelu( self.d_bn4(ops.conv2d(h3_concat, self.options['df_dim']*8, 1,1,1,1, name = 'd_h3_conv_new'))) #4 # h4 = ops.linear(tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') # return tf.nn.sigmoid(h4), h4
def discriminator(self, image, t_text_embedding): with tf.variable_scope("discriminator", reuse=tf.AUTO_REUSE): if self.options['vgg']: h0 = ops.lrelu( ops.conv2d(image, 3, stride=1, name='d_h0a_conv')) h0 = ops.lrelu(ops.conv2d(h0, 3, stride=2, name='d_h0_conv')) h1 = ops.lrelu( self.d_bn1a( ops.conv2d(h0, self.options['df_dim'] * 2, stride=1, name='d_h1a_conv'))) h1 = ops.lrelu( self.d_bn1( ops.conv2d(h0, self.options['df_dim'] * 2, stride=2, name='d_h1_conv'))) h2 = ops.lrelu( self.d_bn2a( ops.conv2d(h1, self.options['df_dim'] * 4, stride=1, name='d_h2a_conv'))) h2 = ops.lrelu( self.d_bn2( ops.conv2d(h2, self.options['df_dim'] * 4, stride=2, name='d_h2_conv'))) h3 = ops.lrelu( self.d_bn3a( ops.conv2d(h2, self.options['df_dim'] * 8, stride=1, name='d_h3a_conv'))) h3 = ops.lrelu( self.d_bn3( ops.conv2d(h3, self.options['df_dim'] * 8, stride=2, name='d_h3_conv'))) else: if self.options['extra_64']: image = ops.lrelu( ops.conv2d(image, 3, stride=1, name='d_h0a_conv')) h0 = ops.lrelu( ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) # 32 if self.options['extra_32']: h0 = ops.lrelu( self.d_bn1a( ops.conv2d(h0, self.options['df_dim'], stride=1, name='d_h1a_conv'))) h1 = ops.lrelu( self.d_bn1( ops.conv2d(h0, self.options['df_dim'] * 2, name='d_h1_conv'))) # 16 h2 = ops.lrelu( self.d_bn2( ops.conv2d(h1, self.options['df_dim'] * 4, name='d_h2_conv'))) # 8 h3 = ops.lrelu( self.d_bn3( ops.conv2d(h2, self.options['df_dim'] * 8, name='d_h3_conv'))) # 4 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1, 4, 4, 1], name='tiled_embeddings') h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_new = ops.lrelu( self.d_bn4( ops.conv2d(h3_concat, self.options['df_dim'] * 8, 1, 1, 1, 1, name='d_h3_conv_new'))) # 4 h4 = ops.linear( tf.reshape(h3_new, [self.options['batch_size'], -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4), h4
def generator(self, t_z, t_text_embedding): s = self.options['image_size'] #64 x 64 s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding')) #self.options['t_dim', 256] z_concat = tf.concat([t_z, reduced_text_embedding], 1) #t_z is batch_size, z_dim, which is 100 z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16, 'g_h0_lin') h0 = tf.reshape(z_, [ -1, s16, s16, self.options['gf_dim'] * 8 ]) #[-1, 4, 4, 64 * 8] gf_dim is number of filters in the first layer h0 = tf.nn.relu(self.g_bn0(h0)) if self.options['vgg']: h1 = ops.deconv2d(h0, [ self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4 ], stride=2, name='g_h1') #8 h1 = tf.nn.relu(self.g_bn1(h1)) h1 = ops.deconv2d(h1, [ self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4 ], stride=1, name='g_h1b') h1 = tf.nn.relu(self.g_bn1b(h1)) h2 = ops.deconv2d(h1, [ self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2 ], stride=2, name='g_h2') #16 h2 = tf.nn.relu(self.g_bn2(h2)) h2 = ops.deconv2d(h2, [ self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2 ], stride=1, name='g_h2b') h2 = tf.nn.relu(self.g_bn2b(h2)) h3 = ops.deconv2d(h2, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], stride=2, name='g_h3') #32 h3 = tf.nn.relu(self.g_bn3(h3)) h3 = ops.deconv2d(h3, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], stride=1, name='g_h3b') h3 = tf.nn.relu(self.g_bn3b(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], stride=2, name='g_h4') #64 h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3], stride=1, name='g_h4b') return (tf.tanh(h4) / 2. + 0.5) else: h1 = ops.deconv2d(h0, [ self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4 ], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1)) h2 = ops.deconv2d(h1, [ self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2 ], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2)) h3 = ops.deconv2d(h2, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3)) if self.options['extra_32']: h3 = ops.deconv2d(h3, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], stride=1, name='g_h3b') h3 = tf.nn.relu(self.g_bn3b(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') if self.options['extra_64']: h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3], stride=1, name='g_h4b') return (tf.tanh(h4) / 2. + 0.5)
def sampler(self, t_z, t_text_embedding): tf.get_variable_scope().reuse_variables() s = self.options['image_size'] s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16) reduced_text_embedding = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding')) z_concat = tf.concat([t_z, reduced_text_embedding], 1) z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0, train=False)) if self.options['vgg']: h1 = ops.deconv2d(h0, [ self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4 ], stride=2, name='g_h1') #8 h1 = tf.nn.relu(self.g_bn1(h1, train=False)) h1 = ops.deconv2d(h1, [ self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4 ], stride=1, name='g_h1b') h1 = tf.nn.relu(self.g_bn1b(h1, train=False)) h2 = ops.deconv2d(h1, [ self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2 ], stride=2, name='g_h2') #16 h2 = tf.nn.relu(self.g_bn2(h2, train=False)) h2 = ops.deconv2d(h2, [ self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2 ], stride=1, name='g_h2b') h2 = tf.nn.relu(self.g_bn2b(h2, train=False)) h3 = ops.deconv2d(h2, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], stride=2, name='g_h3') #32 h3 = tf.nn.relu(self.g_bn3(h3, train=False)) h3 = ops.deconv2d(h3, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], stride=1, name='g_h3b') h3 = tf.nn.relu(self.g_bn3b(h3, train=False)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], stride=2, name='g_h4') #64 h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3], stride=1, name='g_h4b') return (tf.tanh(h4) / 2. + 0.5) else: h1 = ops.deconv2d(h0, [ self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4 ], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1, train=False)) h2 = ops.deconv2d(h1, [ self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2 ], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2, train=False)) h3 = ops.deconv2d(h2, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3, train=False)) if self.options['extra_32']: h3 = ops.deconv2d(h3, [ self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1 ], stride=1, name='g_h3b') h3 = tf.nn.relu(self.g_bn3b(h3, train=False)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') if self.options['extra_64']: h4 = ops.deconv2d(h4, [self.options['batch_size'], s, s, 3], stride=1, name='g_h4b') return (tf.tanh(h4) / 2. + 0.5)
def discriminator(self, image, t_text_embedding, n_classes, t_training, reuse=False): if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu( ops.conv2d(image, self.options['df_dim'], name='d_h0_conv')) # 64 h1 = ops.lrelu( slim.batch_norm(ops.conv2d(h0, self.options['df_dim'] * 2, name='d_h1_conv'), reuse=reuse, is_training=t_training, scope='d_bn1')) # 32 h2 = ops.lrelu( slim.batch_norm(ops.conv2d(h1, self.options['df_dim'] * 4, name='d_h2_conv'), reuse=reuse, is_training=t_training, scope='d_bn2')) # 16 h3 = ops.lrelu( slim.batch_norm(ops.conv2d(h2, self.options['df_dim'] * 8, name='d_h3_conv'), reuse=reuse, is_training=t_training, scope='d_bn3')) # 8 h3_shape = h3.get_shape().as_list() # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu( ops.linear(t_text_embedding, self.options['t_dim'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2) tiled_embeddings = tf.tile(reduced_text_embeddings, [1, h3_shape[1], h3_shape[1], 1], name='tiled_embeddings') h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_new = ops.lrelu( slim.batch_norm(ops.conv2d(h3_concat, self.options['df_dim'] * 8, 1, 1, 1, 1, name='d_h3_conv_new'), reuse=reuse, is_training=t_training, scope='d_bn4')) # 4 h3_flat = tf.reshape(h3_new, [self.options['batch_size'], -1]) h4 = ops.linear(h3_flat, 1, 'd_h4_lin_rw') h4_aux = ops.linear(h3_flat, n_classes, 'd_h4_lin_ac') return tf.nn.sigmoid(h4), h4, tf.nn.sigmoid(h4_aux), h4_aux
def generator(self, word2vec, reuse=False): with tf.variable_scope("generator") as scope: if reuse: scope.reuse_variables() filter_sizes = [3, 4, 5] embedding_size = 400 num_filters = 800 sequence_length = 15 embedded_chars_expanded = tf.expand_dims(word2vec, -1) # ?xlengthxfeaturex1 # Create a convolution + maxpool layer for each filter size pooled_outputs = [] for i, filter_size in enumerate(filter_sizes): #with tf.name_scope("conv-maxpool-%s" % filter_size): # Convolution Layer filter_shape = [filter_size, embedding_size, 1, num_filters] # W = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1), name="W") # b = tf.Variable(tf.constant(0.1, shape=[num_filters]), name="b") if i == 0: h = tf.nn.relu(self.gbn0( ops.conv2dv1(embedded_chars_expanded, num_filters, filter_shape, name='g_h%d_conv' % i))) # 16 elif i == 1: h = tf.nn.relu(self.gbn1( ops.conv2dv1(embedded_chars_expanded, num_filters, filter_shape, name='g_h%d_conv' % i))) # 16 else: h = tf.nn.relu(self.gbn2( ops.conv2dv1(embedded_chars_expanded, num_filters, filter_shape, name='g_h%d_conv' % i))) # 16 # conv = tf.nn.conv2d( # self.embedded_chars_expanded, # W, # strides=[1, 1, 1, 1], # padding="VALID", # name="conv") # Apply nonlinearity #h = tf.nn.relu(tf.nn.bias_add(conv, b), name="relu") # Maxpooling over the outputs pooled = tf.nn.max_pool( h, ksize=[1, sequence_length - filter_size + 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID', name="pool") pooled_outputs.append(pooled) # Combine all the pooled features num_filters_total = num_filters * len(filter_sizes) h_pool = tf.concat(pooled_outputs, 3) h_pool_flat = tf.reshape(h_pool, [-1, num_filters_total]) s = self.options['image_size'] s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16) reduced_text_embedding = ops.lrelu(ops.linear(h_pool_flat, self.options['t_dim'], 'g_embedding')) z_ = ops.linear(reduced_text_embedding, self.options['gf_dim'] * 8 * s16 * s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8]) h0 = tf.nn.relu(self.g_bn0(h0)) h1 = ops.deconv2d(h0, [self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4], name='g_h1') h1 = tf.nn.relu(self.g_bn1(h1)) h2 = ops.deconv2d(h1, [self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2], name='g_h2') h2 = tf.nn.relu(self.g_bn2(h2)) h3 = ops.deconv2d(h2, [self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1], name='g_h3') h3 = tf.nn.relu(self.g_bn3(h3)) h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3], name='g_h4') return (tf.tanh(h4) / 2. + 0.5), h_pool_flat
def __init__(self, size_obs, size_act, net_struct = [100, 100, 100, 100], name='dbg'): self.tensorboardpath = 'tensorboards/' + name self.train_writer = tf.summary.FileWriter(self.tensorboardpath) self.ModelPath = 'Models/Imitation' + name self.mse_train = [] self.mse_val = [] self.last_epoch = 0 size_inpt = 200 self.obs = tf.placeholder(tf.float32, shape=(None, size_obs)) self.ret = tf.placeholder(tf.float32, shape=(None)) act_trn = self.obs act_tst = self.obs prev_layer_size = size_obs #Hidden layers self.l2_reg = 1e-8 self.Q_lr = tf.placeholder(tf.float32, shape=(None)) self.lr = tf.placeholder(tf.float32, shape=(None)) if 1: for idx, l in enumerate(net_struct): act_trn, act_tst = ops.cascade_bn_relu_trn_tst( act_trn, prev_layer_size, l, name='layer' + str(idx), input_tst = act_tst) prev_layer_size += l w = tf.Variable(tf.random_uniform([prev_layer_size, size_act],minval = -1., maxval = 1.), name='net_output_w') * 1e-3 b = tf.Variable(tf.random_uniform([size_act],minval = -1., maxval = 1.), name='net_output_bias') * 1e-3 else: for idx, l in enumerate(net_struct): act_trn = ops.linear(act_trn, l, 'layer' + str(idx)) w = tf.Variable(tf.random_uniform([l, size_act],minval = -1., maxval = 1.), name='net_output_w') * 1e-2 b = tf.Variable(tf.random_uniform([size_act],minval = -1., maxval = 1.), name='net_output_bias') * 1e-2 self.yhat = tf.reshape(tf.matmul(act_trn, w) + b, [-1, size_act]) self.yhat_tst = tf.reshape(tf.matmul(act_tst, w) + b, [-1, size_act]) self.obs_act = tf.concat((self.obs, self.yhat),1) self.Q = Q(size_obs + size_act, tf.stop_gradient(self.obs_act)) self.act = tf.placeholder(tf.float32, shape=(None)) self.l2_loss = tf.reduce_mean(tf.square(self.yhat - self.act)) self.adv_loss = tf.reduce_mean(tf.square(self.yhat_tst - self.act)) #-1*tf.gather_nd(output_tst, self.y_raw, axis=1)output_tst[list(np.arange(bs)),self.y_raw] self.advers = tf.gradients(self.l2_loss, self.obs) t_vars = tf.trainable_variables() net_vars = [var for var in t_vars if 'net_' in var.name] self.reg_loss = tf.reduce_sum([tf.reduce_sum(tf.square(var)) for var in net_vars])*self.l2_reg optimizer = tf.train.AdamOptimizer(learning_rate=self.lr) gvs = optimizer.compute_gradients(self.l2_loss + self.reg_loss - self.Q.yhat * self.Q_lr + self.Q.l2_loss) self.grad_norm = tf.reduce_mean([tf.reduce_mean(grad) for grad, var in gvs if grad is not None]) clip_norm = 100 clip_single = 1 capped_gvs = [(tf.clip_by_value(grad, -1*clip_single,clip_single), var) for grad, var in gvs if grad is not None] capped_gvs = [(tf.clip_by_norm(grad, clip_norm), var) for grad, var in capped_gvs if grad is not None] self.optimizer = optimizer.apply_gradients(capped_gvs) #self.optimizer = tf.train.AdamOptimizer(self.lr).minimize(self.l2_loss) self.cur_Q_lr = 0 self.session = tf.Session() self.session.run(tf.global_variables_initializer()) self.Saver = tf.train.Saver()