def create_cond_critic_proj(cls, xinput, input_clusters, var_scope, critic_layers, clusters_no, reuse=None): """ Class method that instantiates a Critic and creates a conditional critic with the original projection conditioning method. Parameters ---------- xinput : Tensor Tensor containing the input cells. input_clusters : Tensor Tensor containing the corresponding cluster indexes of the input cells. var_scope : str Variable scope used for the created tensors. critic_layers : list List of integers corresponding to the number of neurons of each layer of the critic. clusters_no : int Number of clusters. reuse : Boolean Whether to reuse the already existing Tensors. Default is None. Returns ------- A Creator object with the defined architecture. """ with tf.variable_scope(var_scope, reuse=reuse): for i_lay, output_size in enumerate(critic_layers): with tf.variable_scope("layers_" + str(i_lay + 1)): xinput = layers.relu( xinput, output_size, weights_initializer=layers.variance_scaling_initializer(mode="FAN_AVG"), biases_initializer=tf.zeros_initializer()) with tf.variable_scope("layers_" + 'proj'): proj_weights_m = tf.get_variable( "proj_weights_m", [clusters_no, critic_layers[-1], 1], dtype=tf.float32, initializer=layers.xavier_initializer()) proj_weights = tf.nn.embedding_lookup(proj_weights_m, input_clusters) output_proj = tf.einsum('ij,ijk->ik', xinput, proj_weights) with tf.variable_scope("layers_" + 'output'): output = layers.linear( xinput, 1, weights_initializer=layers.xavier_initializer(), biases_initializer=tf.zeros_initializer()) dist = tf.add(output_proj, output) return cls(xinput, dist, var_scope, critic_layers, input_clusters=input_clusters, clusters_no=clusters_no, reuse=reuse)
def create_MLP(self, name, output_dim, hidden_sizes, hidden_W_init=tf_layers.xavier_initializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=tf_layers.xavier_initializer(), output_b_init=tf.zeros_initializer(), weight_normalization=False, ): all_params = OrderedDict() cur_shape = self.input_shape with tf.variable_scope(name): for idx, hidden_size in enumerate(hidden_sizes): W, b, cur_shape = make_dense_layer( cur_shape, num_units=hidden_size, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_norm=weight_normalization, ) all_params['W' + str(idx)] = W all_params['b' + str(idx)] = b W, b, _ = make_dense_layer( cur_shape, num_units=output_dim, name='output', W=output_W_init, b=output_b_init, weight_norm=weight_normalization, ) all_params['W' + str(len(hidden_sizes))] = W all_params['b' + str(len(hidden_sizes))] = b return all_params
def position_wise_feed_forward_mine(inputs, num_units1=2048, num_units2=512, reuse=None): with tf.variable_scope("feed-forward-networks"): W1 = tf.get_variable("weight1", [inputs.get_shape()[-1], num_units1], initializer=xavier_initializer()) b1 = tf.get_variable('bias1', [num_units1], initializer=tf.constant_initializer(0.1)) outputs = tf.einsum('aij,jk->aik', inputs, W1) + b1 # [batch, length_q, num_units1] W2 = tf.get_variable("weight1", [outputs.get_shape()[-1], num_units2], initializer=xavier_initializer()) b2 = tf.get_variable('bias1', [num_units2], initializer=tf.constant_initializer(0.1)) outputs = tf.einsum('aij,jk->aik', inputs, W2) + b2 # [batch, length_q, num_units1] # residual connection outputs += inputs # Normalize outputs = tf.layers.batch_normalization(outputs) return outputs
def create_MLP( self, name, output_dim, hidden_sizes, hidden_W_init=tf_layers.xavier_initializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=tf_layers.xavier_initializer(), output_b_init=tf.zeros_initializer(), weight_normalization=False, ): all_params = OrderedDict() cur_shape = self.input_shape with tf.variable_scope(name): for idx, hidden_size in enumerate(hidden_sizes): W, b, cur_shape = make_dense_layer( cur_shape, num_units=hidden_size, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_norm=weight_normalization, ) all_params['W' + str(idx)] = W all_params['b' + str(idx)] = b all_params['W' + str(idx) + '_stepsize'] = tf.Variable( self.init_flr_full * tf.ones_like(W), name="W" + str(idx) + '_stepsize') all_params['b' + str(idx) + '_stepsize'] = tf.Variable( self.init_flr_full * tf.ones_like(b), name="b" + str(idx) + '_stepsize') W, b, _ = make_dense_layer( cur_shape, num_units=output_dim, name='output', W=output_W_init, b=output_b_init, weight_norm=weight_normalization, ) all_params['W' + str(len(hidden_sizes))] = W all_params['b' + str(len(hidden_sizes))] = b all_params['W' + str(len(hidden_sizes)) + '_stepsize'] = tf.Variable( self.init_flr_full * tf.ones_like(W), name="W" + str(len(hidden_sizes)) + '_stepsize') all_params['b' + str(len(hidden_sizes)) + '_stepsize'] = tf.Variable( self.init_flr_full * tf.ones_like(b), name="b" + str(len(hidden_sizes)) + '_stepsize') all_params['bias_transformation'] = tf.Variable( tf.ones(self.latent_dim), name="bias_transformation") all_params['bias_transformation_stepsize'] = tf.Variable( self.init_flr_full * tf.ones_like(all_params['bias_transformation']), name="bias_transformation_stepsize") return all_params
def _linear(self, input_tensor, output_nums, l2_reg, activation_fn=None): if l2_reg <= 0: return layers.fully_connected(input_tensor, output_nums, activation_fn=activation_fn, weights_initializer=layers.xavier_initializer(), biases_initializer=layers.xavier_initializer(),) else: return layers.fully_connected(input_tensor, output_nums, activation_fn=activation_fn, weights_initializer=layers.xavier_initializer(), biases_initializer=layers.xavier_initializer(), weights_regularizer=layers.l2_regularizer(l2_reg), biases_regularizer=layers.l2_regularizer(l2_reg))
def create_MLP( self, name, output_dim, latent_dim, num_total_tasks, hidden_sizes, hidden_W_init=tf_layers.xavier_initializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=tf_layers.xavier_initializer(), output_b_init=tf.zeros_initializer(), weight_normalization=False, ): all_params = OrderedDict() cur_shape = self.input_shape with tf.variable_scope(name): for idx, hidden_size in enumerate(hidden_sizes): W, b, cur_shape = make_dense_layer( cur_shape, num_units=hidden_size, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_norm=weight_normalization, ) all_params['W' + str(idx)] = W all_params['b' + str(idx)] = b W, b, _ = make_dense_layer( cur_shape, num_units=output_dim, name='output', W=output_W_init, b=output_b_init, weight_norm=weight_normalization, ) all_params['W' + str(len(hidden_sizes))] = W all_params['b' + str(len(hidden_sizes))] = b all_params['latent_means'] = tf.get_variable( "latent_means", shape=(num_total_tasks, latent_dim), initializer=tf.random_normal_initializer) all_params['latent_stds'] = tf.get_variable( "latent_stds", shape=(num_total_tasks, latent_dim), initializer=tf.zeros_initializer) all_params['latent_means_stepsize'] = tf.Variable( self.step_size * tf.ones((latent_dim, )), name="latent_means_stepsize") all_params['latent_stds_stepsize'] = tf.Variable( self.step_size * tf.ones((latent_dim, )), name="latent_stds_stepsize") return all_params
def embedding(inputs, vocab_size, num_units, zero_pad=True, scale=True, reuse=None): """ :param inputs: A `Tensor` with type `int32` or `int64` containing the ids to be looked up in `lookup table`. shape is [batch, sentence_len] :param vocab_size: vocabulary size :param num_units: Number of embedding hidden units. in the paper, it is called d_model :param zero_pad: If True, all the values of the fist row (id 0) should be constant zeros. :param scale: If True. the outputs is multiplied by sqrt num_units. :param reuse: Boolean, whether to reuse the weights of a previous layer by the same name. :return: A `Tensor` with one more rank than inputs's. The last dimensionality should be `num_units`. """ with tf.variable_scope("embedding-layer", reuse=reuse): embedding = tf.get_variable("embedding", [vocab_size, num_units], initializer=xavier_initializer()) if zero_pad: embedding = tf.concat([tf.zeros([1, num_units]), embedding[1:, :]], axis=0) # index=0 for nil word output = tf.nn.embedding_lookup(embedding, inputs) # [batch, sentence_len, num_units] if scale: output = output * np.sqrt(num_units) return output
def gen_net(self, z, y): with tf.variable_scope('generator') as scope: yb = tf.reshape(y, shape=[self.batch_size, 1, 1, self.y_dim]) #Reshape the input noise(the precondition of the CGAN into a shape 64x1x1x10) z = tf.concat([z, y], 1) c1, c2 = int( self.output_size / 4), int(self.output_size / 2 ) # 10 stand for the num of labels d1 = tf.nn.relu(batch_normal(fully_connect(z, output_size=1024, scope='gen_fully'), scope='gen_bn1')) d1 = tf.concat([d1, y], 1) d2 = tf.nn.relu(batch_normal(fully_connect(d1, output_size=7*7*2*64, scope='gen_fully2'), scope='gen_bn2')) d2 = tf.reshape(d2, [self.batch_size, c1, c1, 64 * 2]) d2 = conv_cond_concat(d2, yb) d3 = tf.nn.relu(batch_normal(de_conv(d2, output_shape=[self.batch_size, c2, c2, 128], name='gen_deconv1'), scope='gen_bn3')) d3 = conv_cond_concat(d3, yb) d4 = de_conv(d3, output_shape=[self.batch_size, self.output_size, self.output_size, self.channel], name='gen_deconv2', initializer = xavier_initializer()) return tf.nn.sigmoid(d4)
def get_discriminator_net(self, name, images, reuse=False): batch_size = self.get_batch_size() with tf.variable_scope("discriminator", reuse=reuse) as scope: # concat yb = tf.reshape(self.labels, shape=[batch_size, 1, 1, -1]) net = ops.conv_cond_concat(images, yb) # TODO: why is output_dim is self.data.get_number_of_labels() here?? net, w1 = ops.conv2d( net, self.data.get_number_of_labels(), 4, 4, 2, 2, scope='dis_conv1', weights_initializer=self.get_weights_initializer()) tf.add_to_collection('weight_1', w1) net = ops.lrelu(net) net = ops.conv_cond_concat(net, yb) tf.add_to_collection('ac_1', net) net, w2 = ops.conv2d( net, batch_size, 4, 4, 2, 2, scope='dis_conv2', weights_initializer=self.get_weights_initializer()) tf.add_to_collection('weight_2', w2) net = ops.lrelu( ops.batch_norm(net, self.is_training, scope='dis_bn1')) tf.add_to_collection('ac_2', net) net = tf.reshape(net, [batch_size, -1]) net = tf.concat([net, self.labels], 1) # TODO: figure out what implications the change in "output_size" will have. net = ops.linear( net, output_size=1024, scope='dis_fully1', weights_initializer=self.get_weights_initializer()) net = ops.batch_norm(net, self.is_training, scope='dis_bn2', reuse=reuse) net = ops.lrelu(net) net = tf.concat([net, self.labels], 1) out = ops.linear(net, output_size=1, scope='dis_fully2', weights_initializer=xavier_initializer()) return tf.nn.sigmoid(out, name=name), out
def composite_model(frames, encoder_len=5, decoder_future_len=5, decoder_reconst_len=5, noise_std=0.0, uniform_init=True, num_channels=3, keep_prob_dropout=0.9, fc_conv_layer=True): """ Args: frames: 5D array of batch with videos - shape(batch_size, num_frames, frame_width, frame_higth, num_channels) encoder_len: number of frames that shall be encoded decoder_future_sequence_length: number of frames that shall be decoded from the hidden_repr noise_std: standard deviation of the gaussian noise to be added to the hidden representation uniform_init: specifies if the weight initialization should be drawn from gaussian or uniform distribution (default:uniform) num_channels: number of channels for generated frames fc_conv_layer: indicates whether fully connected layer shall be added between encoder and decoder Returns: frame_gen: array of generated frames (Tensors) """ assert all([len > 0 for len in [encoder_len, decoder_future_len, decoder_reconst_len]]) initializer = tf_layers.xavier_initializer(uniform=uniform_init) hidden_repr, mu, sigma = encoder_model(frames, encoder_len, initializer, fc_conv_layer=fc_conv_layer) # add noise if noise_std != 0.0: hidden_repr = hidden_repr + tf.random_normal(shape=hidden_repr.get_shape(), mean=0.0, stddev=noise_std, dtype=tf.float32) frames_pred = decoder_model(hidden_repr, decoder_future_len, initializer, num_channels=num_channels, keep_prob_dropout=keep_prob_dropout, scope='decoder_pred', fc_conv_layer=fc_conv_layer) frames_reconst = decoder_model(hidden_repr, decoder_reconst_len, initializer, num_channels=num_channels, keep_prob_dropout=keep_prob_dropout, scope='decoder_reconst', fc_conv_layer=fc_conv_layer) return frames_pred, frames_reconst, hidden_repr, mu, sigma
def dis_net(self, images, y, reuse=False): with tensorflow.variable_scope('discriminator') as scope: if reuse == True: scope.reuse_variables() yb = tensorflow.reshape(y, shape=[self.batch_size, 1, 1, self.y_dim]) concat_data = conv_cond_concat(images, yb) conv1, w1 = conv2d(concat_data, output_dim=10, name='dis_conv1') tensorflow.add_to_collection('weight_1', w1) conv1 = lrelu(conv1) conv1 = conv_cond_concat(conv1, yb) tensorflow.add_to_collection('ac_1', conv1) conv2, w2 = conv2d(conv1, output_dim=64, name='dis_conv2') tensorflow.add_to_collection('weight_2', w2) conv2 = lrelu(batch_normal(conv2, scope='dis_bn1')) tensorflow.add_to_collection('ac_2', conv2) f1 = lrelu( batch_normal(fully_connected(conv2, output_size=1024, scope='dis_fully1'), scope='dis_bn2', reuse=reuse)) f1 = tensorflow.concat([f1, y], 1) out = fully_connected(f1, output_size=1, scope='dis_fully2', initializer=xavier_initializer()) return tensorflow.nn.sigmoid(out), out
def gern_net(self, z, y): with tensorflow.variable_scope('generator') as scope: yb = tensorflow.reshape(y, shape=[self.batch_size, 1, 1, self.y_dim]) z = tensorflow.concat([z, y], 1) c1, c2 = int(self.output_size / 4), int(self.output_size / 2) d1 = tensorflow.nn.relu( batch_normal(fully_connected(z, output_size=1024, scope='gen_fully1'), scope='gen_bn1')) d1 = tensorflow.concat([d1, y], 1) d2 = tensorflow.nn.relu( batch_normal(fully_connected(d1, output_size=7 * 7 * 2 * 64, scope='gen_fully2'), scope='gen_bn2')) d2 = tensorflow.reshape(d2, [self.batch_size, c1, c1, 64 * 2]) d2 = conv_cond_concat(d2, yb) d3 = tensorflow.nn.relu( batch_normal(de_conv2d( d2, output_shape=[self.batch_size, c2, c2, 128], name='gen_deconv1'), scope='gen_bn3')) d3 = conv_cond_concat(d3, yb) d4 = de_conv2d(d3, output_shape=[ self.batch_size, self.output_size, self.output_size, self.channels ], name='gen_deconv2', initializer=xavier_initializer()) return tensorflow.nn.sigmoid(d4)
def dis_net(self, feature_vector, y, reuse=False): with tf.variable_scope("discriminator") as scope: if reuse == True: scope.reuse_variables() # concat concat_data = tf.concat([feature_vector, y], 1) f1 = tf.nn.relu( batch_normal(fully_connect(concat_data, output_size=10, scope="dis_fully1"), scope="dis_bn1")) f1 = tf.concat([f1, y], 1) f2 = lrelu( batch_normal(fully_connect(f1, output_size=10, scope="dis_fully1"), scope="dis_bn1")) f2 = tf.concat([f2, y], 1) out = fully_connect(f2, output_size=1, scope='dis_fully2', initializer=xavier_initializer()) return tf.nn.sigmoid(out), out
def embedding_mine(vocab_size, num_units, zero_pad, scale, reuse=None): with tf.variable_scope("embedding", reuse=reuse): embedding = tf.get_variable("embedding", [vocab_size, num_units], initializer=xavier_initializer()) if zero_pad: embedding = tf.concat([tf.zeros([1, num_units]), embedding[1:, :]], axis=0) # index=0 for nil word if scale: embedding /= np.sqrt(num_units) return embedding
def get_generator_net(self, name): batch_size = self.get_batch_size() with tf.variable_scope('generator') as scope: z = tf.concat([self.noise, self.labels], 1) net = ops.linear( z, output_size=1024, scope='gen_fully1', weights_initializer=self.get_weights_initializer()) net = ops.batch_norm(net, self.is_training, scope='gen_bn1') net = ops.lrelu(net) net = tf.concat([net, self.labels], 1) # Wonder what this will be doing if the size is not divisible by 4 h, w = self.data.shape[0] // 4, self.data.shape[1] // 4 net = ops.linear( net, output_size=h * w * 2 * batch_size, scope='gen_fully2', weights_initializer=self.get_weights_initializer()) net = ops.batch_norm(net, self.is_training, scope='gen_bn2') net = ops.lrelu(net) net = tf.reshape(net, [batch_size, h, w, 2 * batch_size]) yb = tf.reshape(self.labels, shape=[batch_size, 1, 1, -1]) net = ops.conv_cond_concat(net, yb) h, w = self.data.shape[0] // 2, self.data.shape[1] // 2 net = ops.deconv2d( net, [batch_size, h, w, 2 * batch_size], 4, 4, 2, 2, scope='gen_deconv1', weights_initializer=self.get_weights_initializer()) net = ops.batch_norm(net, self.is_training, scope='gen_bn3') net = ops.lrelu(net) net = ops.conv_cond_concat(net, yb) out = ops.deconv2d(net, self.images.shape, 4, 4, 2, 2, scope='gen_deconv2', weights_initializer=xavier_initializer()) return tf.nn.sigmoid(out, name=name), out
def gern_net(self, z, y): with tf.variable_scope('generator') as scope: z = tf.concat([z, y], 1) d1 = tf.nn.relu( batch_normal(fully_connect(z, output_size=11, scope="gen_fully1"), scope="gen_bn1")) d2 = tf.nn.relu( batch_normal(fully_connect(d1, output_size=11, scope="gen_fully2"), scope="gen_bn2")) return fully_connect(d2, output_size=4, scope="gen_fully3", initializer=xavier_initializer())
def create_critic(cls, xinput, var_scope, critic_layers, reuse=None): """ Class method that instantiates a Critic and creates a non-conditional critic. Parameters ---------- xinput : Tensor Tensor containing the input cells. var_scope : str Variable scope used for the created tensors. critic_layers : list List of integers corresponding to the number of neurons of each layer of the critic. reuse : Boolean Whether to reuse the already existing Tensors. Default is None. Returns ------- A Creator object with the defined architecture. """ with tf.variable_scope(var_scope, reuse=reuse): for i_lay, output_size in enumerate(critic_layers): with tf.variable_scope("layers_" + str(i_lay + 1)): xinput = layers.relu( xinput, output_size, weights_initializer=layers. variance_scaling_initializer(mode="FAN_AVG"), biases_initializer=tf.zeros_initializer()) with tf.variable_scope("layers_" + 'output'): output = layers.linear( xinput, 1, weights_initializer=layers.xavier_initializer(), biases_initializer=tf.zeros_initializer()) return cls(xinput, output, var_scope, critic_layers, reuse=reuse)
def construct_weights(self): msra_init = lambda shape: tf_layers.xavier_initializer(uniform=True)( shape) create_W = lambda shape, name: tf.Variable( msra_init(shape), name=name, collections=[ tf.compat.v1.GraphKeys.GLOBAL_VARIABLES, tf.compat.v1.GraphKeys .WEIGHTS ]) create_b = lambda shape, name: tf.Variable(msra_init(shape), name=name) create_Wb = lambda inp, out, i: (create_W([inp, out], f'layer{i}W'), create_b([out], f'layer{i}b')) in_sizes = [self.n_in] + self.hidden out_sizes = self.hidden + [self.n_out] return [ create_Wb(in_size, out_size, i) for i, (in_size, out_size) in enumerate(zip(in_sizes, out_sizes)) ]
def composite_model(frames, encoder_len=5, decoder_future_len=5, decoder_reconst_len=5, uniform_init=True, num_channels=3, fc_conv_layer=True): """ Args: frames: 5D array of batch with videos - shape(batch_size, num_frames, frame_width, frame_higth, num_channels) encoder_len: number of frames that shall be encoded decoder_future_sequence_length: number of frames that shall be decoded from the hidden_repr uniform_init: specifies if the weight initialization should be drawn from gaussian or uniform distribution (default:uniform) num_channels: number of channels for generated frames fc_conv_layer: indicates whether fully connected layer shall be added between encoder and decoder Returns: frame_gen: array of generated frames (Tensors) """ assert all([len > 0 for len in [encoder_len, decoder_future_len, decoder_reconst_len]]) initializer = tf_layers.xavier_initializer(uniform=uniform_init) hidden_repr = encoder_model(frames, encoder_len, initializer, fc_conv_layer=fc_conv_layer) frames_pred = decoder_model(hidden_repr, decoder_future_len, initializer, num_channels=num_channels, scope='decoder_pred', fc_conv_layer=fc_conv_layer) frames_reconst = decoder_model(hidden_repr, decoder_reconst_len, initializer, num_channels=num_channels, scope='decoder_reconst', fc_conv_layer=fc_conv_layer) return frames_pred, frames_reconst, hidden_repr
def get_token_embeddings(vocab_size, num_units, zero_pad=True): '''Constructs token embedding matrix. Note that the column of index 0's are set to zeros. vocab_size: scalar. V. num_units: embedding dimensionalty. E. zero_pad: Boolean. If True, all the values of the first row (id = 0) should be constant zero To apply query/key masks easily, zero pad is turned on. Returns weight variable: (V, E) ''' with tf.variable_scope('shared_weight_matrix'): embeddings = tf.get_variable("weight_mat", dtype=tf.float32, shape=(vocab_size, num_units), initializer=xavier_initializer()) if zero_pad: embeddings = tf.concat( (tf.zeros(shape=[1, num_units]), embeddings[1:, :]), 0) return embeddings
def __init__(self): self.sequence_len = 30 self.num_sentences = 6 self.sentence_len = int( self.sequence_len / self.num_sentences) # the number of words of each sencentenc self.num_classes = 3 self.learning_rate = 0.01 self.batch_size = 8 self.decay_steps = 1000 self.decay_rate = 0.9 self.vocab_size = 10000 self.embed_size = 100 self.hidden_size = 100 self.is_training = True # self.need_sentence_level_attention_encoder_flag = need_sentence_level_attention_encoder_flag self.multi_label_flag = False self.clip_gradients = 5.0 self.cell_type = 'gru' self.dropout_keep_prob = 0.5 self.initializer = xavier_initializer() self.l2_lambda = 0.5
def build_graph_better(self): """Build the cnn graph. """ print('Building graph') with self.graph.as_default(): # Input data self.images = tf.placeholder(tf.float32, shape=(None, fine_size, fine_size, 1)) self.labels = tf.placeholder(tf.float32, shape=(None, target_size, target_size, 1)) self.training = tf.placeholder(tf.bool) self.keep_dropout = tf.placeholder(tf.float32) global_step = tf.Variable(0,trainable=False) # 160 -> 80 conv1 = tf.layers.conv2d(self.images, filters=16, kernel_size=21, strides=2, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv1 = batch_norm_layer(conv1, self.training, 'bn1') conv1 = tf.nn.relu(conv1) print('conv1 shape = %s' % conv1.shape) pool1 = tf.layers.max_pooling2d(conv1, pool_size=3, strides=2, padding='same') print('pool1 shape = %s' % pool1.shape) # 80 -> 40 conv12 = tf.layers.conv2d(conv1, filters=64, kernel_size=11, strides=2, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv12 = batch_norm_layer(conv12, self.training, 'bn12') conv12 = tf.nn.relu(conv12) print('conv12 shape = %s' % conv12.shape) # 40 -> 20 conv2 = tf.layers.conv2d(conv12, filters=256, kernel_size=5, strides=2, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv2 = batch_norm_layer(conv2, self.training, 'bn2') conv2 = tf.nn.relu(conv2) print('conv2 shape = %s' % conv2.shape) # pool2 = tf.layers.max_pooling2d(conv2, pool_size=3, strides=2, padding='same') # print('pool2 shape = %s' % pool2.shape) # 20 -> 10 conv3 = tf.layers.conv2d(conv2, filters=256, kernel_size=5, strides=2, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv3 = batch_norm_layer(conv3, self.training, 'bn3') conv3 = tf.nn.relu(conv3) print('conv3 shape = %s' % conv3.shape) # 10 -> 10 conv4 = tf.layers.conv2d(conv3, filters=256, kernel_size=3, strides=1, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv4 = batch_norm_layer(conv4, self.training, 'bn4') conv4 = tf.nn.relu(conv4) print('conv4 shape = %s' % conv4.shape) # 10 -> 10 conv42 = tf.layers.conv2d(conv4, filters=256, kernel_size=3, strides=1, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv42 = batch_norm_layer(conv42, self.training, 'bn42') conv42 = tf.nn.relu(conv42) print('conv42 shape = %s' % conv42.shape) # 10 -> 10 conv43 = tf.layers.conv2d(conv42, filters=256, kernel_size=3, strides=1, padding='same', kernel_initializer = xavier_initializer(uniform=False)) conv43 = batch_norm_layer(conv43, self.training, 'bn43') conv43 = tf.nn.relu(conv43) print('conv43 shape = %s' % conv43.shape) fc5 = tf.reshape(conv43, [-1, 256 * 10 * 10]) print('fc5 input shape = %s' % fc5.shape) fc5 = tf.contrib.layers.fully_connected(fc5, 5000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc5 = batch_norm_layer(fc5, self.training, 'bn5') fc5 = tf.nn.dropout(fc5, self.keep_dropout) print('fc5 output shape = %s' % fc5.shape) fc6 = tf.contrib.layers.fully_connected(fc5, 5000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) print('fc6 input shape = %s' % fc6.shape) fc6 = batch_norm_layer(fc6, self.training, 'bn6') fc6 = tf.nn.dropout(fc6, self.keep_dropout) # Output layer. Sigmoid function is used to gain result between 0 and 1 out = tf.contrib.layers.fully_connected(fc6, target_size * target_size, tf.nn.sigmoid, weights_initializer=xavier_initializer(uniform=False)) # out = tf.multiply(tf.add(tf.sign(tf.subtract(out, tf.constant(0.5))), tf.constant(1.)), 0.5) # Reshape the output to a 2d image self.result = tf.reshape(out, [-1, target_size, target_size, 1]) l1_loss = tf.losses.absolute_difference(self.labels, self.result) variation_loss = tf.image.total_variation(self.result) # Loss is a combination of L1 norm loss and total variation loss self.loss = tf.reduce_mean(l1_loss + variation_loss * variation_loss_scale) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): #self.optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(self.loss) self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss) self.saver = tf.train.Saver() #### define the saving part print('Graph building finished')
def inference(self): # parameters for the network conv_kernel_sizes = [[3, 3], [3, 3], [3, 3], [3, 3]] conv_strides = [[1, 1], [1, 1], [1, 1], [1, 1]] conv_filters = [32, 64, 128, 256] pool_strides = [[2, 2], [2, 2], [2, 2], [2, 2]] upconv_kernel_size = [[2, 2], [2, 2], [2, 2], [2, 2]] upconv_strides = [[2, 2], [2, 2], [2, 2], [2, 2]] upconv_filters = [256, 128, 64, 32] upconv_map_sizes = [[32, 32], [64, 64], [128, 128], [256, 256]] relu = tf.nn.relu xavier = xavier_initializer() num_layers = 2 image = self.x image.set_shape([None, self.input_shape[0], self.input_shape[1], self.input_shape[2]]) stack = [] # convolutional layers with tf.variable_scope("conv", reuse=tf.AUTO_REUSE): for layer in range(len(conv_kernel_sizes)): for cell in range(num_layers): image = Conv2D(filters=conv_filters[layer], kernel_size=conv_kernel_sizes[layer], strides=conv_strides[layer], padding='same', activation=relu, kernel_initializer=xavier)(image) stack.append(image) image = MaxPooling2D(pool_size=pool_strides[layer], strides=pool_strides[layer])(image) # steady stage with tf.variable_scope("steady_conv", reuse=tf.AUTO_REUSE): for cell in range(num_layers): image = Conv2D(filters=1024, kernel_size=[3, 3], strides=[1, 1], padding='same', activation=relu, kernel_initializer=xavier)(image) image = Dropout(rate=0.1)(image) # upconvolutional layers with tf.variable_scope("upconv", reuse=tf.AUTO_REUSE): for layer in range(len(upconv_kernel_size)): prev_image = stack.pop() prev_image = tf.image.resize_bilinear(prev_image, size=upconv_map_sizes[layer]) image = Conv2DTranspose(filters=upconv_filters[layer], kernel_size=upconv_kernel_size[layer], strides=upconv_strides[layer], kernel_initializer=xavier)(image) image = tf.concat([image, prev_image], axis=3) for cell in range(num_layers): image = Conv2D(filters=upconv_filters[layer], kernel_size=conv_kernel_sizes[layer], strides=conv_strides[layer], padding='same', activation=relu, kernel_initializer=xavier)(image) # getting class logit with tf.variable_scope("class", reuse=tf.AUTO_REUSE): logit = Conv2D(filters=2, kernel_size=[1, 1], strides=[1, 1], padding='same', kernel_initializer=xavier)(image) self.result_logit = logit soft_target = tf.nn.softmax(logit) one_hot = tf.arg_max(soft_target, dimension=3) one_hot = tf.expand_dims(one_hot, -1) image = tf.concat([one_hot, one_hot, one_hot], axis=3) image = tf.cast(image, dtype=tf.uint8) return 255 * image
def create_cond_generator(cls, z_input, batch_size, latent_dim, output_cells_dim, var_scope, gen_layers, output_lsn, gen_cond_type, clusters_ratios, is_training, clusters_no=None, input_clusters=None, reuse=None): """ Class method that instantiates a Generator and creates a conditional generator. Parameters ---------- z_input : Tensor Tensor containing the noise used as input by the generator. batch_size : int Batch size used during the training. latent_dim : int Dimension of the latent space used from which the input noise of the generator is sampled. output_cells_dim : int Dimension of the output cells (i.e. the number of genes). var_scope : str Variable scope used for the created tensors. gen_layers : list List of integers corresponding to the number of neurons of each layer of the generator. output_lsn : int, None Parameter of the LSN layer at the output of the critic (i.e. total number of counts per generated cell). gen_cond_type : str conditional normalization layers used in the generator, can be either "batchnorm" or "layernorm". If anything else, it won't be added in the model (which means no conditional generation). clusters_ratios : Tensor Placeholder containing the list of cluster ratios of the input data. is_training : Tensor Boolean placeholder encoding for whether we're in training or inference mode (for the batch normalization). clusters_no : int Number of clusters. Default is None. input_clusters : Tensor Placeholders for the cluster indexes that should be used for conditional generation. Default is None. reuse : Boolean Whether to reuse the already existing Tensors. Default is None. Returns ------- A conditional Generator object with the defined architecture. """ with tf.variable_scope(var_scope, reuse=reuse): for i_lay, size in enumerate(gen_layers): with tf.variable_scope("generator_layers_" + str(i_lay + 1)): z_input = layers.linear( z_input, size, weights_initializer=layers.xavier_initializer(), biases_initializer=None) if i_lay != -1: if gen_cond_type == "batchnorm": z_input = batchnorm( [0], z_input, is_training=is_training, labels=input_clusters, n_labels=clusters_no) elif gen_cond_type == "layernorm": z_input = layernorm([1], z_input, labels=input_clusters, n_labels=clusters_no) z_input = tf.nn.relu(z_input) with tf.variable_scope("generator_layers_" + 'output'): fake_outputs = layers.relu( z_input, output_cells_dim, weights_initializer=layers.variance_scaling_initializer(mode="FAN_AVG"), biases_initializer=tf.zeros_initializer()) if output_lsn is not None: gammas_output = tf.Variable( np.ones(z_input.shape.as_list()[0]) * output_lsn, trainable=False) sigmas = tf.reduce_sum(fake_outputs, axis=1) scale_ls = tf.cast(gammas_output, dtype=tf.float32) / \ (sigmas + sys.float_info.epsilon) fake_outputs = tf.transpose(tf.transpose(fake_outputs) * scale_ls) return cls(fake_outputs, batch_size, latent_dim, output_cells_dim, var_scope, gen_layers, output_lsn, gen_cond_type=gen_cond_type, is_training=is_training, clusters_ratios=clusters_ratios, clusters_no=clusters_no, input_clusters=input_clusters, reuse=reuse)
def build_graph_best(self): print('Building graph') with self.graph.as_default(): # Input data self.images = tf.placeholder(tf.float32, shape=(None, fine_size, fine_size, 1)) self.labels = tf.placeholder(tf.float32, shape=(None, target_size, target_size, 1)) self.training = tf.placeholder(tf.bool) self.keep_dropout = tf.placeholder(tf.float32) global_step = tf.Variable(0,trainable=False) # 80 -> 40 -> 20 if self.NN: fc1 = tf.contrib.layers.fully_connected(tf.reshape(self.images, [-1, fine_size * fine_size]), 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc1 = batch_norm_layer(fc1, self.training, 'bn1') fc1 = tf.nn.dropout(fc1, self.keep_dropout) fc2 = tf.contrib.layers.fully_connected(fc1, 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc2 = batch_norm_layer(fc2, self.training, 'bn2') fc2 = tf.nn.dropout(fc2, self.keep_dropout) fc3 = tf.contrib.layers.fully_connected(fc2, 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc3 = batch_norm_layer(fc3, self.training, 'bn3') fc3 = tf.nn.dropout(fc3, self.keep_dropout) fc4 = tf.contrib.layers.fully_connected(fc3, 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc4 = batch_norm_layer(fc4, self.training, 'bn4') fc4 = tf.nn.dropout(fc4, self.keep_dropout) fc5 = tf.contrib.layers.fully_connected(fc4, 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc5 = batch_norm_layer(fc5, self.training, 'bn5') fc5 = tf.nn.dropout(fc5, self.keep_dropout) fc6 = tf.contrib.layers.fully_connected(fc5, 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc6 = batch_norm_layer(fc6, self.training, 'bn6') fc6 = tf.nn.dropout(fc6, self.keep_dropout) fc7 = tf.contrib.layers.fully_connected(fc6, 3000, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc7 = batch_norm_layer(fc7, self.training, 'bn7') train_out = tf.nn.dropout(fc7, self.keep_dropout) else: conv1_1 = tf.layers.conv2d(inputs=self.images, filters=64, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool1: conv1_1 = tf.layers.max_pooling2d(inputs=conv1_1, pool_size=2, strides=2) if BN: conv1_1 = tf.layers.batch_normalization(inputs=conv1_1, axis=3) print('conv1 shape = %s' % conv1_1.shape) conv1_2 = tf.layers.conv2d(inputs=conv1_1, filters=64, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool2: conv1_2 = tf.layers.max_pooling2d(inputs=conv1_2, pool_size=2, strides=2) if BN: conv1_2 = tf.layers.batch_normalization(inputs=conv1_2, axis=3) print('conv2 shape = %s' % conv1_2.shape) conv2_1 = tf.layers.conv2d(inputs=conv1_2, filters=128, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool3: conv2_1 = tf.layers.max_pooling2d(inputs=conv2_1, pool_size=2, strides=2) if BN: conv2_1 = tf.layers.batch_normalization(inputs=conv2_1, axis=3) conv2_2 = tf.layers.conv2d(inputs=conv2_1, filters=128, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool4: conv2_2 = tf.layers.max_pooling2d(inputs=conv2_2, pool_size=2, strides=2) if BN: conv2_2 = tf.layers.batch_normalization(inputs=conv2_2, axis=3) print('conv2_2 shape = %s' % conv2_2.shape) conv3_1 = tf.layers.conv2d(inputs=conv2_2, filters=256, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool5: conv3_1 = tf.layers.max_pooling2d(inputs=conv3_1, pool_size=2, strides=2) if BN: conv3_1 = tf.layers.batch_normalization(inputs=conv3_1, axis=3) conv3_2 = tf.layers.conv2d(inputs=conv3_1, filters=256, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool6: conv3_2 = tf.layers.max_pooling2d(inputs=conv3_2, pool_size=2, strides=2) if BN: conv3_2 = tf.layers.batch_normalization(inputs=conv3_2, axis=3) conv3_3 = tf.layers.conv2d(inputs=conv3_2, filters=256, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool7: conv3_3 = tf.layers.max_pooling2d(inputs=conv3_3, pool_size=2, strides=2) if BN: conv3_3 = tf.layers.batch_normalization(inputs=conv3_3, axis=3) print('conv3_3 shape = %s' % conv3_3.shape) conv4_1 = tf.layers.conv2d(inputs=conv3_3, filters=512, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool8: conv4_1 = tf.layers.max_pooling2d(inputs=conv4_1, pool_size=2, strides=2) if BN: conv4_1 = tf.layers.batch_normalization(inputs=conv4_1, axis=3) conv4_2 = tf.layers.conv2d(inputs=conv4_1, filters=512, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool9: conv4_2 = tf.layers.max_pooling2d(inputs=conv4_2, pool_size=2, strides=2) if BN: conv4_2 = tf.layers.batch_normalization(inputs=conv4_2, axis=3) conv4_3 = tf.layers.conv2d(inputs=conv4_2, filters=512, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool10: conv4_3 = tf.layers.max_pooling2d(inputs=conv4_3, pool_size=2, strides=2) if BN: conv4_3 = tf.layers.batch_normalization(inputs=conv4_3, axis=3) print('conv4_3 shape = %s' % conv4_3.shape) conv5_1 = tf.layers.conv2d(inputs=conv4_3, filters=512, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool11: conv5_1 = tf.layers.max_pooling2d(inputs=conv5_1, pool_size=2, strides=2) if BN: conv5_1 = tf.layers.batch_normalization(inputs=conv5_1, axis=3) conv5_2 = tf.layers.conv2d(inputs=conv5_1, filters=512, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool12: conv5_2 = tf.layers.max_pooling2d(inputs=conv5_2, pool_size=2, strides=2) if BN: conv5_2 = tf.layers.batch_normalization(inputs=conv5_2, axis=3) conv5_3 = tf.layers.conv2d(inputs=conv5_2, filters=512, kernel_size=3, strides=1, padding="same", activation=tf.nn.relu) if pool13: conv5_3 = tf.layers.max_pooling2d(inputs=conv5_3, pool_size=2, strides=2) if BN: conv5_3 = tf.layers.batch_normalization(inputs=conv5_3, axis=3) print('conv5_3 shape = %s' % conv5_3.shape) #conv1 = tf.layers.conv2d(self.images, filters=64, kernel_size=3, strides=1, padding='same', # kernel_initializer = xavier_initializer(uniform=False)) #conv1 = batch_norm_layer(conv1, self.training, 'bn1') #conv1 = tf.nn.relu(conv1) #print('conv1 shape = %s' % conv1.shape) #pool1 = tf.layers.max_pooling2d(conv1, pool_size=3, strides=2, padding='same') #print('pool1 shape = %s' % pool1.shape) flat = tf.contrib.layers.flatten(inputs=conv5_3) flat = tf.layers.dropout(inputs=flat, rate=dropout, training=self.training) print('flat shape = %s' % flat.shape) #fc5 = tf.reshape(conv5_3, [-1, 4096]) #print('flat shape = %s' % fc5.shape) fc1 = tf.contrib.layers.fully_connected(flat, 4096, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc2 = batch_norm_layer(fc1, self.training, 'bn5') fc3 = tf.contrib.layers.fully_connected(fc2, 4096, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc3 = batch_norm_layer(fc2, self.training, 'bn7') fc4 = tf.contrib.layers.fully_connected(fc3, target_size*target_size, tf.nn.relu, weights_initializer=xavier_initializer(uniform=False)) fc4 = batch_norm_layer(fc3, self.training, 'bn8') train_out = tf.nn.dropout(fc2, self.keep_dropout) ##tra out = tf.contrib.layers.fully_connected(train_out, target_size * target_size, tf.nn.sigmoid, weights_initializer=xavier_initializer(uniform=False)) # out = tf.multiply(tf.add(tf.sign(tf.subtract(out, tf.constant(0.5))), tf.constant(1.)), 0.5) self.result = tf.reshape(out, [-1, target_size, target_size, 1]) if self.l2_loss: l2_loss = tf.reduce_mean(tf.losses.mean_squared_error(self.result,self.labels)) variation_loss = tf.image.total_variation(self.result) self.loss = tf.reduce_mean(l2_loss + variation_loss * variation_loss_importance) else: l1_loss = tf.losses.absolute_difference(self.labels, self.result) ###### loss function need to be changed to l2 variation_loss = tf.image.total_variation(self.result) self.loss = tf.reduce_mean(l1_loss + variation_loss * variation_loss_importance) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): #self.optimizer = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(self.loss) self.optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(self.loss) self.saver = tf.train.Saver() #### define the saving part print('Graph building finished')
def gern_net(self, z, y): #G的输出层不加BN层 with tf.variable_scope('generator') as scope: # ? 1 1 10 yb = tf.reshape(y, shape=[self.batch_size, 1, 1, self.y_dim]) # ? 110 把z和y相连 z = tf.concat([z, y], 1) #在同一行的后面加,即列数增加 (64,100+10) # 7 14 计算中间层大小 c1, c2 = int( self.output_size / 4), int(self.output_size / 2 ) #7,14 # 10 stand for the num of labels # ? 1024 d1 = tf.nn.relu(batch_normal(fully_connect(z, output_size=1024, scope='gen_fully'), scope='gen_bn1')) #(64,1024) # ? 1034 在第一个全连接层后面在连接y d1 = tf.concat([d1, y], 1) #(64,1034) # 全连接层2 ? 7*7*2*64 -> c1*c1*2*self.batch_size d2 = tf.nn.relu(batch_normal(fully_connect(d1, output_size=c1*c2*self.batch_size, scope='gen_fully2'), scope='gen_bn2')) #c1*c1*2*self.batch_size??? #64,7*7*2*64 # ? 7 7 128 d2 = tf.reshape(d2, [self.batch_size, c1, c1, self.batch_size*2]) #64,7,7,128 # ? 7 7 138 d2 = conv_cond_concat(d2, yb)# 又将y加到后面 # ? 14 14 128 d3 = tf.nn.relu(batch_normal(de_conv(d2, output_shape=[self.batch_size, c2, c2, 128], name='gen_deconv1'), scope='gen_bn3'))#64,14,14,128 # ? 14 14 138 d3 = conv_cond_concat(d3, yb) # 再加一次 # 输出 ? 28 28 1 d4 = de_conv(d3, output_shape=[self.batch_size, self.output_size, self.output_size, self.channel], name='gen_deconv2', initializer = xavier_initializer()) #64,28,28,1 return tf.nn.sigmoid(d4)
def dis_net(self, images, y, reuse=False): #D的输入层不加BN层 with tf.variable_scope("discriminator") as scope: if reuse == True: scope.reuse_variables() # mnist data's shape is (28 , 28 , 1) # ? 1 1 10 yb = tf.reshape(y, shape=[self.batch_size, 1, 1, self.y_dim]) # concat ? 28 28 11 concat_data = conv_cond_concat(images, yb) # ? 14 14 10 # w1 3 3 11 10 卷积核 conv1, w1 = conv2d(concat_data, output_dim=10, name='dis_conv1') tf.add_to_collection('weight_1', w1) conv1 = lrelu(conv1) # ? 14 14 20 conv1 = conv_cond_concat(conv1, yb) # 再连接条件 tf.add_to_collection('ac_1', conv1) # ? 7 7 64 # w2 3 3 20 64 conv2, w2 = conv2d(conv1, output_dim=64, name='dis_conv2') tf.add_to_collection('weight_2', w2) conv2 = lrelu(batch_normal(conv2, scope='dis_bn1')) tf.add_to_collection('ac_2', conv2) #将元素conv2添加到列表ac_2中 # ? 3136 -> 7*7*64 conv2 = tf.reshape(conv2, [self.batch_size, -1]) # ? 3146 conv2 = tf.concat([conv2, y], 1) # 再加 # ? 1024 f1 = lrelu(batch_normal(fully_connect(conv2, output_size=1024, scope='dis_fully1'), scope='dis_bn2', reuse=reuse)) # ? 1034 f1 = tf.concat([f1, y], 1) # 加一个全连接 ? 1 out = fully_connect(f1, output_size=1, scope='dis_fully2', initializer = xavier_initializer()) return tf.nn.sigmoid(out), out