def build_graph(self, inputs, add_summaries=False): prev_layer = inputs for i, layer_dim in enumerate(self.layer_dims): prev_layer = FC_layer(prev_layer, layer_dim, nonlinearity=self.nonlinearity, is_training=self.input_handler.is_training, dropout_prob=self.dropout_prob, name='layer_{}'.format(i + 1)) if add_summaries: variable_summaries(prev_layer) return prev_layer
def build_graph(self, inputs, add_summaries=False): prev_layer = tf.expand_dims(inputs, 3) for i, (num_filters, filter_dim) in enumerate(zip(self.num_filter_channels, self.filter_dims)): prev_layer = tf.layers.conv2d(prev_layer, num_filters, filter_dim, activation=self.nonlinearity, name='CNN_layer{}'.format(i + 1)) if add_summaries: variable_summaries(prev_layer) if self.maxpooling: prev_layer = tf.layers.max_pooling2d(prev_layer, self.pool_dims[i], self.pool_strides[i], name='Pool_layer{}'.format(i + 1)) flattened = tf.reshape(prev_layer, [-1, tf.reduce_prod(prev_layer.get_shape()[1:])]) return flattened
def build_graph(self, inputs, add_summaries=False): # build list of recurrent cells cells = [] for i, layer_dim in enumerate(self.layer_dims): cell = recurrent_cell(self.cell_type, layer_dim, self.input_handler.is_training, self.dropout_prob, 'RNN_cell{}'.format(i + 1)) cells.append(cell) # create multilayer recurrent cell with tf.name_scope('multilayer_cell'): cell = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True) # create the actual recurrent architecture with tf.name_scope('outputs'): outputs, last_states = tf.nn.dynamic_rnn( cell=cell, inputs=inputs, sequence_length=self.input_handler.sequence_lengths, dtype=tf.float32 ) if add_summaries: variable_summaries(outputs) # return outputs # for seq2seq return outputs[:, -1, :]
def negQ(self, x, y, reuse=False): szs = [FLAGS.l1size, FLAGS.l2size] assert (len(szs) >= 1) fc = tflearn.fully_connected bn = tflearn.batch_normalization lrelu = tflearn.activations.leaky_relu if reuse: tf.get_variable_scope().reuse_variables() nLayers = len(szs) us = [] zs = [] z_zs = [] z_ys = [] z_us = [] reg = 'L2' prevU = x for i in range(nLayers): with tf.variable_scope('u' + str(i)) as s: u = fc(prevU, szs[i], reuse=reuse, scope=s, regularizer=reg) if i < nLayers - 1: u = tf.nn.relu(u) if FLAGS.icnn_bn: u = bn(u, reuse=reuse, scope=s, name='bn') variable_summaries(u, suffix='u{}'.format(i)) us.append(u) prevU = u prevU, prevZ = x, y for i in range(nLayers + 1): sz = szs[i] if i < nLayers else 1 z_add = [] if i > 0: with tf.variable_scope('z{}_zu_u'.format(i)) as s: zu_u = fc(prevU, szs[i - 1], reuse=reuse, scope=s, activation='relu', bias=True, regularizer=reg, bias_init=tf.constant_initializer(1.)) variable_summaries(zu_u, suffix='zu_u{}'.format(i)) with tf.variable_scope('z{}_zu_proj'.format(i)) as s: z_zu = fc(tf.mul(prevZ, zu_u), sz, reuse=reuse, scope=s, bias=False, regularizer=reg) variable_summaries(z_zu, suffix='z_zu{}'.format(i)) z_zs.append(z_zu) z_add.append(z_zu) with tf.variable_scope('z{}_yu_u'.format(i)) as s: yu_u = fc(prevU, self.dimA, reuse=reuse, scope=s, bias=True, regularizer=reg, bias_init=tf.constant_initializer(1.)) variable_summaries(yu_u, suffix='yu_u{}'.format(i)) with tf.variable_scope('z{}_yu'.format(i)) as s: z_yu = fc(tf.mul(y, yu_u), sz, reuse=reuse, scope=s, bias=False, regularizer=reg) z_ys.append(z_yu) variable_summaries(z_yu, suffix='z_yu{}'.format(i)) z_add.append(z_yu) with tf.variable_scope('z{}_u'.format(i)) as s: z_u = fc(prevU, sz, reuse=reuse, scope=s, bias=True, regularizer=reg, bias_init=tf.constant_initializer(0.)) variable_summaries(z_u, suffix='z_u{}'.format(i)) z_us.append(z_u) z_add.append(z_u) z = tf.add_n(z_add) variable_summaries(z, suffix='z{}_preact'.format(i)) if i < nLayers: # z = tf.nn.relu(z) z = lrelu(z, alpha=FLAGS.lrelu) variable_summaries(z, suffix='z{}_act'.format(i)) zs.append(z) prevU = us[i] if i < nLayers else None prevZ = z z = tf.reshape(z, [-1], name='energies') return z
if data_loader.__module__ == 'datasetLoaders.RandomManifoldDataLoader' or data_loader.__module__ == 'datasetLoaders.ToyDataLoader': train_generator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['generator_cost'], var_list=generator_vars, global_step=global_step) train_discriminator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step) train_transport_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['transporter_cost'], var_list=transport_vars, global_step=global_step) else: train_generator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['generator_cost'], var_list=generator_vars, global_step=global_step) train_discriminator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step) train_transport_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['transporter_cost'], var_list=transport_vars, global_step=global_step) helper.variable_summaries(train_outs_dict['generator_cost'], '/generator_cost') helper.variable_summaries(train_outs_dict['discriminator_cost'], '/discriminator_cost') helper.variable_summaries(train_outs_dict['transporter_cost'], '/transporter_cost') init = tf.global_variables_initializer() saver = tf.train.Saver() sess = tf.InteractiveSession() merged_summaries = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(global_args.exp_dir+'/summaries', sess.graph) sess.run(init) if global_args.restore: print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir)) try: helper.load_checkpoint(saver, sess, global_args.global_exp_dir+global_args.restore_dir) print("=> Loaded checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir)) except: print("=> FAILED to load checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
def add_prediction_summaries(self, **kwargs): for node, output_handler in self.output_handlers.iteritems(): with tf.variable_scope(node): variable_summaries(output_handler.predictions)
model = Model(vars(global_args)) global_step = tf.Variable(0.0, name='global_step', trainable=False) with tf.variable_scope("training"): tf.set_random_seed(global_args.seed) additional_inputs_tf = tf.placeholder(tf.bool, [1]) batch_tf, input_dict_func = helper.tf_batch_and_input_dict(batch, additional_inputs_tf) train_outs_dict, test_outs_dict = model.inference(batch_tf, additional_inputs_tf) discriminator_vars = [v for v in tf.trainable_variables() if 'Discriminator' in v.name] train_discriminator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001, beta1=0.9, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step) helper.variable_summaries(train_outs_dict['discriminator_cost'], '/discriminator_cost') init = tf.global_variables_initializer() saver = tf.train.Saver() sess = tf.InteractiveSession() merged_summaries = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(global_args.exp_dir+'/summaries', sess.graph) sess.run(init) if global_args.restore: print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir)) try: helper.load_checkpoint(saver, sess, global_args.global_exp_dir+global_args.restore_dir) print("=> Loaded checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir)) except: print("=> FAILED to load checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir)) def train(epoch):
obs_dist = model.obs_dist sample_obs_dist, obs_sample_out_tf, latent_sample_out_tf = model.generative_model( batch_tf) if global_args.optimizer_class == 'RmsProp': train_step_tf = tf.train.RMSPropOptimizer( learning_rate=global_args.learning_rate, momentum=0.9).minimize(batch_loss_tf, global_step=global_step) elif global_args.optimizer_class == 'Adam': train_step_tf = tf.train.AdamOptimizer( learning_rate=global_args.learning_rate, beta1=0.9, beta2=0.999, epsilon=1e-08).minimize(batch_loss_tf, global_step=global_step) helper.variable_summaries(batch_loss_tf, '/batch_loss_tf') init = tf.global_variables_initializer() saver = tf.train.Saver() sess = tf.InteractiveSession() merged_summaries = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(global_args.exp_dir + '/summaries', sess.graph) sess.run(init) if global_args.restore: print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir + global_args.restore_dir)) try: helper.load_checkpoint( saver, sess, global_args.global_exp_dir + global_args.restore_dir)
learning_rate=0.0001, beta1=0.9, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['generator_cost'], var_list=generator_vars, global_step=global_step) train_discriminator_step_tf = tf.train.AdamOptimizer( learning_rate=0.0001, beta1=0.9, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step) train_transport_step_tf = tf.train.AdamOptimizer( learning_rate=0.0001, beta1=0.1, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['variational_cost'], var_list=transport_vars, global_step=global_step) helper.variable_summaries(train_outs_dict['generator_cost'], '/generator_cost') helper.variable_summaries(train_outs_dict['discriminator_cost'], '/discriminator_cost') helper.variable_summaries(train_outs_dict['variational_cost'], '/variational_cost') init = tf.global_variables_initializer() saver = tf.train.Saver() sess = tf.InteractiveSession() merged_summaries = tf.summary.merge_all() summary_writer = tf.summary.FileWriter(global_args.exp_dir + '/summaries', sess.graph) sess.run(init) if global_args.restore: print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir + global_args.restore_dir))