def build_graph(self, inputs, add_summaries=False):
prev_layer = inputs
for i, layer_dim in enumerate(self.layer_dims):
prev_layer = FC_layer(prev_layer, layer_dim, nonlinearity=self.nonlinearity,
is_training=self.input_handler.is_training, dropout_prob=self.dropout_prob,
name='layer_{}'.format(i + 1))
if add_summaries: variable_summaries(prev_layer)
return prev_layer
def build_graph(self, inputs, add_summaries=False):
prev_layer = tf.expand_dims(inputs, 3)
for i, (num_filters, filter_dim) in enumerate(zip(self.num_filter_channels, self.filter_dims)):
prev_layer = tf.layers.conv2d(prev_layer,
num_filters, filter_dim,
activation=self.nonlinearity,
name='CNN_layer{}'.format(i + 1))
if add_summaries: variable_summaries(prev_layer)
if self.maxpooling:
prev_layer = tf.layers.max_pooling2d(prev_layer,
self.pool_dims[i], self.pool_strides[i],
name='Pool_layer{}'.format(i + 1))
flattened = tf.reshape(prev_layer, [-1, tf.reduce_prod(prev_layer.get_shape()[1:])])
return flattened
def build_graph(self, inputs, add_summaries=False):
# build list of recurrent cells
cells = []
for i, layer_dim in enumerate(self.layer_dims):
cell = recurrent_cell(self.cell_type, layer_dim,
self.input_handler.is_training, self.dropout_prob, 'RNN_cell{}'.format(i + 1))
cells.append(cell)
# create multilayer recurrent cell
with tf.name_scope('multilayer_cell'):
cell = tf.contrib.rnn.MultiRNNCell(cells, state_is_tuple=True)
# create the actual recurrent architecture
with tf.name_scope('outputs'):
outputs, last_states = tf.nn.dynamic_rnn(
cell=cell,
inputs=inputs,
sequence_length=self.input_handler.sequence_lengths,
dtype=tf.float32
)
if add_summaries: variable_summaries(outputs)
# return outputs # for seq2seq
return outputs[:, -1, :]
def negQ(self, x, y, reuse=False):
szs = [FLAGS.l1size, FLAGS.l2size]
assert (len(szs) >= 1)
fc = tflearn.fully_connected
bn = tflearn.batch_normalization
lrelu = tflearn.activations.leaky_relu
if reuse:
tf.get_variable_scope().reuse_variables()
nLayers = len(szs)
us = []
zs = []
z_zs = []
z_ys = []
z_us = []
reg = 'L2'
prevU = x
for i in range(nLayers):
with tf.variable_scope('u' + str(i)) as s:
u = fc(prevU, szs[i], reuse=reuse, scope=s, regularizer=reg)
if i < nLayers - 1:
u = tf.nn.relu(u)
if FLAGS.icnn_bn:
u = bn(u, reuse=reuse, scope=s, name='bn')
variable_summaries(u, suffix='u{}'.format(i))
us.append(u)
prevU = u
prevU, prevZ = x, y
for i in range(nLayers + 1):
sz = szs[i] if i < nLayers else 1
z_add = []
if i > 0:
with tf.variable_scope('z{}_zu_u'.format(i)) as s:
zu_u = fc(prevU,
szs[i - 1],
reuse=reuse,
scope=s,
activation='relu',
bias=True,
regularizer=reg,
bias_init=tf.constant_initializer(1.))
variable_summaries(zu_u, suffix='zu_u{}'.format(i))
with tf.variable_scope('z{}_zu_proj'.format(i)) as s:
z_zu = fc(tf.mul(prevZ, zu_u),
sz,
reuse=reuse,
scope=s,
bias=False,
regularizer=reg)
variable_summaries(z_zu, suffix='z_zu{}'.format(i))
z_zs.append(z_zu)
z_add.append(z_zu)
with tf.variable_scope('z{}_yu_u'.format(i)) as s:
yu_u = fc(prevU,
self.dimA,
reuse=reuse,
scope=s,
bias=True,
regularizer=reg,
bias_init=tf.constant_initializer(1.))
variable_summaries(yu_u, suffix='yu_u{}'.format(i))
with tf.variable_scope('z{}_yu'.format(i)) as s:
z_yu = fc(tf.mul(y, yu_u),
sz,
reuse=reuse,
scope=s,
bias=False,
regularizer=reg)
z_ys.append(z_yu)
variable_summaries(z_yu, suffix='z_yu{}'.format(i))
z_add.append(z_yu)
with tf.variable_scope('z{}_u'.format(i)) as s:
z_u = fc(prevU,
sz,
reuse=reuse,
scope=s,
bias=True,
regularizer=reg,
bias_init=tf.constant_initializer(0.))
variable_summaries(z_u, suffix='z_u{}'.format(i))
z_us.append(z_u)
z_add.append(z_u)
z = tf.add_n(z_add)
variable_summaries(z, suffix='z{}_preact'.format(i))
if i < nLayers:
# z = tf.nn.relu(z)
z = lrelu(z, alpha=FLAGS.lrelu)
variable_summaries(z, suffix='z{}_act'.format(i))
zs.append(z)
prevU = us[i] if i < nLayers else None
prevZ = z
z = tf.reshape(z, [-1], name='energies')
return z
if data_loader.__module__ == 'datasetLoaders.RandomManifoldDataLoader' or data_loader.__module__ == 'datasetLoaders.ToyDataLoader':
train_generator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['generator_cost'], var_list=generator_vars, global_step=global_step)
train_discriminator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step)
train_transport_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['transporter_cost'], var_list=transport_vars, global_step=global_step)
else:
train_generator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['generator_cost'], var_list=generator_vars, global_step=global_step)
train_discriminator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step)
train_transport_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.5, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['transporter_cost'], var_list=transport_vars, global_step=global_step)
helper.variable_summaries(train_outs_dict['generator_cost'], '/generator_cost')
helper.variable_summaries(train_outs_dict['discriminator_cost'], '/discriminator_cost')
helper.variable_summaries(train_outs_dict['transporter_cost'], '/transporter_cost')
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.InteractiveSession()
merged_summaries = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(global_args.exp_dir+'/summaries', sess.graph)
sess.run(init)
if global_args.restore:
print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
try:
helper.load_checkpoint(saver, sess, global_args.global_exp_dir+global_args.restore_dir)
print("=> Loaded checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
except: print("=> FAILED to load checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
def add_prediction_summaries(self, **kwargs):
for node, output_handler in self.output_handlers.iteritems():
with tf.variable_scope(node):
variable_summaries(output_handler.predictions)
model = Model(vars(global_args))
global_step = tf.Variable(0.0, name='global_step', trainable=False)
with tf.variable_scope("training"):
tf.set_random_seed(global_args.seed)
additional_inputs_tf = tf.placeholder(tf.bool, [1])
batch_tf, input_dict_func = helper.tf_batch_and_input_dict(batch, additional_inputs_tf)
train_outs_dict, test_outs_dict = model.inference(batch_tf, additional_inputs_tf)
discriminator_vars = [v for v in tf.trainable_variables() if 'Discriminator' in v.name]
train_discriminator_step_tf = tf.train.AdamOptimizer(learning_rate=0.0001,
beta1=0.9, beta2=0.999, epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'], var_list=discriminator_vars, global_step=global_step)
helper.variable_summaries(train_outs_dict['discriminator_cost'], '/discriminator_cost')
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.InteractiveSession()
merged_summaries = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(global_args.exp_dir+'/summaries', sess.graph)
sess.run(init)
if global_args.restore:
print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
try:
helper.load_checkpoint(saver, sess, global_args.global_exp_dir+global_args.restore_dir)
print("=> Loaded checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
except: print("=> FAILED to load checkpoint: '{}'".format(global_args.global_exp_dir+global_args.restore_dir))
def train(epoch):
obs_dist = model.obs_dist
sample_obs_dist, obs_sample_out_tf, latent_sample_out_tf = model.generative_model(
batch_tf)
if global_args.optimizer_class == 'RmsProp':
train_step_tf = tf.train.RMSPropOptimizer(
learning_rate=global_args.learning_rate,
momentum=0.9).minimize(batch_loss_tf, global_step=global_step)
elif global_args.optimizer_class == 'Adam':
train_step_tf = tf.train.AdamOptimizer(
learning_rate=global_args.learning_rate,
beta1=0.9,
beta2=0.999,
epsilon=1e-08).minimize(batch_loss_tf, global_step=global_step)
helper.variable_summaries(batch_loss_tf, '/batch_loss_tf')
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.InteractiveSession()
merged_summaries = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(global_args.exp_dir + '/summaries',
sess.graph)
sess.run(init)
if global_args.restore:
print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir +
global_args.restore_dir))
try:
helper.load_checkpoint(
saver, sess,
global_args.global_exp_dir + global_args.restore_dir)
learning_rate=0.0001, beta1=0.9, beta2=0.999,
epsilon=1e-08).minimize(train_outs_dict['generator_cost'],
var_list=generator_vars,
global_step=global_step)
train_discriminator_step_tf = tf.train.AdamOptimizer(
learning_rate=0.0001, beta1=0.9, beta2=0.999,
epsilon=1e-08).minimize(train_outs_dict['discriminator_cost'],
var_list=discriminator_vars,
global_step=global_step)
train_transport_step_tf = tf.train.AdamOptimizer(
learning_rate=0.0001, beta1=0.1, beta2=0.999,
epsilon=1e-08).minimize(train_outs_dict['variational_cost'],
var_list=transport_vars,
global_step=global_step)
helper.variable_summaries(train_outs_dict['generator_cost'],
'/generator_cost')
helper.variable_summaries(train_outs_dict['discriminator_cost'],
'/discriminator_cost')
helper.variable_summaries(train_outs_dict['variational_cost'],
'/variational_cost')
init = tf.global_variables_initializer()
saver = tf.train.Saver()
sess = tf.InteractiveSession()
merged_summaries = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(global_args.exp_dir + '/summaries',
sess.graph)
sess.run(init)
if global_args.restore:
print("=> Loading checkpoint: '{}'".format(global_args.global_exp_dir +
global_args.restore_dir))
