def load_model(self, sess, save_file):
"""Load previously saved model parameters
Args:
sess (tf.Session object): current session object to run graph
save_file (str): full path to saved model
"""
if not os.path.isfile(save_file + '.meta'):
raise exc.InputError(str('%s is not a valid filename' % save_file))
self.saver.restore(sess, save_file)
print('Model loaded from %s' % save_file)
def load_model(self, sess, save_file=None):
"""Load previously saved model parameters
Args:
sess (tf.Session object): current session object to run graph
save_file (str): full path to saved model
"""
if not os.path.isfile(save_file + '.meta'):
raise exc.InputError(str('%s is not a valid filename' % save_file))
self.saver.restore(sess, save_file)
print('Model loaded from %s' % save_file)
# from https://github.com/AYLIEN/gan-intro/blob/master/gan.py
# import argparse
# def main(args):
# model = GAN(
# DataDistribution(),
# GeneratorDistribution(range=8),
# args.num_steps,
# args.batch_size,
# args.minibatch,
# args.log_every,
# args.anim
# )
# model.train()
#
#
# def parse_args():
# parser = argparse.ArgumentParser()
# parser.add_argument('--num-steps', type=int, default=1200,
# help='the number of training steps to take')
# parser.add_argument('--batch-size', type=int, default=12,
# help='the batch size')
# parser.add_argument('--minibatch', type=bool, default=False,
# help='use minibatch discrimination')
# parser.add_argument('--log-every', type=int, default=10,
# help='print loss after this many steps')
# return parser.parse_args()
#
#
# if __name__ == '__main__':
# main(parse_args())
def __init__(self,
layers_encoder=None,
layer_latent=None,
layers_decoder=None,
num_classes=None,
act_func='relu',
learning_rate=1e-3):
"""Constructor for VAE class
Args:
layers_encoder (list of ints): size of each layer in encoder,
including input layer
layer_latent (int): size of latent layer
layers_decoder (list of ints): size of each layer in decoder,
including output layer
num_classes (int): number of classes in data
act_func (str): activation function for network layers
['relu'] | 'sigmoid' | 'tanh' | 'linear' | 'softplus' | 'elu'
learning_rate (scalar): global learning rate for gradient descent
methods
Raises:
InputError if layers_encoder is not specified
InputError if layers_latent is not specified
InputError if layers_decoder is not specified
InputError if num_classes is not specified
InputError if act_func is not a valid string
"""
# input checking
if layers_encoder is None:
raise exc.InputError('Must specify layer sizes for encoder')
if layer_latent is None:
raise exc.InputError('Must specify number of latent dimensions')
if layers_decoder is None:
raise exc.InputError('Must specify layer sizes for decoder')
if num_classes is None:
raise exc.InputError('Must specify number of classes')
self.num_classes = num_classes
# concatenate input and labels
self.input_size = layers_encoder[0]
self.layers_encoder = layers_encoder[1:]
# concatenate lvs and labels
self.layer_latent = layer_latent + num_classes
self.layers_decoder = layers_decoder
if act_func == 'relu':
self.act_func = tf.nn.relu
elif act_func == 'sigmoid':
self.act_func = tf.sigmoid
elif act_func == 'tanh':
self.act_func = tf.tanh
elif act_func == 'linear':
self.act_func = tf.identity
elif act_func == 'softplus':
self.act_func = tf.nn.softplus
elif act_func == 'elu':
self.act_func = tf.nn.elu
else:
raise exc.InputError('Invalid activation function')
self.learning_rate = learning_rate
# define useful constants
self.num_lvs = self.layer_latent
self.num_layers_enc = len(self.layers_encoder)
self.num_layers_dec = len(self.layers_decoder)
# for saving and restoring models
self.graph = tf.Graph() # must be initialized before graph creation
# build model graph
with self.graph.as_default():
# define pipeline for feeding data into model
with tf.variable_scope('data'):
self._initialize_data_pipeline()
# initialize weights and create encoder model
with tf.variable_scope('encoder'):
self._define_recognition_network()
# initialize weights and create decoder model
with tf.variable_scope('decoder'):
self._define_generator_network()
# define loss function
with tf.variable_scope('loss'):
self._define_loss()
# define optimizer
with tf.variable_scope('optimizer'):
self._define_optimizer()
# add additional ops
# for saving and restoring models
self.saver = tf.train.Saver() # must be init after var creation
# collect all summaries into a single op
self.merge_summaries = tf.summary.merge_all()
# add variable initialization op to graph
self.init = tf.global_variables_initializer()
def train_iters(self,
sess,
data=None,
batch_size=128,
iters_training=1000,
iters_disp=None,
iters_ckpt=None,
iters_summary=None,
output_dir=None):
"""
Network training by specifying number of iterations rather than
epochs. Used for easily generating sample outputs during training
Args:
sess (tf.Session object): current session object to run graph
data (DataReader object): input to network
batch_size (int, optional): batch size used by the gradient
descent-based optimizers
iters_training (int, optional): number of iters for gradient
descent-based optimizers
iters_disp (int, optional): number of iters between updates to
the console
iters_ckpt (int, optional): number of iters between saving
checkpoint files
iters_summary (int, optional): number of iters between saving
network summary information
output_dir (string, optional): absolute path for saving checkpoint
files and summary files; must be present if either iters_ckpt
or iters_summary is not 'None'.
Returns:
None
Raises:
InputError: If data is not specified
InputError: If iters_ckpt is not None and output_dir is None
InputError: If iters_summary is not None and output_dir is None
"""
# check input
if data is None:
raise exc.InputError('data reader must be specified')
if iters_ckpt is not None and output_dir is None:
raise exc.InputError('output_dir must be specified to save model')
if iters_summary is not None and output_dir is None:
raise exc.InputError('output_dir must be specified to save ' +
'summaries')
# initialize file writers
if iters_summary is not None:
test_writer = tf.summary.FileWriter(
os.path.join(output_dir, 'summaries', 'test'), sess.graph)
# begin training
with self.graph.as_default():
# start training loop
for iter_ in range(iters_training):
# get batch of data for this training step
x = data.train.next_batch(batch_size)
# draw random samples for latent layer
eps = np.random.normal(size=(batch_size, self.num_lvs))
# one step of optimization routine
sess.run(self.train_step,
feed_dict={
self.x: x[0],
self.labels: x[1],
self.eps: eps
})
# print training updates
if iters_disp is not None and iter_ % iters_disp == 0:
# print updates using test set
x = data.test.next_batch(data.test.num_examples)
eps = np.random.normal(size=(data.test.num_examples,
self.num_lvs))
cost = sess.run(self.cost,
feed_dict={
self.x: x[0],
self.labels: x[1],
self.eps: eps
})
print('Iter %03d:' % iter_)
print(' test cost = %2.5f' % cost)
# save model checkpoints
if iters_ckpt is not None and iter_ % iters_ckpt == 0:
save_file = os.path.join(output_dir, 'ckpts',
str('epoch_%05g.ckpt' % iter_))
self.save_model(sess, save_file)
# save model summaries
if iters_summary is not None and iter_ % iters_summary == 0:
# output summaries using test set
x = data.test.next_batch(data.test.num_examples)
eps = np.random.normal(size=(data.test.num_examples,
self.num_lvs))
summary = sess.run(self.merge_summaries,
feed_dict={
self.x: x[0],
self.labels: x[1],
self.eps: eps
})
test_writer.add_summary(summary, iter_)
def __init__(self,
layers_gen=None,
layers_disc=None,
act_func='relu',
learning_rate=1e-3):
""" Constructor for GAN class
Args:
layers_gen (list of ints): size of each layer in generator,
including output layer
layers_disc (list of ints): size of each layer in discriminator,
including output layer
act_func (str): activation function for network layers
['relu'] | 'sigmoid' | 'tanh' | 'linear' | 'softplus' | 'elu'
learning_rate (scalar): global learning rate for gradient descent
methods
Raises:
InputError if layers_gen is not specified
InputError if layers_disc is not specified
InputError if act_func is not a valid string
"""
# input checking
if layers_gen is None:
raise exc.InputError('Must specify layer sizes for generator')
if layers_disc is None:
raise exc.InputError('Must specify layer sizes for discrimantor')
self.gen_input_size = layers_gen[0]
self.layers_gen = layers_gen[1:]
self.disc_input_size = layers_disc[0]
self.layers_disc = layers_disc[1:]
if act_func == 'relu':
self.act_func = tf.nn.relu
elif act_func == 'sigmoid':
self.act_func = tf.sigmoid
elif act_func == 'tanh':
self.act_func = tf.tanh
elif act_func == 'linear':
self.act_func = tf.identity
elif act_func == 'softplus':
self.act_func = tf.nn.softplus
elif act_func == 'elu':
self.act_func = tf.nn.elu
else:
raise exc.InputError('Invalid activation function')
self.learning_rate = learning_rate
# define useful constants
self.num_layers_gen = len(self.layers_gen)
self.num_layers_disc = len(self.layers_disc)
# for saving and restoring models
self.graph = tf.Graph() # must be initialized before graph creation
with self.graph.as_default():
# define pipeline for feeding data into model
with tf.variable_scope('data'):
self._initialize_data_pipeline()
# initialize weights and create generator
with tf.variable_scope('generator'):
self._define_generator_network()
# initialize weights and create discriminator
self._define_discriminator()
# define loss function
with tf.variable_scope('loss'):
self._define_loss()
# define optimizer
with tf.variable_scope('optimizer'):
self._define_optimizer()
# add additional ops
# for saving and restoring models
self.saver = tf.train.Saver() # must be init after var creation
# collect all summaries into a single op
self.merge_summaries = tf.summary.merge_all()
# add variable initialization op to graph
self.init = tf.global_variables_initializer()
def train(self,
sess,
data=None,
batch_size=128,
epochs_training=10,
epochs_disp=None,
epochs_ckpt=None,
epochs_summary=None,
output_dir=None):
"""Network training
Args:
sess (tf.Session object): current session object to run graph
data (DataReader object): input to network
batch_size (int, optional): batch size used by the gradient
descent-based optimizers
epochs_training (int, optional): number of epochs for gradient
descent-based optimizers
epochs_disp (int, optional): number of epochs between updates to
the console
epochs_ckpt (int, optional): number of epochs between saving
checkpoint files
epochs_summary (int, optional): number of epochs between saving
network summary information
output_dir (string, optional): absolute path for saving checkpoint
files and summary files; must be present if either epochs_ckpt
or epochs_summary is not 'None'.
Returns:
None
Raises:
InputError: If data is not specified
InputError: If epochs_ckpt is not None and output_dir is None
InputError: If epochs_summary is not None and output_dir is None
"""
# check input
if data is None:
raise exc.InputError('data reader must be specified')
if epochs_ckpt is not None and output_dir is None:
raise exc.InputError('output_dir must be specified to save model')
if epochs_summary is not None and output_dir is None:
raise exc.InputError('output_dir must be specified to save ' +
'summaries')
# initialize file writers
if epochs_summary is not None:
test_writer = tf.summary.FileWriter(
os.path.join(output_dir, 'summaries', 'test'), sess.graph)
# define distribution of latent variables
rand_dist = 'normal'
with self.graph.as_default():
# begin training
for epoch in range(epochs_training):
num_batches = int(data.train.num_examples / batch_size)
# start training loop
for batch in range(num_batches):
# one step of optimization routine for disc network
x = data.train.next_batch(batch_size)
if rand_dist == 'normal':
z = np.random.normal(size=(batch_size,
self.gen_input_size))
elif rand_dist == 'uniform':
z = np.random.uniform(low=-1.0,
high=1.0,
size=(batch_size,
self.gen_input_size))
sess.run(self.train_step_disc,
feed_dict={
self.gen_input: z,
self.img_real: x[0]
})
# one step of optimization routine for gen network
if rand_dist == 'normal':
z = np.random.normal(size=(batch_size,
self.gen_input_size))
elif rand_dist == 'uniform':
z = np.random.uniform(low=-1.0,
high=1.0,
size=(batch_size,
self.gen_input_size))
sess.run(self.train_step_gen,
feed_dict={self.gen_input: z})
# print training updates
if epochs_disp is not None and epoch % epochs_disp == 0:
# print updates using test set
x = data.test.next_batch(data.test.num_examples)
if rand_dist == 'normal':
z = np.random.normal(size=(data.test.num_examples,
self.gen_input_size))
elif rand_dist == 'uniform':
z = np.random.uniform(low=-1.0,
high=1.0,
size=(data.test.num_examples,
self.gen_input_size))
[loss_gen,
loss_disc] = sess.run([self.loss_gen, self.loss_disc],
feed_dict={
self.gen_input: z,
self.img_real: x[0]
})
print('Epoch %03d:' % epoch)
print(' test loss gen = %2.5f' % loss_gen)
print(' test loss dis = %2.5f' % loss_disc)
# save model checkpoints
if epochs_ckpt is not None and epoch % epochs_ckpt == 0:
save_file = os.path.join(output_dir, 'ckpts',
str('epoch_%05g.ckpt' % epoch))
self.save_model(sess, save_file)
# save model summaries
if epochs_summary is not None and \
epoch % epochs_summary == 0:
# output summaries using test set
x = data.test.next_batch(data.test.num_examples)
if rand_dist == 'normal':
z = np.random.normal(size=(data.test.num_examples,
self.gen_input_size))
elif rand_dist == 'uniform':
z = np.random.uniform(low=-1.0,
high=1.0,
size=(data.test.num_examples,
self.gen_input_size))
summary = sess.run(self.merge_summaries,
feed_dict={
self.gen_input: z,
self.img_real: x[0]
})
test_writer.add_summary(summary, epoch)
