def train_generator(args, load_recent=True):
'''Train the generator via classical approach'''
logging.debug('Batcher...')
batcher = Batcher(args.data_dir, args.batch_size, args.seq_length)
logging.debug('Vocabulary...')
with open(os.path.join(args.save_dir_gen, 'config.pkl'), 'w') as f:
cPickle.dump(args, f)
with open(os.path.join(args.save_dir_gen, 'real_beer_vocab.pkl'),
'w') as f:
cPickle.dump((batcher.chars, batcher.vocab), f)
logging.debug('Creating generator...')
generator = Generator(args, is_training=True)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=True)) as sess:
tf.initialize_all_variables().run()
saver = tf.train.Saver(tf.all_variables())
if load_recent:
ckpt = tf.train.get_checkpoint_state(args.save_dir_gen)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for epoch in xrange(args.num_epochs):
# Anneal learning rate
new_lr = args.learning_rate * (args.decay_rate**epoch)
sess.run(tf.assign(generator.lr, new_lr))
batcher.reset_batch_pointer()
state = generator.initial_state.eval()
for batch in xrange(batcher.num_batches):
start = time.time()
x, y = batcher.next_batch()
feed = {
generator.input_data: x,
generator.targets: y,
generator.initial_state: state
}
# train_loss, state, _ = sess.run([generator.cost, generator.final_state, generator.train_op], feed)
train_loss, _ = sess.run([generator.cost, generator.train_op],
feed)
end = time.time()
print '{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}' \
.format(epoch * batcher.num_batches + batch,
args.num_epochs * batcher.num_batches,
epoch, train_loss, end - start)
if (epoch * batcher.num_batches +
batch) % args.save_every == 0:
checkpoint_path = os.path.join(args.save_dir_gen,
'model.ckpt')
saver.save(sess,
checkpoint_path,
global_step=epoch * batcher.num_batches + batch)
print 'Generator model saved to {}'.format(checkpoint_path)
def train(params):
data_loader = Batcher(params)
params.vocab_size = data_loader.vocab_size
if not os.path.isdir(params.save_dir):
os.makedirs(params.save_dir)
with open(os.path.join(params.save_dir, 'config.pkl'), 'wb') as f:
cPickle.dump(params, f)
with open(os.path.join(params.save_dir, 'chars_vocab.pkl'), 'wb') as f:
cPickle.dump((data_loader.chars, data_loader.vocab), f)
model = Model(params)
with tf.Session() as sess:
summaries = tf.summary.merge_all()
writer = tf.summary.FileWriter(
os.path.join(params.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S")))
writer.add_graph(sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=50)
for e in range(params.num_epochs):
sess.run(tf.assign(model.lr, params.learning_rate * (0.97**e)))
data_loader.reset_batch_pointer()
state = sess.run(model.initial_state)
for b in range(data_loader.num_batches):
start = time.time()
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.targets: y}
for i, (c, h) in enumerate(model.initial_state):
feed[c] = state[i].c
feed[h] = state[i].h
train_loss, state, _ = sess.run(
[model.cost, model.final_state, model.train_op], feed)
summ, train_loss, state, _ = sess.run(
[summaries, model.cost, model.final_state, model.train_op],
feed)
writer.add_summary(summ, e * data_loader.num_batches + b)
end = time.time()
logging.info(
"Epoch #{e} / Batch #{b} -- Loss {train_loss:.3f} "
"Time {time_diff:.3f}".format(e=e,
b=b,
train_loss=train_loss,
time_diff=end - start))
if e % params.save_every == 0 or e == params.num_epochs - 1:
checkpoint_path = os.path.join(params.save_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=e)
def train_generator(args, load_recent=True):
'''Train the generator via classical approach'''
logging.debug('Batcher...')
batcher = Batcher(args.data_dir, args.batch_size, args.seq_length)
logging.debug('Vocabulary...')
with open(os.path.join(args.save_dir_gen, 'config.pkl'), 'w') as f:
cPickle.dump(args, f)
with open(os.path.join(args.save_dir_gen, 'real_beer_vocab.pkl'), 'w') as f:
cPickle.dump((batcher.chars, batcher.vocab), f)
logging.debug('Creating generator...')
generator = Generator(args, is_training = True)
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess:
tf.initialize_all_variables().run()
saver = tf.train.Saver(tf.all_variables())
if load_recent:
ckpt = tf.train.get_checkpoint_state(args.save_dir_gen)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for epoch in xrange(args.num_epochs):
# Anneal learning rate
new_lr = args.learning_rate * (args.decay_rate ** epoch)
sess.run(tf.assign(generator.lr, new_lr))
batcher.reset_batch_pointer()
state = generator.initial_state.eval()
for batch in xrange(batcher.num_batches):
start = time.time()
x, y = batcher.next_batch()
feed = {generator.input_data: x, generator.targets: y, generator.initial_state: state}
# train_loss, state, _ = sess.run([generator.cost, generator.final_state, generator.train_op], feed)
train_loss, _ = sess.run([generator.cost, generator.train_op], feed)
end = time.time()
print '{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}' \
.format(epoch * batcher.num_batches + batch,
args.num_epochs * batcher.num_batches,
epoch, train_loss, end - start)
if (epoch * batcher.num_batches + batch) % args.save_every == 0:
checkpoint_path = os.path.join(args.save_dir_gen, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step = epoch * batcher.num_batches + batch)
print 'Generator model saved to {}'.format(checkpoint_path)
