def train_generator(args, load_recent=True): '''Train the generator via classical approach''' logging.debug('Batcher...') batcher = Batcher(args.data_dir, args.batch_size, args.seq_length) logging.debug('Vocabulary...') with open(os.path.join(args.save_dir_gen, 'config.pkl'), 'w') as f: cPickle.dump(args, f) with open(os.path.join(args.save_dir_gen, 'real_beer_vocab.pkl'), 'w') as f: cPickle.dump((batcher.chars, batcher.vocab), f) logging.debug('Creating generator...') generator = Generator(args, is_training=True) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess: tf.initialize_all_variables().run() saver = tf.train.Saver(tf.all_variables()) if load_recent: ckpt = tf.train.get_checkpoint_state(args.save_dir_gen) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) for epoch in xrange(args.num_epochs): # Anneal learning rate new_lr = args.learning_rate * (args.decay_rate**epoch) sess.run(tf.assign(generator.lr, new_lr)) batcher.reset_batch_pointer() state = generator.initial_state.eval() for batch in xrange(batcher.num_batches): start = time.time() x, y = batcher.next_batch() feed = { generator.input_data: x, generator.targets: y, generator.initial_state: state } # train_loss, state, _ = sess.run([generator.cost, generator.final_state, generator.train_op], feed) train_loss, _ = sess.run([generator.cost, generator.train_op], feed) end = time.time() print '{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}' \ .format(epoch * batcher.num_batches + batch, args.num_epochs * batcher.num_batches, epoch, train_loss, end - start) if (epoch * batcher.num_batches + batch) % args.save_every == 0: checkpoint_path = os.path.join(args.save_dir_gen, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=epoch * batcher.num_batches + batch) print 'Generator model saved to {}'.format(checkpoint_path)
def train(params): data_loader = Batcher(params) params.vocab_size = data_loader.vocab_size if not os.path.isdir(params.save_dir): os.makedirs(params.save_dir) with open(os.path.join(params.save_dir, 'config.pkl'), 'wb') as f: cPickle.dump(params, f) with open(os.path.join(params.save_dir, 'chars_vocab.pkl'), 'wb') as f: cPickle.dump((data_loader.chars, data_loader.vocab), f) model = Model(params) with tf.Session() as sess: summaries = tf.summary.merge_all() writer = tf.summary.FileWriter( os.path.join(params.log_dir, time.strftime("%Y-%m-%d-%H-%M-%S"))) writer.add_graph(sess.graph) sess.run(tf.global_variables_initializer()) saver = tf.train.Saver(tf.global_variables(), max_to_keep=50) for e in range(params.num_epochs): sess.run(tf.assign(model.lr, params.learning_rate * (0.97**e))) data_loader.reset_batch_pointer() state = sess.run(model.initial_state) for b in range(data_loader.num_batches): start = time.time() x, y = data_loader.next_batch() feed = {model.input_data: x, model.targets: y} for i, (c, h) in enumerate(model.initial_state): feed[c] = state[i].c feed[h] = state[i].h train_loss, state, _ = sess.run( [model.cost, model.final_state, model.train_op], feed) summ, train_loss, state, _ = sess.run( [summaries, model.cost, model.final_state, model.train_op], feed) writer.add_summary(summ, e * data_loader.num_batches + b) end = time.time() logging.info( "Epoch #{e} / Batch #{b} -- Loss {train_loss:.3f} " "Time {time_diff:.3f}".format(e=e, b=b, train_loss=train_loss, time_diff=end - start)) if e % params.save_every == 0 or e == params.num_epochs - 1: checkpoint_path = os.path.join(params.save_dir, 'model.ckpt') saver.save(sess, checkpoint_path, global_step=e)
def train_generator(args, load_recent=True): '''Train the generator via classical approach''' logging.debug('Batcher...') batcher = Batcher(args.data_dir, args.batch_size, args.seq_length) logging.debug('Vocabulary...') with open(os.path.join(args.save_dir_gen, 'config.pkl'), 'w') as f: cPickle.dump(args, f) with open(os.path.join(args.save_dir_gen, 'real_beer_vocab.pkl'), 'w') as f: cPickle.dump((batcher.chars, batcher.vocab), f) logging.debug('Creating generator...') generator = Generator(args, is_training = True) with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True)) as sess: tf.initialize_all_variables().run() saver = tf.train.Saver(tf.all_variables()) if load_recent: ckpt = tf.train.get_checkpoint_state(args.save_dir_gen) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) for epoch in xrange(args.num_epochs): # Anneal learning rate new_lr = args.learning_rate * (args.decay_rate ** epoch) sess.run(tf.assign(generator.lr, new_lr)) batcher.reset_batch_pointer() state = generator.initial_state.eval() for batch in xrange(batcher.num_batches): start = time.time() x, y = batcher.next_batch() feed = {generator.input_data: x, generator.targets: y, generator.initial_state: state} # train_loss, state, _ = sess.run([generator.cost, generator.final_state, generator.train_op], feed) train_loss, _ = sess.run([generator.cost, generator.train_op], feed) end = time.time() print '{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}' \ .format(epoch * batcher.num_batches + batch, args.num_epochs * batcher.num_batches, epoch, train_loss, end - start) if (epoch * batcher.num_batches + batch) % args.save_every == 0: checkpoint_path = os.path.join(args.save_dir_gen, 'model.ckpt') saver.save(sess, checkpoint_path, global_step = epoch * batcher.num_batches + batch) print 'Generator model saved to {}'.format(checkpoint_path)