else:
decoder = GreedyDecoder(config, model_weights, src_test, trg_test)
decoder.translate()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--config", help="path to json config", required=True)
args = parser.parse_args()
config = read_config(args.config)
model_weights = os.path.join(config['data']['save_dir'],
config['data']['preload_weights'])
src, trg = read_nmt_data(src=config['data']['src'],
config=config,
trg=config['data']['trg'])
src_test, trg_test = read_nmt_data(
src=config['data']['test_src'],
config=config,
trg=None #trg=config['data']['test_trg']
)
if trg_test is None:
trg_test = {}
src_test['word2id'] = src['word2id']
src_test['id2word'] = src['id2word']
trg_test['word2id'] = trg['word2id']
# define a new Handler to log to console as well
console = logging.StreamHandler()
# optional, set the logging level
console.setLevel(logging.INFO)
# set a format which is the same for console use
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
print 'Reading data ...'
src, trg = read_nmt_data(src=config['data']['src'],
config=config,
trg=config['data']['trg'])
src_val, trg_val = read_nmt_data(src=config['data']['val_src'],
config=config,
trg=config['data']['val_trg'])
batch_size = config['data']['batch_size']
max_length = config['data']['max_src_length']
src_vocab_size = len(src['word2id'])
trg_vocab_size = len(trg['word2id'])
logging.info('Model Parameters : ')
logging.info('Task : %s ' % (config['data']['task']))
logging.info('Model : %s ' % (config['model']['seq2seq']))
logging.info('Source Language : %s ' % (config['model']['src_lang']))
def train_nmt(FLAGS=None, buckets=None, save_before_training=False):
"""Train a source->target translation model using some bilingual data."""
assert FLAGS is not None
assert buckets is not None
# Prepare data for training
print('Preparing data in %s' % FLAGS.data_dir)
src_train, tgt_train, src_dev, tgt_dev, _, _ = data_utils.prepare_nmt_data(
FLAGS)
# summary_op = tf.merge_all_summaries()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
nan_detected = False
# Create model.
print('Creating layers.')
if FLAGS.model == "seq2seq":
model = build_ops.create_seq2seq_model(sess,
False,
FLAGS=FLAGS,
buckets=buckets)
else:
model = build_ops.create_nmt_model(sess,
False,
FLAGS=FLAGS,
buckets=buckets)
if save_before_training:
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
# tf.train.write_graph(sess.graph_def, '/home/gian/train2', 'graph.pbtxt')
# Read data into buckets and compute their sizes.
print('Reading development and training data (limit: %d).' %
FLAGS.max_train_data_size)
dev_set = read_nmt_data(src_dev, tgt_dev, FLAGS=FLAGS, buckets=buckets)
train_set = read_nmt_data(src_train,
tgt_train,
max_size=FLAGS.max_train_data_size,
FLAGS=FLAGS,
buckets=buckets)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(buckets))]
train_total_size = float(sum(train_bucket_sizes))
print("Total number of steps per epoch: %d" %
(train_total_size / FLAGS.batch_size))
# A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
# to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
# the size if i-th training bucket, as used later.
train_buckets_scale = [
sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))
]
# This is the training loop.
step_time = 0.0
words_time = 0.0
n_target_words = 0
summary_writer = None
if FLAGS.log_tensorboard:
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
sess.graph_def)
print("Optimization started...")
while model.epoch.eval() < FLAGS.max_epochs:
saved = False
start_time = time.time()
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = numpy.random.random_sample()
bucket_id = min([
i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01
])
# Get a batch and make a step.
encoder_inputs, decoder_inputs, target_weights, n_words = model.get_train_batch(
train_set, bucket_id)
n_target_words += n_words
# session, encoder_inputs, decoder_inputs, target_weights, bucket_id
# note: step loss is averaged across the batch
gradient_norm, step_loss, _ = model.train_step(
session=sess,
encoder_inputs=encoder_inputs,
decoder_inputs=decoder_inputs,
target_weights=target_weights,
bucket_id=bucket_id,
validation_step=False)
current_step = model.global_step.eval()
if summary_writer is not None:
summary_str = sess.run(model.summary_op)
summary_writer.add_summary(summary_str, current_step)
# step_loss = numpy.nan
if numpy.isnan(step_loss) or numpy.isinf(step_loss):
numpy.set_printoptions(linewidth=200)
print('\nNaN detected\n')
nan_detected = True
print("\nStep loss:")
print(step_loss)
print("\nEncoder inputs: ")
print(encoder_inputs)
print("\nDecoder inputs: ")
print(decoder_inputs)
print("\nTarget weights inputs: ")
print(target_weights)
print("\nGradient norm: ")
print(gradient_norm)
break
currloss = model.current_loss.eval()
sess.run(model.current_loss.assign(currloss + step_loss))
# increase the number of seen samples
sess.run(model.samples_seen_update_op)
if current_step % FLAGS.steps_verbosity == 0:
closs = model.current_loss.eval()
gstep = model.global_step.eval()
avgloss = closs / gstep
sess.run(model.avg_loss.assign(avgloss))
target_words_speed = n_target_words / words_time
loss = model.avg_loss.eval()
ppx = math.exp(loss) if loss < 300 else float('inf')
if ppx > 1000.0:
print(
'epoch %d gl.step %d lr.rate %.4f steps-time %.2f avg.loss %.8f avg.ppx > %.8f - avg. %.2f K target words/sec'
% (model.epoch.eval(), model.global_step.eval(),
model.learning_rate.eval(), step_time, loss, 1000.0,
(target_words_speed / 1000.0)))
else:
print(
'epoch %d gl.step %d lr.rate %.4f steps-time %.2f avg.loss %.8f avg.ppx %.8f - avg. %.2f K target words/sec'
% (model.epoch.eval(), model.global_step.eval(),
model.learning_rate.eval(), step_time, loss, ppx,
(target_words_speed / 1000.0)))
n_target_words = 0
step_time = 0.0
words_time = 0.0
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.steps_per_checkpoint == 0:
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir,
FLAGS.model_name)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
saved = True
# update epoch number
if model.samples_seen.eval() >= train_total_size:
sess.run(model.epoch_update_op)
ep = model.epoch.eval()
print("Epoch %d finished..." % (ep - 1))
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir,
FLAGS.model_name)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
if ep >= FLAGS.max_epochs:
if not saved:
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir,
FLAGS.model_name)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
finished = True
break
print("Epoch %d started..." % ep)
sess.run(model.samples_seen_reset_op)
if FLAGS.start_decay > 0:
if FLAGS.stop_decay > 0:
if FLAGS.start_decay <= model.epoch.eval(
) <= FLAGS.stop_decay:
sess.run(model.learning_rate_decay_op)
else:
if FLAGS.start_decay <= model.epoch.eval():
sess.run(model.learning_rate_decay_op)
if current_step % FLAGS.steps_per_validation == 0:
total_eval_loss = 0.0
total_ppx = 0.0
print('\n')
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(buckets)):
n_steps = len(dev_set[bucket_id]) / model.batch_size
bucket_loss = 0.0
for _ in xrange(n_steps):
encoder_inputs, decoder_inputs, target_weights, _ = model.get_train_batch(
dev_set, bucket_id)
_, eval_loss, _ = model.train_step(
session=sess,
encoder_inputs=encoder_inputs,
decoder_inputs=decoder_inputs,
target_weights=target_weights,
bucket_id=bucket_id,
validation_step=True)
bucket_loss += eval_loss
bucket_avg_loss = bucket_loss / n_steps
total_eval_loss += bucket_avg_loss
eval_ppx = math.exp(
bucket_avg_loss) if eval_loss < 300 else float('inf')
total_ppx += eval_ppx
print(' eval: bucket %d perplexity %.4f' %
(bucket_id, eval_ppx))
avg_eval_loss = total_eval_loss / len(buckets)
avg_ppx = math.exp(
avg_eval_loss) if avg_eval_loss < 300 else float('inf')
if avg_ppx > 1000.0:
print('\n eval: averaged perplexity > 1000.0')
else:
print('\n eval: averaged perplexity %.8f' % avg_ppx)
print(' eval: averaged loss %.8f\n' % avg_eval_loss)
sys.stdout.flush()
estop = FLAGS.early_stop_patience
# check early stop - if early stop patience is greater than 0, test it
if estop > 0:
if avg_eval_loss < model.best_eval_loss.eval():
sess.run(model.best_eval_loss.assign(avg_eval_loss))
sess.run(model.estop_counter_reset_op)
# Save checkpoint
print('Saving the best model so far...')
best_model_path = os.path.join(
FLAGS.best_models_dir, FLAGS.model_name + '-best')
model.saver_best.save(sess,
best_model_path,
global_step=model.global_step)
else:
# if FLAGS.early_stop_after_epoch is equal to 0, it will monitor from the beginning
if model.epoch.eval() >= FLAGS.early_stop_after_epoch:
sess.run(model.estop_counter_update_op)
if model.estop_counter.eval() >= estop:
print('\nEARLY STOP!\n')
finished = True
break
print('\n best valid. loss: %.8f' %
model.best_eval_loss.eval())
print('early stop patience: %d - max %d\n' %
(int(model.estop_counter.eval()), estop))
step_time += (time.time() - start_time) / FLAGS.steps_verbosity
words_time += (time.time() - start_time)
print("\nTraining finished!!\n")
if not nan_detected:
# # Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess,
checkpoint_path,
global_step=model.global_step)
print("Final validation:")
total_eval_loss = 0.0
total_ppx = 0.0
print('\n')
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(buckets)):
n_steps = len(dev_set[bucket_id]) / model.batch_size
bucket_loss = 0.0
for _ in xrange(n_steps):
encoder_inputs, decoder_inputs, target_weights, _ = model.get_train_batch(
dev_set, bucket_id)
_, eval_loss, _ = model.train_step(
session=sess,
encoder_inputs=encoder_inputs,
decoder_inputs=decoder_inputs,
target_weights=target_weights,
bucket_id=bucket_id)
bucket_loss += eval_loss
bucket_avg_loss = bucket_loss / n_steps
total_eval_loss += bucket_avg_loss
eval_ppx = math.exp(
bucket_avg_loss) if eval_loss < 300 else float('inf')
total_ppx += eval_ppx
print(' eval: bucket %d perplexity %.4f' %
(bucket_id, eval_ppx))
avg_eval_loss = total_eval_loss / len(buckets)
avg_ppx = math.exp(
avg_eval_loss) if avg_eval_loss < 300 else float('inf')
if avg_ppx > 1000.0:
print('\n eval: averaged perplexity > 1000.0')
else:
print('\n eval: averaged perplexity %.8f' % avg_ppx)
print(' eval: averaged loss %.8f\n' % avg_eval_loss)
print('\n best valid. loss during training: %.8f' %
model.best_eval_loss.eval())
sys.stdout.flush()
def train_nmt(FLAGS=None, buckets=None, save_before_training=False):
"""Train a source->target translation model using some bilingual data."""
assert FLAGS is not None
assert buckets is not None
# Prepare data for training
print('Preparing data in %s' % FLAGS.data_dir)
src_train, tgt_train, src_dev, tgt_dev, _, _ = data_utils.prepare_nmt_data(FLAGS)
# summary_op = tf.merge_all_summaries()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
nan_detected = False
# Create model.
print('Creating layers.')
if FLAGS.model == "seq2seq":
model = build_ops.create_seq2seq_model(sess, False, FLAGS=FLAGS, buckets=buckets)
else:
model = build_ops.create_nmt_model(sess, False, FLAGS=FLAGS, buckets=buckets)
if save_before_training:
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
# tf.train.write_graph(sess.graph_def, '/home/gian/train2', 'graph.pbtxt')
# Read data into buckets and compute their sizes.
print('Reading development and training data (limit: %d).' % FLAGS.max_train_data_size)
dev_set = read_nmt_data(src_dev, tgt_dev, FLAGS=FLAGS, buckets=buckets)
train_set = read_nmt_data(src_train, tgt_train, max_size=FLAGS.max_train_data_size, FLAGS=FLAGS,
buckets=buckets)
train_bucket_sizes = [len(train_set[b]) for b in xrange(len(buckets))]
train_total_size = float(sum(train_bucket_sizes))
print("Total number of steps per epoch: %d" % (train_total_size / FLAGS.batch_size))
# A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
# to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
# the size if i-th training bucket, as used later.
train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
for i in xrange(len(train_bucket_sizes))]
# This is the training loop.
step_time = 0.0
words_time = 0.0
n_target_words = 0
summary_writer = None
if FLAGS.log_tensorboard:
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph_def)
print("Optimization started...")
while model.epoch.eval() < FLAGS.max_epochs:
saved = False
start_time = time.time()
# Choose a bucket according to data distribution. We pick a random number
# in [0, 1] and use the corresponding interval in train_buckets_scale.
random_number_01 = numpy.random.random_sample()
bucket_id = min([i for i in xrange(len(train_buckets_scale))
if train_buckets_scale[i] > random_number_01])
# Get a batch and make a step.
encoder_inputs, decoder_inputs, target_weights, n_words = model.get_train_batch(
train_set, bucket_id
)
n_target_words += n_words
# session, encoder_inputs, decoder_inputs, target_weights, bucket_id
# note: step loss is averaged across the batch
gradient_norm, step_loss, _ = model.train_step(session=sess, encoder_inputs=encoder_inputs,
decoder_inputs=decoder_inputs,
target_weights=target_weights,
bucket_id=bucket_id,
validation_step=False)
current_step = model.global_step.eval()
if summary_writer is not None:
summary_str = sess.run(model.summary_op)
summary_writer.add_summary(summary_str, current_step)
# step_loss = numpy.nan
if numpy.isnan(step_loss) or numpy.isinf(step_loss):
numpy.set_printoptions(linewidth=200)
print('\nNaN detected\n')
nan_detected = True
print("\nStep loss:")
print(step_loss)
print("\nEncoder inputs: ")
print(encoder_inputs)
print("\nDecoder inputs: ")
print(decoder_inputs)
print("\nTarget weights inputs: ")
print(target_weights)
print("\nGradient norm: ")
print(gradient_norm)
break
currloss = model.current_loss.eval()
sess.run(model.current_loss.assign(currloss + step_loss))
# increase the number of seen samples
sess.run(model.samples_seen_update_op)
if current_step % FLAGS.steps_verbosity == 0:
closs = model.current_loss.eval()
gstep = model.global_step.eval()
avgloss = closs / gstep
sess.run(model.avg_loss.assign(avgloss))
target_words_speed = n_target_words / words_time
loss = model.avg_loss.eval()
ppx = math.exp(loss) if loss < 300 else float('inf')
if ppx > 1000.0:
print(
'epoch %d gl.step %d lr.rate %.4f steps-time %.2f avg.loss %.8f avg.ppx > %.8f - avg. %.2f K target words/sec' %
(model.epoch.eval(), model.global_step.eval(), model.learning_rate.eval(),
step_time, loss, 1000.0, (target_words_speed / 1000.0)))
else:
print(
'epoch %d gl.step %d lr.rate %.4f steps-time %.2f avg.loss %.8f avg.ppx %.8f - avg. %.2f K target words/sec' %
(model.epoch.eval(), model.global_step.eval(), model.learning_rate.eval(),
step_time, loss, ppx, (target_words_speed / 1000.0)))
n_target_words = 0
step_time = 0.0
words_time = 0.0
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.steps_per_checkpoint == 0:
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
saved = True
# update epoch number
if model.samples_seen.eval() >= train_total_size:
sess.run(model.epoch_update_op)
ep = model.epoch.eval()
print("Epoch %d finished..." % (ep - 1))
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
if ep >= FLAGS.max_epochs:
if not saved:
# Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
finished = True
break
print("Epoch %d started..." % ep)
sess.run(model.samples_seen_reset_op)
if FLAGS.start_decay > 0:
if FLAGS.stop_decay > 0:
if FLAGS.start_decay <= model.epoch.eval() <= FLAGS.stop_decay:
sess.run(model.learning_rate_decay_op)
else:
if FLAGS.start_decay <= model.epoch.eval():
sess.run(model.learning_rate_decay_op)
if current_step % FLAGS.steps_per_validation == 0:
total_eval_loss = 0.0
total_ppx = 0.0
print('\n')
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(buckets)):
n_steps = len(dev_set[bucket_id]) / model.batch_size
bucket_loss = 0.0
for _ in xrange(n_steps):
encoder_inputs, decoder_inputs, target_weights, _ = model.get_train_batch(dev_set,
bucket_id)
_, eval_loss, _ = model.train_step(session=sess, encoder_inputs=encoder_inputs,
decoder_inputs=decoder_inputs, target_weights=target_weights,
bucket_id=bucket_id, validation_step=True)
bucket_loss += eval_loss
bucket_avg_loss = bucket_loss / n_steps
total_eval_loss += bucket_avg_loss
eval_ppx = math.exp(bucket_avg_loss) if eval_loss < 300 else float('inf')
total_ppx += eval_ppx
print(' eval: bucket %d perplexity %.4f' % (bucket_id, eval_ppx))
avg_eval_loss = total_eval_loss / len(buckets)
avg_ppx = math.exp(avg_eval_loss) if avg_eval_loss < 300 else float('inf')
if avg_ppx > 1000.0:
print('\n eval: averaged perplexity > 1000.0')
else:
print('\n eval: averaged perplexity %.8f' % avg_ppx)
print(' eval: averaged loss %.8f\n' % avg_eval_loss)
sys.stdout.flush()
estop = FLAGS.early_stop_patience
# check early stop - if early stop patience is greater than 0, test it
if estop > 0:
if avg_eval_loss < model.best_eval_loss.eval():
sess.run(model.best_eval_loss.assign(avg_eval_loss))
sess.run(model.estop_counter_reset_op)
# Save checkpoint
print('Saving the best model so far...')
best_model_path = os.path.join(FLAGS.best_models_dir, FLAGS.model_name + '-best')
model.saver_best.save(sess, best_model_path, global_step=model.global_step)
else:
# if FLAGS.early_stop_after_epoch is equal to 0, it will monitor from the beginning
if model.epoch.eval() >= FLAGS.early_stop_after_epoch:
sess.run(model.estop_counter_update_op)
if model.estop_counter.eval() >= estop:
print('\nEARLY STOP!\n')
finished = True
break
print('\n best valid. loss: %.8f' % model.best_eval_loss.eval())
print('early stop patience: %d - max %d\n' % (int(model.estop_counter.eval()), estop))
step_time += (time.time() - start_time) / FLAGS.steps_verbosity
words_time += (time.time() - start_time)
print("\nTraining finished!!\n")
if not nan_detected:
# # Save checkpoint
checkpoint_path = os.path.join(FLAGS.train_dir, FLAGS.model_name)
model.saver.save(sess, checkpoint_path, global_step=model.global_step)
print("Final validation:")
total_eval_loss = 0.0
total_ppx = 0.0
print('\n')
# Run evals on development set and print their perplexity.
for bucket_id in xrange(len(buckets)):
n_steps = len(dev_set[bucket_id]) / model.batch_size
bucket_loss = 0.0
for _ in xrange(n_steps):
encoder_inputs, decoder_inputs, target_weights, _ = model.get_train_batch(dev_set,
bucket_id)
_, eval_loss, _ = model.train_step(session=sess, encoder_inputs=encoder_inputs,
decoder_inputs=decoder_inputs, target_weights=target_weights,
bucket_id=bucket_id)
bucket_loss += eval_loss
bucket_avg_loss = bucket_loss / n_steps
total_eval_loss += bucket_avg_loss
eval_ppx = math.exp(bucket_avg_loss) if eval_loss < 300 else float('inf')
total_ppx += eval_ppx
print(' eval: bucket %d perplexity %.4f' % (bucket_id, eval_ppx))
avg_eval_loss = total_eval_loss / len(buckets)
avg_ppx = math.exp(avg_eval_loss) if avg_eval_loss < 300 else float('inf')
if avg_ppx > 1000.0:
print('\n eval: averaged perplexity > 1000.0')
else:
print('\n eval: averaged perplexity %.8f' % avg_ppx)
print(' eval: averaged loss %.8f\n' % avg_eval_loss)
print('\n best valid. loss during training: %.8f' % model.best_eval_loss.eval())
sys.stdout.flush()
filemode='w')
# define a new Handler to log to console as well
console = logging.StreamHandler()
# optional, set the logging level
console.setLevel(logging.INFO)
# set a format which is the same for console use
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
# tell the handler to use this format
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
print 'Reading data ...'
src, _ = read_nmt_data(src=config['data']['src'], trg=None)
src_test, _ = read_nmt_data(src=config['data']['test_src'], trg=None)
batch_size = config['data']['batch_size']
max_length = config['data']['max_src_length']
src_vocab_size = len(src['word2id'])
logging.info('Model Parameters : ')
logging.info('Task : %s ' % (config['data']['task']))
logging.info('Model : %s ' % (config['model']['seq2seq']))
logging.info('Source Language : %s ' % (config['model']['src_lang']))
logging.info('Target Language : %s ' % (config['model']['src_lang']))
logging.info('Source Word Embedding Dim : %s' %
(config['model']['dim_word_src']))
logging.info('Target Word Embedding Dim : %s' %