def evaluate():
"""Evaluate punctuator."""
config = get_config(FLAGS.model)
config.num_steps = 1
config.batch_size = 128
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(
-config.init_scale, config.init_scale)
input_batch, label_batch, mask_batch = punc_input_lstm.eval_inputs(FLAGS.data_path + "/test.pkl",
batch_size=config.batch_size)
with tf.variable_scope("Model", reuse=None, initializer=initializer):
mtest = lstm.LSTMPunctuator(input_batch=input_batch, label_batch=label_batch,
mask_batch=mask_batch, is_training=False, config=config)
sv = tf.train.Supervisor()
with sv.managed_session() as session:
logging.info("Number of parameters: {}".format(utils.count_number_trainable_params()))
ckpt = tf.train.get_checkpoint_state(FLAGS.save_path)
if ckpt and ckpt.model_checkpoint_path:
logging.info("Model checkpoint file path: " + ckpt.model_checkpoint_path)
sv.saver.restore(session, ckpt.model_checkpoint_path)
else:
logging.info("No checkpoint file found")
return
epoch_size = punc_input_lstm.get_epoch_size(FLAGS.data_path + "/test.pkl",
config.batch_size, config.num_steps,
EXAMPLES_PER_FILE=1)
test_perplexity, predicts = lstm.run_epoch(session, mtest, verbose=True, epoch_size=epoch_size, debug=True)
logging.info("Test Perplexity: %.3f" % test_perplexity)
logging.info("predicts' length = {}".format(len(predicts)))
pred_file = os.path.join(FLAGS.save_path, "predict.txt")
with open(pred_file, "w") as f:
f.write(str(predicts) + '\n')
test_data=np.load(FLAGS.data_path + "/test.pkl")
labels = test_data["outputs"][:len(predicts)]
label_file = os.path.join(FLAGS.save_path, "label.txt")
with open(label_file, "w") as f:
f.write(str(labels) + '\n')
precision, recall, fscore, support = score(labels, predicts)#score(predicts, labels)
accuracy = accuracy_score(labels, predicts)
logging.info('precision: {}'.format(precision))
logging.info('recall: {}'.format(recall))
logging.info('fscore: {}'.format(fscore))
logging.info('support: {}'.format(support))
logging.info('accuracy: {}'.format(accuracy))
def get_predicts(inputs, outputs, masks, get_post=False):
config = get_config(FLAGS.model)
config.num_steps = 1
config.batch_size = 1
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
# Generate LSTM batch
input_batch, label_batch, mask_batch = punc_input_lstm.eval_inputs(
"",
batch_size=config.batch_size,
inputs=inputs,
outputs=outputs,
masks=masks)
with tf.variable_scope("Model", reuse=None, initializer=initializer):
mtest = lstm.LSTMPunctuator(input_batch=input_batch,
label_batch=label_batch,
mask_batch=mask_batch,
is_training=False,
config=config)
sv = tf.train.Supervisor()
with sv.managed_session() as session:
logging.info("Number of parameters: {}".format(
utils.count_number_trainable_params()))
ckpt = tf.train.get_checkpoint_state(FLAGS.save_path)
if ckpt and ckpt.model_checkpoint_path:
logging.info("Model checkpoint file path: " +
ckpt.model_checkpoint_path)
sv.saver.restore(session, ckpt.model_checkpoint_path)
else:
logging.info("No checkpoint file found")
return
epoch_size = len(inputs) #// config.batch_size
test_perplexity, predicts = lstm.run_epoch(session,
mtest,
verbose=True,
epoch_size=epoch_size,
get_post=get_post,
debug=False)
logging.info("Test Perplexity: %.3f" % test_perplexity)
return predicts
def train():
""" Train Punctuator for a number of epochs."""
config = get_config(FLAGS.model)
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(
-config.init_scale, config.init_scale)
input_batch, label_batch, mask_batch, files = punc_input_lstm.inputs(
os.path.join(FLAGS.data_path, "train"),
num_steps=config.num_steps,
batch_size=config.batch_size)
with tf.variable_scope("Model", reuse=None, initializer=initializer):
m = out2hidden.LSTMPunctuator2(input_batch=input_batch, label_batch=label_batch,
mask_batch=mask_batch, is_training=True, config=config)
tf.summary.scalar("Training_Loss", m.cost)
tf.summary.scalar("Learning_Rate", m.lr)
sv = tf.train.Supervisor(logdir=FLAGS.save_path)
with sv.managed_session() as session:
logging.info("Number of parameters: {}".format(utils.count_number_trainable_params()))
logging.info(session.run(files))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
epoch_size = punc_input_lstm.get_epoch_size(FLAGS.data_path + "/train.pkl",
config.batch_size, config.num_steps)
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay ** max(i + 1 - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
logging.info("Epoch: %d Learning rate: %.3f" % (i + 1, session.run(m.lr)))
train_perplexity = out2hidden.run_epoch(session, m, eval_op=m.train_op, verbose=True,
epoch_size=epoch_size)
logging.info("Epoch: %d Train Perplexity: %.3f" % (i + 1, train_perplexity))
coord.request_stop()
coord.join(threads)
if FLAGS.save_path:
logging.info("Saving model to %s." % FLAGS.save_path)
sv.saver.save(session, FLAGS.save_path,
global_step=sv.global_step)
def train():
""" Train Punctuator for a number of epochs."""
config = get_config(FLAGS.model)
with tf.Graph().as_default(), tf.device('/cpu:0'):
epoch_size = punc_input_lstm.get_epoch_size(
FLAGS.data_path + "/train.pkl", config.batch_size,
config.num_steps)
epoch_size = epoch_size // FLAGS.num_gpus
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False,
dtype=tf.int32)
lr = tf.Variable(0.0, trainable=False)
tf.summary.scalar("Learning_Rate", lr)
opt = tf.train.GradientDescentOptimizer(lr)
new_lr = tf.placeholder(tf.float32, shape=[], name="new_learning_rate")
lr_update = tf.assign(lr, new_lr)
def assign_lr(session, lr_value):
session.run(lr_update, feed_dict={new_lr: lr_value})
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
input_batch, label_batch, mask_batch, files = punc_input_lstm.inputs(
os.path.join(FLAGS.data_path, "train"),
num_steps=config.num_steps,
batch_size=config.batch_size)
tower_grads = []
with tf.variable_scope("Model", reuse=None, initializer=initializer):
for i in range(FLAGS.num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('gpu_%d' % i) as scope:
m = LSTMPunctuator(input_batch=input_batch,
label_batch=label_batch,
mask_batch=mask_batch,
is_training=True,
config=config)
loss = m.cost
tf.get_variable_scope().reuse_variables()
grads = opt.compute_gradients(loss)
tower_grads.append(grads)
tf.summary.scalar("Training_Loss", m.cost)
with tf.device('/gpu:0'):
grads_and_tvars = average_gradients(tower_grads)
grads = []
tvars = []
for g, t in grads_and_tvars:
grads.append(g)
tvars.append(t)
grads, _ = tf.clip_by_global_norm(grads, config.max_grad_norm)
# Apply the gradients to adjust the shared variables.
# That operation also increments `global_step`.
apply_gradients_op = opt.apply_gradients(zip(grads, tvars),
global_step=global_step)
train_op = apply_gradients_op
sv = tf.train.Supervisor(logdir=FLAGS.save_path)
with sv.managed_session(config=tf.ConfigProto(
allow_soft_placement=True)) as session:
logging.info("Number of parameters: {}".format(
utils.count_number_trainable_params()))
logging.info(session.run(files))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay**max(i + 1 - config.max_epoch, 0.0)
assign_lr(session, config.learning_rate * lr_decay)
logging.info("Epoch: %d Learning rate: %.3f" %
(i + 1, session.run(lr)))
train_perplexity = lstm.run_epoch(session,
m,
eval_op=train_op,
verbose=True,
epoch_size=epoch_size,
num_gpus=FLAGS.num_gpus)
logging.info("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
coord.request_stop()
coord.join(threads)
if FLAGS.save_path:
logging.info("Saving model to %s." % FLAGS.save_path)
sv.saver.save(session,
FLAGS.save_path,
global_step=sv.global_step)
def main(args):
""" Train Punctuator for a number of epochs."""
utils.makedir(args.save_folder)
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-args.init_scale,
args.init_scale)
input_batch, label_batch, mask_batch, files = punc_input_lstm.inputs(
os.path.join(args.train_data, "train"),
num_steps=args.bptt_step,
batch_size=args.batch_size)
with tf.variable_scope("Model", reuse=None, initializer=initializer):
m = lstm.LSTMPunctuator(input_batch=input_batch,
label_batch=label_batch,
mask_batch=mask_batch,
is_training=True,
vocab_size=args.vocab_size,
embedding_size=args.embedding_size,
hidden_size=args.hidden_size,
proj_size=args.proj_size,
hidden_layers=args.hidden_layers,
num_class=args.num_class,
max_norm=args.max_norm,
batch_size=args.batch_size,
bptt_step=args.bptt_step)
tf.summary.scalar("Training_Loss", m.cost)
tf.summary.scalar("Learning_Rate", m.lr)
sv = tf.train.Supervisor(logdir=args.save_folder)
with sv.managed_session() as session:
logging.info(args)
logging.info("Number of parameters: {}".format(
utils.count_number_trainable_params()))
logging.info(session.run(files))
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
epoch_size = punc_input_lstm.get_epoch_size(
args.train_data + "/train.pkl", args.batch_size,
args.bptt_step)
for i in range(args.epochs):
lr_decay = args.lr_decay**max(i + 1 - args.start_decay_epoch,
0.0)
m.assign_lr(session, args.lr * lr_decay)
logging.info("Epoch: %d Learning rate: %.3f" %
(i + 1, session.run(m.lr)))
train_perplexity = lstm.run_epoch(session,
m,
eval_op=m.train_op,
verbose=True,
epoch_size=epoch_size)
logging.info("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
coord.request_stop()
coord.join(threads)
if args.save_folder:
logging.info("Saving model to %s." % args.save_folder)
sv.saver.save(session,
args.save_folder,
global_step=sv.global_step)