def inference_bulk(config):
"""Inference for test file
"""
# Build input data
test_file = 'data/test.txt'
test_data = Input(test_file, config)
print('max_sentence_length = %d' % test_data.max_sentence_length)
print('loading input data ... done')
# Create model
model = Model(config)
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, config.restore)
print('model restored')
feed_dict = {
model.input_data_word_ids: test_data.sentence_word_ids,
model.input_data_wordchr_ids: test_data.sentence_wordchr_ids,
model.input_data_pos_ids: test_data.sentence_pos_ids,
model.input_data_etc: test_data.sentence_etc,
model.output_data: test_data.sentence_tag
}
logits, logits_indices, trans_params, output_data_indices, length, test_loss = \
sess.run([model.logits, model.logits_indices, model.trans_params, model.output_data_indices, model.length, model.loss], feed_dict=feed_dict)
print('test precision, recall, f1(token): ')
TokenEval.compute_f1(config.class_size, logits, test_data.sentence_tag,
length)
if config.use_crf:
viterbi_sequences = viterbi_decode(logits, trans_params, length)
tag_preds = test_data.logits_indices_to_tags_seq(
viterbi_sequences, length)
else:
tag_preds = test_data.logits_indices_to_tags_seq(
logits_indices, length)
tag_corrects = test_data.logits_indices_to_tags_seq(
output_data_indices, length)
test_prec, test_rec, test_f1 = ChunkEval.compute_f1(
tag_preds, tag_corrects)
print('test precision, recall, f1(chunk): ', test_prec, test_rec,
test_f1)
def dev_step(model, data, summary_writer, epoch):
"""Evaluate dev data
"""
sess = model.sess
runopts = tf.RunOptions(report_tensor_allocations_upon_oom=True)
sum_loss = 0.0
sum_accuracy = 0.0
sum_f1 = 0.0
sum_output_indices = None
sum_logits_indices = None
sum_sentence_lengths = None
trans_params = None
global_step = 0
prog = Progbar(target=data.num_batches)
iterator = data.dataset.make_initializable_iterator()
next_element = iterator.get_next()
sess.run(iterator.initializer)
# evaluate on dev data sliced by batch_size to prevent OOM(Out Of Memory).
for idx in range(data.num_batches):
try:
dataset = sess.run(next_element)
except tf.errors.OutOfRangeError:
break
feed_dict = feed.build_feed_dict(model, dataset,
data.max_sentence_length, False)
if 'bert' in model.config.emb_class:
# compute bert embedding at runtime
bert_embeddings = sess.run([model.bert_embeddings_subgraph],
feed_dict=feed_dict,
options=runopts)
# update feed_dict
feed.update_feed_dict(model, feed_dict, bert_embeddings,
dataset['bert_wordidx2tokenidx'])
global_step, logits_indices, sentence_lengths, loss, accuracy, f1 = \
sess.run([model.global_step, model.logits_indices, model.sentence_lengths, \
model.loss, model.accuracy, model.f1], feed_dict=feed_dict)
prog.update(idx + 1, [('dev loss', loss), ('dev accuracy', accuracy),
('dev f1', f1)])
sum_loss += loss
sum_accuracy += accuracy
sum_f1 += f1
sum_output_indices = np_concat(sum_output_indices,
np.argmax(dataset['tags'], 2))
sum_logits_indices = np_concat(sum_logits_indices, logits_indices)
sum_sentence_lengths = np_concat(sum_sentence_lengths,
sentence_lengths)
idx += 1
avg_loss = sum_loss / data.num_batches
avg_accuracy = sum_accuracy / data.num_batches
avg_f1 = sum_f1 / data.num_batches
tag_preds = model.config.logits_indices_to_tags_seq(
sum_logits_indices, sum_sentence_lengths)
tag_corrects = model.config.logits_indices_to_tags_seq(
sum_output_indices, sum_sentence_lengths)
tf.logging.debug('\n[epoch %s/%s] dev precision, recall, f1(token): ' %
(epoch, model.config.epoch))
token_f1, l_token_prec, l_token_rec, l_token_f1 = TokenEval.compute_f1(
model.config.class_size, sum_logits_indices, sum_output_indices,
sum_sentence_lengths)
tf.logging.debug('[' + ' '.join([str(x) for x in l_token_prec]) + ']')
tf.logging.debug('[' + ' '.join([str(x) for x in l_token_rec]) + ']')
tf.logging.debug('[' + ' '.join([str(x) for x in l_token_f1]) + ']')
chunk_prec, chunk_rec, chunk_f1 = ChunkEval.compute_f1(
tag_preds, tag_corrects)
tf.logging.debug('dev precision(chunk), recall(chunk), f1(chunk): %s, %s, %s' % \
(chunk_prec, chunk_rec, chunk_f1) + \
'(invalid for bert due to X tag)')
# create summaries manually.
summary_value = [
tf.Summary.Value(tag='loss', simple_value=avg_loss),
tf.Summary.Value(tag='accuracy', simple_value=avg_accuracy),
tf.Summary.Value(tag='f1', simple_value=avg_f1),
tf.Summary.Value(tag='token_f1', simple_value=token_f1),
tf.Summary.Value(tag='chunk_f1', simple_value=chunk_f1)
]
summaries = tf.Summary(value=summary_value)
summary_writer.add_summary(summaries, global_step)
return token_f1, chunk_f1, avg_f1
def dev_step(sess, model, config, data, summary_writer, epoch):
idx = 0
nbatches = (len(data.sentence_tags) + config.dev_batch_size -
1) // config.dev_batch_size
prog = Progbar(target=nbatches)
sum_loss = 0.0
sum_accuracy = 0.0
sum_logits = None
sum_sentence_lengths = None
trans_params = None
global_step = 0
# evaluate on dev data sliced by dev_batch_size to prevent OOM
for ptr in range(0, len(data.sentence_tags), config.dev_batch_size):
config.is_training = False
feed_dict = {
model.input_data_pos_ids:
data.sentence_pos_ids[ptr:ptr + config.dev_batch_size],
model.output_data:
data.sentence_tags[ptr:ptr + config.dev_batch_size],
model.is_train:
config.is_training,
model.sentence_length:
data.max_sentence_length
}
feed_dict[model.input_data_word_ids] = data.sentence_word_ids[
ptr:ptr + config.dev_batch_size]
feed_dict[model.input_data_wordchr_ids] = data.sentence_wordchr_ids[
ptr:ptr + config.dev_batch_size]
if config.emb_class == 'elmo':
feed_dict[
model.
elmo_input_data_wordchr_ids] = data.sentence_elmo_wordchr_ids[
ptr:ptr + config.dev_batch_size]
if config.emb_class == 'bert':
feed_dict[
model.
bert_input_data_token_ids] = data.sentence_bert_token_ids[
ptr:ptr + config.batch_size]
feed_dict[
model.
bert_input_data_token_masks] = data.sentence_bert_token_masks[
ptr:ptr + config.batch_size]
feed_dict[
model.
bert_input_data_segment_ids] = data.sentence_bert_segment_ids[
ptr:ptr + config.batch_size]
global_step, logits, trans_params, sentence_lengths, loss, accuracy = \
sess.run([model.global_step, model.logits, model.trans_params, model.sentence_lengths, \
model.loss, model.accuracy], feed_dict=feed_dict)
prog.update(idx + 1, [('dev loss', loss), ('dev accuracy', accuracy)])
sum_loss += loss
sum_accuracy += accuracy
sum_logits = np_concat(sum_logits, logits)
sum_sentence_lengths = np_concat(sum_sentence_lengths,
sentence_lengths)
idx += 1
sum_loss = sum_loss / nbatches
sum_accuracy = sum_accuracy / nbatches
print('[epoch %s/%s] dev precision, recall, f1(token): ' %
(epoch, config.epoch))
token_f1 = TokenEval.compute_f1(config.class_size, sum_logits,
data.sentence_tags, sum_sentence_lengths)
if config.use_crf:
viterbi_sequences = viterbi_decode(sum_logits, trans_params,
sum_sentence_lengths)
tag_preds = data.logits_indices_to_tags_seq(viterbi_sequences,
sum_sentence_lengths)
else:
sum_logits_indices = np.argmax(sum_logits, 2)
tag_preds = data.logits_indices_to_tags_seq(sum_logits_indices,
sum_sentence_lengths)
sum_output_data_indices = np.argmax(data.sentence_tags, 2)
tag_corrects = data.logits_indices_to_tags_seq(sum_output_data_indices,
sum_sentence_lengths)
prec, rec, f1 = ChunkEval.compute_f1(tag_preds, tag_corrects)
print('dev precision, recall, f1(chunk): ', prec, rec, f1,
', this is no meaningful for emb_class=bert')
chunk_f1 = f1
m = chunk_f1
# create summaries manually
summary_value = [
tf.Summary.Value(tag='loss_1', simple_value=sum_loss),
tf.Summary.Value(tag='accuracy_1', simple_value=sum_accuracy),
tf.Summary.Value(tag='token_f1', simple_value=token_f1),
tf.Summary.Value(tag='chunk_f1', simple_value=chunk_f1)
]
summaries = tf.Summary(value=summary_value)
summary_writer.add_summary(summaries, global_step)
m = token_f1
return m
def do_train(model, config, train_data, dev_data, test_data):
learning_rate_init=0.001 # initial
learning_rate_final=0.0001 # final
learning_rate=learning_rate_init
intermid_epoch = 20 # after this epoch, change learning rate
maximum = 0
session_conf = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
if config.restore is not None:
saver.restore(sess, config.restore)
print('model restored')
# summary setting
loss_summary = tf.summary.scalar('loss', model.loss)
acc_summary = tf.summary.scalar('accuracy', model.accuracy)
train_summary_op = tf.summary.merge([loss_summary, acc_summary])
train_summary_dir = os.path.join(config.summary_dir, 'summaries', 'train')
train_summary_writer = tf.summary.FileWriter(train_summary_dir, sess.graph)
dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
dev_summary_dir = os.path.join(config.summary_dir, 'summaries', 'dev')
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir, sess.graph)
# training steps
for e in range(config.epoch):
# run epoch
idx = 0
nbatches = (len(train_data.sentence_word_ids) + config.batch_size - 1) // config.batch_size
prog = Progbar(target=nbatches)
for ptr in range(0, len(train_data.sentence_word_ids), config.batch_size):
feed_dict={model.input_data_word_ids: train_data.sentence_word_ids[ptr:ptr + config.batch_size],
model.input_data_wordchr_ids: train_data.sentence_wordchr_ids[ptr:ptr + config.batch_size],
model.input_data_pos_ids: train_data.sentence_pos_ids[ptr:ptr + config.batch_size],
model.input_data_etc: train_data.sentence_etc[ptr:ptr + config.batch_size],
model.output_data: train_data.sentence_tag[ptr:ptr + config.batch_size],
model.learning_rate:learning_rate}
step, train_summaries, _, train_loss, train_accuracy = \
sess.run([model.global_step, train_summary_op, model.train_op, model.loss, model.accuracy], feed_dict=feed_dict)
prog.update(idx + 1, [('train loss', train_loss), ('train accuracy', train_accuracy)])
train_summary_writer.add_summary(train_summaries, step)
idx += 1
# evaluate on dev data
feed_dict={model.input_data_word_ids: dev_data.sentence_word_ids,
model.input_data_wordchr_ids: dev_data.sentence_wordchr_ids,
model.input_data_pos_ids: dev_data.sentence_pos_ids,
model.input_data_etc: dev_data.sentence_etc,
model.output_data: dev_data.sentence_tag}
step, dev_summaries, logits, logits_indices, trans_params, output_data_indices, length, dev_loss, dev_accuracy = \
sess.run([model.global_step, dev_summary_op, model.logits, model.logits_indices, model.trans_params, model.output_data_indices, model.length, model.loss, model.accuracy], feed_dict=feed_dict)
print('epoch: %d / %d, step: %d, dev loss: %s, dev accuracy: %s' % (e, config.epoch, step, dev_loss, dev_accuracy))
dev_summary_writer.add_summary(dev_summaries, step)
print('dev precision, recall, f1(token): ')
token_f1 = TokenEval.compute_f1(config.class_size, logits, dev_data.sentence_tag, length)
if config.use_crf:
viterbi_sequences = viterbi_decode(logits, trans_params, length)
tag_preds = dev_data.logits_indices_to_tags_seq(viterbi_sequences, length)
else:
tag_preds = dev_data.logits_indices_to_tags_seq(logits_indices, length)
tag_corrects = dev_data.logits_indices_to_tags_seq(output_data_indices, length)
dev_prec, dev_rec, dev_f1 = ChunkEval.compute_f1(tag_preds, tag_corrects)
print('dev precision, recall, f1(chunk): ', dev_prec, dev_rec, dev_f1)
chunk_f1 = dev_f1
# save best model
'''
m = chunk_f1 # slightly lower than token-based f1 for test
'''
m = token_f1
if m > maximum:
print('new best f1 score!')
maximum = m
save_path = saver.save(sess, config.checkpoint_dir + '/' + 'model_max.ckpt')
print('max model saved in file: %s' % save_path)
feed_dict={model.input_data_word_ids: test_data.sentence_word_ids,
model.input_data_wordchr_ids: test_data.sentence_wordchr_ids,
model.input_data_pos_ids: test_data.sentence_pos_ids,
model.input_data_etc: test_data.sentence_etc,
model.output_data: test_data.sentence_tag}
step, logits, logits_indices, trans_params, output_data_indices, length, test_loss, test_accuracy = \
sess.run([model.global_step, model.logits, model.logits_indices, model.trans_params, model.output_data_indices, model.length, model.loss, model.accuracy], feed_dict=feed_dict)
print('epoch: %d / %d, step: %d, test loss: %s, test accuracy: %s' % (e, config.epoch, step, test_loss, test_accuracy))
print('test precision, recall, f1(token): ')
TokenEval.compute_f1(config.class_size, logits, test_data.sentence_tag, length)
if config.use_crf:
viterbi_sequences = viterbi_decode(logits, trans_params, length)
tag_preds = test_data.logits_indices_to_tags_seq(viterbi_sequences, length)
else:
tag_preds = test_data.logits_indices_to_tags_seq(logits_indices, length)
tag_corrects = test_data.logits_indices_to_tags_seq(output_data_indices, length)
test_prec, test_rec, test_f1 = ChunkEval.compute_f1(tag_preds, tag_corrects)
print('test precision, recall, f1(chunk): ', test_prec, test_rec, test_f1)
# learning rate change
if e > intermid_epoch: learning_rate=learning_rate_final
def inference_bulk(config):
"""Inference for test file
"""
# Build input data
test_file = 'data/test.txt'
test_data = Input(test_file, config, build_output=True)
print('loading input data ... done')
# Create model
model = Model(config)
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
# Restore model
feed_dict = {}
if not config.use_elmo:
feed_dict = {model.wrd_embeddings_init: config.embvec.wrd_embeddings}
sess.run(tf.global_variables_initializer(), feed_dict=feed_dict)
saver = tf.train.Saver()
saver.restore(sess, config.restore)
print('model restored')
feed_dict = {
model.input_data_pos_ids: test_data.sentence_pos_ids,
model.input_data_etcs: test_data.sentence_etcs,
model.output_data: test_data.sentence_tags,
model.is_train: False,
model.sentence_length: test_data.max_sentence_length
}
if config.use_elmo:
feed_dict[
model.
elmo_input_data_wordchr_ids] = test_data.sentence_elmo_wordchr_ids
else:
feed_dict[model.input_data_word_ids] = test_data.sentence_word_ids
feed_dict[
model.input_data_wordchr_ids] = test_data.sentence_wordchr_ids
logits, trans_params, sentence_lengths = \
sess.run([model.logits, model.trans_params, model.sentence_lengths], \
feed_dict=feed_dict)
print('test precision, recall, f1(token): ')
TokenEval.compute_f1(config.class_size, logits, test_data.sentence_tags,
sentence_lengths)
if config.use_crf:
viterbi_sequences = viterbi_decode(logits, trans_params,
sentence_lengths)
tag_preds = test_data.logits_indices_to_tags_seq(
viterbi_sequences, sentence_lengths)
else:
logits_indices = np.argmax(logits, 2)
tag_preds = test_data.logits_indices_to_tags_seq(
logits_indices, sentence_lengths)
output_data_indices = np.argmax(test_data.sentence_tags, 2)
tag_corrects = test_data.logits_indices_to_tags_seq(
output_data_indices, sentence_lengths)
test_prec, test_rec, test_f1 = ChunkEval.compute_f1(
tag_preds, tag_corrects)
print('test precision, recall, f1(chunk): ', test_prec, test_rec, test_f1)
sess.close()