def write_to_tensorboard(self, writer, iterator_info):
global_step = iterator_info.batches_seen
writer.add_scalar('loss', self.batch_stats['loss'] / self.batch_length,
global_step)
writer.add_scalar(
'attention_entropy',
self.batch_stats['attention_entropy'] / self.batch_length,
global_step)
writer.add_scalar(
'traceback_attention_entropy',
self.batch_stats['traceback_attention_entropy'] /
self.batch_length, global_step)
writer.add_scalar('accuracy_sum',
self.batch_stats['accuracy_sum'] / self.batch_length,
global_step)
macro_p, macro_r, macro_f1 = precision_recall_f1(*list(self.counts),
reduce='macro')
micro_p, micro_r, micro_f1 = precision_recall_f1(*list(self.counts),
reduce='micro')
writer.add_scalar('macro_metrics/precision', macro_p, global_step)
writer.add_scalar('macro_metrics/recall', macro_r, global_step)
writer.add_scalar('macro_metrics/f1', macro_f1, global_step)
writer.add_scalar('micro_metrics/precision', micro_p, global_step)
writer.add_scalar('micro_metrics/recall', micro_r, global_step)
writer.add_scalar('micro_metrics/f1', micro_f1, global_step)
nq = self.outputs['labels'].size(1)
code_mask = (torch.arange(nq, device=self.outputs['labels'].device) <
self.outputs['num_codes'].unsqueeze(1))
labels = self.outputs['labels'][code_mask]
scores = self.outputs['scores'][code_mask]
writer.add_histogram('scores_all', scores, global_step)
writer.add_histogram('scores_0', scores[labels == 0], global_step)
writer.add_histogram('scores_1', scores[labels == 1], global_step)
def test(generator, param, flags):
with tf.Session() as sess:
# create model
model = NEF(param, tag_vocabulary)
model.load(sess, flags['checkpoint_dir'])
# print config
print('model was restore from: {}'.format(flags['checkpoint_dir']))
print(model.config)
# sess.run(tf.global_variables_initializer())
# start testing
start_testing = time.time()
gen = generator(test_data, param['batch_size'], 0)
gen_dev = generator(dev_data, param['batch_size'], 0)
print('[ Testing on {}: ... ]'.format(
join(flags['data_dir'], 'russian_test.txt')))
m_pred = []
m_true = []
for data, i in gen:
if not param['crf']:
pred_labels = model.inference_op(data, sess)
mpr = convert(pred_labels, i2t)
m_pred.extend(mpr)
y = refactor_data(data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
mtr = convert(y, i2t)
m_true.extend(mtr)
else:
raise ValueError('It is testing!!! No SRF!')
conllf1 = precision_recall_f1(m_true, m_pred)
print('[ Testing on {}: ... ]'.format(
join(flags['data_dir'], 'russian_dev.txt')))
dev_predictions = []
dev_true = []
for data, i in gen_dev:
pred_labels = model.inference_op(data, sess)
mdf = convert(pred_labels, i2t)
dev_predictions.extend(mdf)
y_dev = refactor_data(data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
mdt = convert(y_dev, i2t)
dev_true.extend(mdt)
conllf1 = precision_recall_f1(dev_true, dev_predictions)
print('[ End. Global Time: {} ]\n'.format(time.time() - start_testing))
tf.reset_default_graph()
return None
def __str__(self):
return ("loss: %f" + "\nattention_entropy: %f" +
"\ntraceback_attention_entropy: %f" + "\naccuracy: %f" +
"\nmacro_averaged - p: %f, r: %f, f1: %f" +
"\nmicro_averaged - p: %f, r: %f, f1: %f") % (
self.batch_stats['loss'] / self.batch_length,
self.batch_stats['attention_entropy'] / self.batch_length,
self.batch_stats['traceback_attention_entropy'] /
self.batch_length,
self.batch_stats['accuracy_sum'] / self.batch_length,
*precision_recall_f1(*list(self.counts), reduce='macro'),
*precision_recall_f1(*list(self.counts), reduce='micro'))
def train(generator, param, flags):
with tf.Session() as sess:
# create model
model = bi_LSTM(param, tag_vocabulary, i2t)
# print config
print(model.path)
sess.run(tf.global_variables_initializer())
# start learning
start_learning = time.time()
for e in range(flags['epochs']):
gen = generator(train_data, param['batch_size'], 0)
train_predictions = []
train_true = []
m_pred = []
m_true = []
start = time.time()
for data, i in gen:
pred_labels, losses = model.train_op(data, sess)
train_predictions.extend(pred_labels)
mpr = convert(pred_labels, i2t)
m_pred.extend(mpr)
y = refactor_data(data, tag_vocabulary, [param['batch_size'], param['maximum_L']])
train_true.extend(y)
mtr = convert(y, i2t)
m_true.extend(mtr)
# TODO fix it
if i % 20:
print('[ Epoch {0}; Loss: {1} ]'.format(e, losses))
result = dict()
result['accuracy'] = accuracy(train_true, train_predictions)
result['f1_nof1'], result['f1_nof2'] = f1s_binary(train_true, train_predictions)
print('\n{}'.format(result))
conllf1 = precision_recall_f1(m_true, m_pred)
if e % flags['checkpoint_freq'] == 0:
if not os.path.isdir(flags['checkpoint_dir']):
os.makedirs(flags['checkpoint_dir'])
model.save(sess, flags['checkpoint_dir'])
print('[ End. Global Time: {} ]\n'.format(time.time() - start_learning))
tf.reset_default_graph()
return None
def train(generator, param, flags):
with tf.Session() as sess:
# create model
model = NEF(param, tag_vocabulary, i2t)
# print config
print(model.path)
sess.run(tf.global_variables_initializer())
# start learning
start_learning = time.time()
for e in range(flags['epochs']):
gen = generator(train_data, param['batch_size'], 0)
gen_dev = generator(dev_data, param['batch_size'], 0)
train_predictions = []
train_true = []
m_pred = []
m_true = []
start = time.time()
for data, i in gen:
pred_labels, losses = model.train_op(data, sess)
train_predictions.extend(pred_labels)
mpr = convert(pred_labels, i2t)
m_pred.extend(mpr)
y = refactor_data(data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
train_true.extend(y)
mtr = convert(y, i2t)
m_true.extend(mtr)
# TODO fix it
if i % 20:
print('[ Epoch {0}; Loss: {1} ]'.format(e, losses))
result = dict()
result['accuracy'] = accuracy(train_true, train_predictions)
result['f1_nof1'], result['f1_nof2'] = f1s_binary(
train_true, train_predictions)
# print('[ Non official f1 (1): {} ]\n'.format(f1_nof1))
# print('[ Non official f1 (2): {} ]\n'.format(f1_nof2))
# print('[ Epoch {0} end; Time: {1} ]\n'.format(e+1, time.time() - start))
print('\n{}'.format(result))
conllf1 = precision_recall_f1(m_true, m_pred)
print('[ Validation on {}: ... ]'.format(
join(flags['data_dir'], 'russian_dev.txt')))
dev_predictions = []
dev_true = []
for data, i in gen_dev:
pred_labels = model.inference_op(data, sess)
dev_predictions.extend(pred_labels)
y_dev = refactor_data(
data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
dev_true.extend(y_dev)
acc = accuracy(dev_true, dev_predictions)
print('[accuracy = {}]\n'.format(acc))
f1_nof1, f1_nof2 = f1s_binary(dev_true, dev_predictions)
print('[ Non official f1 (1): {} ]\n'.format(f1_nof1))
print('[ Non official f1 (2): {} ]\n'.format(f1_nof2))
if e % flags['checkpoint_freq'] == 0:
if not os.path.isdir(flags['checkpoint_dir']):
os.makedirs(flags['checkpoint_dir'])
model.save(sess, flags['checkpoint_dir'])
print('[ End. Global Time: {} ]\n'.format(time.time() -
start_learning))
tf.reset_default_graph()
print('Start testing model from checkpoint: {} '.format(
flags['checkpoint_dir']))
with tf.Session() as sess:
# create model
model = NEF(param, tag_vocabulary, i2t)
# print config
print(model.path)
model.load(sess, flags['checkpoint_dir'])
print('[ model was restored ... ]\n'.format(flags['checkpoint_dir']))
sess.run(tf.global_variables_initializer())
# start testing
gen_test = generator(test_data, param['batch_size'])
test_predictions = []
test_true = []
for data in gen_test:
pred_labels = model.inference_op(data, sess)
test_predictions.extend(pred_labels)
y_test = refactor_data(data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
test_true.extend(y_test)
acc = accuracy(test_true, test_predictions)
print('[accuracy = {}]\n'.format(acc))
f1_nof1, f1_nof2 = f1s_binary(test_true, test_predictions)
print('[ Non official f1 (1): {} ]'.format(f1_nof1))
print('[ Non official f1 (2): {} ]'.format(f1_nof2))
print('[ End of Testing. ]')
tf.reset_default_graph()
return None
def __str__(self):
original_string = super(OutputBatchTest, self).__str__()
if self.summary_counts is not None:
original_string += "\nsummary - p: %f, r: %f, f1: %f" %\
precision_recall_f1(*list(self.summary_counts))
return original_string
y = refactor_data(data, tag_vocabulary, [parameters['batch_size'], parameters['maximum_L']])
print('Refactor input data: \n{}'.format(y))
acc = accuracy(y, pred_labels)
print('[accuracy = {}]\n'.format(acc))
f1_nof1, f1_nof2 = f1s_binary(y, pred_labels)
print('[ Non official f1 (1): {} ]\n'.format(f1_nof1))
print('[ Non official f1 (2): {} ]\n'.format(f1_nof2))
ypred = convert(pred_labels, i2t)
ytrue = convert(y, i2t)
# print('true list: \n{}'.format(ytrue))
# print('pred list: \n{}'.format(ypred))
result = precision_recall_f1(ytrue, ypred)
print(result)
tf.reset_default_graph()
# def doc_inference(adres, generator, param, flags, checkpoint):
# # TODO: move from there
# scorer = 'conlleval.txt'
#
# data = read_dataset(adres, param['maximum_L'], split=False)
#
# words, tag_vocabulary, word_counter = create_vocabulary(data)
#
# with tf.Session() as sess:
def train(generator, param, flags):
with tf.Session() as sess:
# create model
model = NEF(param, tag_vocabulary)
if flags['restore']:
print('[ Model is restoring from: {} ]'.format(flags['data_dir']))
model.load(sess, flags['checkpoint_dir'])
print('[ Model was restored ]')
else:
print('[ Model is initialize from scratch ]')
sess.run(tf.global_variables_initializer())
# print config
print(model.config)
# start learning
start_learning = time.time()
max_f1 = 0
for e in range(flags['epochs']):
gen = generator(train_data, param['batch_size'], 0)
train_predictions = []
train_true = []
m_pred = []
m_true = []
for data, i in gen:
if not param['crf']:
pred_labels, losses = model.train_op(data, sess)
train_predictions.extend(pred_labels)
mpr = convert(pred_labels, i2t)
m_pred.extend(mpr)
y = refactor_data(
data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
train_true.extend(y)
mtr = convert(y, i2t)
m_true.extend(mtr)
else:
losses = model.train_op(data, sess)
print('[ Epoch {0}; Loss: {1} ]'.format(e, losses))
if e % flags['checkpoint_freq'] == 0:
if not param['crf']:
conllf1 = precision_recall_f1(m_true, m_pred)
if conllf1['__total__']['f1'] > max_f1:
max_f1 = conllf1['__total__']['f1']
if not os.path.isdir(flags['checkpoint_dir']):
os.makedirs(flags['checkpoint_dir'])
model.save(sess, flags['checkpoint_dir'])
print('[ model was saved ]')
# validation on dev dataset
print('[ Validation on dev dataset ... ]')
dev_gen = generator(dev_data, param['batch_size'], 0)
dev_pred = []
dev_true = []
for data, i in dev_gen:
if not param['crf']:
pred_labels = model.inference_op(data, sess)
mpr = convert(pred_labels, i2t)
dev_pred.extend(mpr)
y = refactor_data(
data, tag_vocabulary,
[param['batch_size'], param['maximum_L']])
mtr = convert(y, i2t)
dev_true.extend(mtr)
dev_conllf1 = precision_recall_f1(dev_true, dev_pred)
print('[ End. Global Time: {} ]\n'.format(time.time() -
start_learning))
tf.reset_default_graph()
return None