def run_evaluate(self, test):
"""Evaluates performance on test set
Args:
test: dataset that yields tuple of (sentences, tags)
Returns:
metrics: (dict) metrics["acc"] = 98.4, ...
"""
accs = []
correct_preds, total_correct, total_preds = 0., 0., 0.
for words, labels in minibatches(test, self.config.batch_size):
labels_pred, sequence_lengths = self.predict_batch(words)
for lab, lab_pred, length in zip(labels, labels_pred,
sequence_lengths):
lab = lab[:length]
lab_pred = lab_pred[:length]
accs += [a==b for (a, b) in zip(lab, lab_pred)]
lab_chunks = set(get_chunks(lab, self.config.vocab_tags))
lab_pred_chunks = set(get_chunks(lab_pred,
self.config.vocab_tags))
correct_preds += len(lab_chunks & lab_pred_chunks)
total_preds += len(lab_pred_chunks)
total_correct += len(lab_chunks)
p = correct_preds / total_preds if correct_preds > 0 else 0
r = correct_preds / total_correct if correct_preds > 0 else 0
f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0
acc = np.mean(accs)
return {"acc": 100*acc, "f1": 100*f1}
def run_evaluate(self, test):
"""Evaluates performance on test set
Args:
test: dataset that yields tuple of (sentences, tags)
Returns:
metrics: (dict) metrics["acc"] = 98.4, ...
"""
accs = []
correct_preds, incorrect_preds, total_correct, total_preds = 0., 0., 0., 0.
for words, labels in minibatches(test, self.config.batch_size):
labels_pred, sequence_lengths = self.predict_batch(words)
for lab, lab_pred, length in zip(labels, labels_pred,
sequence_lengths):
lab = lab[:length]
lab_pred = lab_pred[:length]
boolean_arr = [a==b for (a, b) in zip(lab, lab_pred)]
accs += boolean_arr
correct_preds += np.sum(boolean_arr)
incorrect_preds += np.sum([a!=b for (a, b) in zip(lab, lab_pred)])
total_preds += len(lab_pred)
total_correct += len(lab)
print(lab==lab_pred)
p = correct_preds / total_preds if correct_preds > 0 else 0
r = correct_preds / total_correct if correct_preds > 0 else 0
print("CP", correct_preds, "IP", incorrect_preds, "TP", total_preds, "TC", total_correct)
f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0
acc = np.mean(accs)
return {"acc": 100*acc, "f1": 100*f1}
def run_predict(self, test):
with codecs.open(self.config.dir_output + "test_result.txt", mode="w", encoding="utf-8") as fp:
self.logger.info("Testing model over test set")
metrics = self.run_evaluate(test)
msg = " - ".join(["{} {:04.2f}".format(k, v)
for k, v in metrics.items()])
self.logger.info(msg)
fp.write("*" * 20 + "\n")
fp.write(msg + "\n")
fp.write("*" * 20 + "\n")
fp.write("\n")
for words, labels in minibatches(test, self.config.batch_size):
labels_pred, sequence_lengths = self.predict_batch(words)
for word, lab, lab_pred, length in list(zip(words, labels, labels_pred, sequence_lengths)):
if self.config.use_chars:
word = word[1][:length]
else:
word = word[:length]
lab = lab[:length]
lab_pred = lab_pred[:length]
decoded_word = [self.idx_to_word[idx] for idx in word]
decoded_lab = [self.idx_to_tag[idx] for idx in lab]
decoded_lab_pred = [self.idx_to_tag[idx] for idx in lab_pred]
word_lab = align_data({"input": decoded_word, "output": decoded_lab})
word_lab_pred = align_data({"input": decoded_word, "output": decoded_lab_pred})
fp.write("word: " + word_lab['input'] + "\n")
fp.write("true: " + word_lab['output'] + "\n")
fp.write("pred: " + word_lab_pred['output'] + "\n")
fp.write("\n")
def main():
# create datasets
train, dev = getDataSet('data/train/hairpin.txt', 'data/train/mrna.txt')
# train model
batch_size = 32
(inputs, labels) = minibatches(train, batch_size)
# inputs: (32, None, 4)
print(labels)
def testNewData(test, config, model):###method to visualise model predictions
from model.data_utils import minibatches
sentences = []
picos=[]
labs = []
labs_pred = []
for words, labels in minibatches(test, config.batch_size):
labels_pred, document_lengths = model.predict_batch(words)
for lList in labels:
for l in lList:
labs.append(config.tag_decode[l])#decode label
for abstract in words:
for sentence in abstract:
prose= ' '.join(config.vocab_decode[w] for w in sentence)#decode from embedding indexes and join back as sentence
sentences.append(prose)
for absTags in labels_pred:
for tag in absTags:
picos.append(config.tag_decode[tag]) #decode tag
truePreds=[]
falsePreds=[]
for i, pico in enumerate(picos):
if labs[i] == pico and re.search('P|I|O', pico):#a correctly predicted pico
truePreds.append('Predicted {} (True label is {}){}{}'.format(pico,labs[i],'-->' , sentences[i]))
elif labs[i] != pico and re.search('P|I|O', labs[i]): #an incorrectly predicted pico
falsePreds.append('Predicted {} (True label is {}){}{}'.format(pico,labs[i],'-->' , sentences[i]))
# maxi=100
# count=0
# print('-----------False--------------')
# for pred in falsePreds:
# if count<maxi:
# print(pred)
# count += 1
# count = 0
#
# count=0
# print('-----------TRUE--------------')
# for pred in truePreds:
# if count<maxi:
# print(pred)
# count += 1
#selection= falsePreds[200:]
for sel in falsePreds:
if re.search('Predicted I',sel):
print(sel)
def main():
'''
evaluate using saved models
'''
# create instance of config
config = Config()
# build model
model = NERModel(config)
model.build()
model.restore_session(config.dir_model)
# create dataset
test = CoNLLDataset(config.filename_test, config.processing_word,
config.processing_tag, config.max_iter)
print(type(config.vocab_words))
# eval uate and interact
#model.evaluate(test)
lab = []
seqs = []
for words, labels in minibatches(test, 1):
temp = []
temp2 = []
w = copy.deepcopy(words)
A = list(w[0])
labels_pred, sequence_lengths = model.predict_batch(words)
#fd, sequence_lengths = model.get_feed_dict(words, dropout=1.0)
for i, y in enumerate(labels_pred[0]):
x = A[0][i]
temp3 = []
for letter in x:
#print(letter)
temp3.append(model.idx_to_char[letter])
temp.append(model.idx_to_tag[y])
temp2.append(''.join(temp3))
#temp2.append(model.config.processing_word[x])
lab.append(temp)
seqs.append(temp2)
print(lab[0:3])
print(seqs[0:3])
#interactive_shell(model)
name = 'pred_give_ingredient_dev.txt'
data = load(config.filename_test)
print(data[0:3])
write_pred(data, lab, name)
def classScores(model, test):
"""Evaluates performance on test set
Args:
test: dataset that yields tuple of (sentences, tags)
Returns:
metrics: (dict) metrics["f1_<label>"] = 98.4, ...
"""
preds = []
labels = []
for words, labs in minibatches(test, model.config.batch_size):
labels_pred, sequence_lengths = model.predict_batch(words)
for lab, lab_pred, length in zip(labs, labels_pred, sequence_lengths):
lab_pred = lab_pred[:length]
lab = lab[:length]
preds.append(lab_pred)
labels.append(lab)
return preds, labels
def run_epoch(self, train, dev, epoch):
"""Performs one complete pass over the train set and evaluate on dev
Args:
train: dataset that yields tuple of sentences, tags
dev: dataset
epoch: (int) index of the current epoch
Returns:
f1: (python float), score to select model on, higher is better
"""
# progbar stuff for logging
batch_size = self.config.batch_size
nbatches = (len(train) + batch_size - 1) // batch_size
prog = Progbar(target=nbatches)
# iterate over dataset
for i, (words, labels) in enumerate(minibatches(train, batch_size)):
fd, _ = self.get_feed_dict(words, labels, self.config.lr,
self.config.dropout)
_, train_loss, summary = self.sess.run(
[self.train_op, self.loss, self.merged], feed_dict=fd)
prog.update(i + 1, [("train loss", train_loss)])
# tensorboard
if i % 10 == 0:
self.file_writer.add_summary(summary, epoch*nbatches + i)
metrics = self.run_evaluate(dev)
msg = " - ".join(["{} {:04.2f}".format(k, v)
for k, v in metrics.items()])
self.logger.info(msg)
return metrics["f1"]
def main():
input = [["emersoN", "lAke", "aNd", "palmer"],
["i", "haVe", "a", "343yaCht123", "m%an", "2543"]]
sentences_padded, _ = pad_sequences(input, '')
sentences = tf.constant(sentences_padded)
lowercase_sentences = lowercase(sentences)
table = lookup.index_table_from_tensor(mapping=tf.constant(['']),
default_value=1)
sequence_lengths = tf.reduce_sum(table.lookup(sentences), 1)
word_table = lookup.index_table_from_file(vocabulary_file="data/words.txt",
num_oov_buckets=1)
char_table = lookup.index_table_from_file(vocabulary_file="data/chars.txt",
default_value=-1)
sentences_shape = tf.shape(sentences, out_type=tf.int64)
# We need to remove chars not in vocab
removed_char_sentences = remove_unknown_chars(sentences, char_table)
split_words = tf.string_split(tf.reshape(removed_char_sentences, [-1]),
delimiter="")
dense_split_words = tf.sparse_tensor_to_dense(split_words,
default_value='')
max_word_len = tf.gather_nd(split_words.dense_shape, [1])
chars_shape = tf.concat([sentences_shape, [max_word_len]], 0)
chars = tf.reshape(dense_split_words, chars_shape)
word_lengths = tf.reduce_sum(table.lookup(chars), 2)
word_ids = word_table.lookup(sentences)
char_ids = char_table.lookup(chars)
word_mask = tf.sequence_mask(sequence_lengths)
word_ids = tf.where(word_mask, word_ids, tf.zeros_like(word_ids))
char_mask = tf.sequence_mask(word_lengths)
char_ids = tf.where(char_mask, char_ids, tf.zeros_like(char_ids))
config = Config()
# build model
model = NERModel(config)
model.build()
dev = CoNLLDataset(config.filename_dev, max_iter=config.max_iter)
train = CoNLLDataset(config.filename_train, max_iter=config.max_iter)
batch_size = model.config.batch_size
# iterate over dataset
for i, (words, labels) in enumerate(minibatches(train, batch_size)):
print "Start"
fd, _ = model.get_feed_dict(words, labels, model.config.lr,
model.config.dropout)
_, train_loss = model.sess.run([model.train_op, model.loss],
feed_dict=fd)
print "train loss", train_loss
metrics = model.run_evaluate(dev)
msg = " - ".join(
["{} {:04.2f}".format(k, v) for k, v in metrics.items()])
print msg
from model.data_utils import CoNLLDataset, minibatches
from model.config import Config
if __name__ == '__main__':
config = Config()
train = CoNLLDataset(config.filename_train, config.processing_word,
config.processing_tag, config.max_iter)
batch_size = config.batch_size
for i, (words, labels, masks) in enumerate(minibatches(train, batch_size)):
for i in words:
for j in i:
print(j)
break