def run_evaluate(self, test): """Evaluates performance on test set Args: test: dataset that yields tuple of (sentences, tags) Returns: metrics: (dict) metrics["acc"] = 98.4, ... """ accs = [] correct_preds, total_correct, total_preds = 0., 0., 0. for words, labels in minibatches(test, self.config.batch_size): labels_pred, sequence_lengths = self.predict_batch(words) for lab, lab_pred, length in zip(labels, labels_pred, sequence_lengths): lab = lab[:length] lab_pred = lab_pred[:length] accs += [a==b for (a, b) in zip(lab, lab_pred)] lab_chunks = set(get_chunks(lab, self.config.vocab_tags)) lab_pred_chunks = set(get_chunks(lab_pred, self.config.vocab_tags)) correct_preds += len(lab_chunks & lab_pred_chunks) total_preds += len(lab_pred_chunks) total_correct += len(lab_chunks) p = correct_preds / total_preds if correct_preds > 0 else 0 r = correct_preds / total_correct if correct_preds > 0 else 0 f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0 acc = np.mean(accs) return {"acc": 100*acc, "f1": 100*f1}
def run_evaluate(self, test): """Evaluates performance on test set Args: test: dataset that yields tuple of (sentences, tags) Returns: metrics: (dict) metrics["acc"] = 98.4, ... """ accs = [] correct_preds, incorrect_preds, total_correct, total_preds = 0., 0., 0., 0. for words, labels in minibatches(test, self.config.batch_size): labels_pred, sequence_lengths = self.predict_batch(words) for lab, lab_pred, length in zip(labels, labels_pred, sequence_lengths): lab = lab[:length] lab_pred = lab_pred[:length] boolean_arr = [a==b for (a, b) in zip(lab, lab_pred)] accs += boolean_arr correct_preds += np.sum(boolean_arr) incorrect_preds += np.sum([a!=b for (a, b) in zip(lab, lab_pred)]) total_preds += len(lab_pred) total_correct += len(lab) print(lab==lab_pred) p = correct_preds / total_preds if correct_preds > 0 else 0 r = correct_preds / total_correct if correct_preds > 0 else 0 print("CP", correct_preds, "IP", incorrect_preds, "TP", total_preds, "TC", total_correct) f1 = 2 * p * r / (p + r) if correct_preds > 0 else 0 acc = np.mean(accs) return {"acc": 100*acc, "f1": 100*f1}
def run_predict(self, test): with codecs.open(self.config.dir_output + "test_result.txt", mode="w", encoding="utf-8") as fp: self.logger.info("Testing model over test set") metrics = self.run_evaluate(test) msg = " - ".join(["{} {:04.2f}".format(k, v) for k, v in metrics.items()]) self.logger.info(msg) fp.write("*" * 20 + "\n") fp.write(msg + "\n") fp.write("*" * 20 + "\n") fp.write("\n") for words, labels in minibatches(test, self.config.batch_size): labels_pred, sequence_lengths = self.predict_batch(words) for word, lab, lab_pred, length in list(zip(words, labels, labels_pred, sequence_lengths)): if self.config.use_chars: word = word[1][:length] else: word = word[:length] lab = lab[:length] lab_pred = lab_pred[:length] decoded_word = [self.idx_to_word[idx] for idx in word] decoded_lab = [self.idx_to_tag[idx] for idx in lab] decoded_lab_pred = [self.idx_to_tag[idx] for idx in lab_pred] word_lab = align_data({"input": decoded_word, "output": decoded_lab}) word_lab_pred = align_data({"input": decoded_word, "output": decoded_lab_pred}) fp.write("word: " + word_lab['input'] + "\n") fp.write("true: " + word_lab['output'] + "\n") fp.write("pred: " + word_lab_pred['output'] + "\n") fp.write("\n")
def main(): # create datasets train, dev = getDataSet('data/train/hairpin.txt', 'data/train/mrna.txt') # train model batch_size = 32 (inputs, labels) = minibatches(train, batch_size) # inputs: (32, None, 4) print(labels)
def testNewData(test, config, model):###method to visualise model predictions from model.data_utils import minibatches sentences = [] picos=[] labs = [] labs_pred = [] for words, labels in minibatches(test, config.batch_size): labels_pred, document_lengths = model.predict_batch(words) for lList in labels: for l in lList: labs.append(config.tag_decode[l])#decode label for abstract in words: for sentence in abstract: prose= ' '.join(config.vocab_decode[w] for w in sentence)#decode from embedding indexes and join back as sentence sentences.append(prose) for absTags in labels_pred: for tag in absTags: picos.append(config.tag_decode[tag]) #decode tag truePreds=[] falsePreds=[] for i, pico in enumerate(picos): if labs[i] == pico and re.search('P|I|O', pico):#a correctly predicted pico truePreds.append('Predicted {} (True label is {}){}{}'.format(pico,labs[i],'-->' , sentences[i])) elif labs[i] != pico and re.search('P|I|O', labs[i]): #an incorrectly predicted pico falsePreds.append('Predicted {} (True label is {}){}{}'.format(pico,labs[i],'-->' , sentences[i])) # maxi=100 # count=0 # print('-----------False--------------') # for pred in falsePreds: # if count<maxi: # print(pred) # count += 1 # count = 0 # # count=0 # print('-----------TRUE--------------') # for pred in truePreds: # if count<maxi: # print(pred) # count += 1 #selection= falsePreds[200:] for sel in falsePreds: if re.search('Predicted I',sel): print(sel)
def main(): ''' evaluate using saved models ''' # create instance of config config = Config() # build model model = NERModel(config) model.build() model.restore_session(config.dir_model) # create dataset test = CoNLLDataset(config.filename_test, config.processing_word, config.processing_tag, config.max_iter) print(type(config.vocab_words)) # eval uate and interact #model.evaluate(test) lab = [] seqs = [] for words, labels in minibatches(test, 1): temp = [] temp2 = [] w = copy.deepcopy(words) A = list(w[0]) labels_pred, sequence_lengths = model.predict_batch(words) #fd, sequence_lengths = model.get_feed_dict(words, dropout=1.0) for i, y in enumerate(labels_pred[0]): x = A[0][i] temp3 = [] for letter in x: #print(letter) temp3.append(model.idx_to_char[letter]) temp.append(model.idx_to_tag[y]) temp2.append(''.join(temp3)) #temp2.append(model.config.processing_word[x]) lab.append(temp) seqs.append(temp2) print(lab[0:3]) print(seqs[0:3]) #interactive_shell(model) name = 'pred_give_ingredient_dev.txt' data = load(config.filename_test) print(data[0:3]) write_pred(data, lab, name)
def classScores(model, test): """Evaluates performance on test set Args: test: dataset that yields tuple of (sentences, tags) Returns: metrics: (dict) metrics["f1_<label>"] = 98.4, ... """ preds = [] labels = [] for words, labs in minibatches(test, model.config.batch_size): labels_pred, sequence_lengths = model.predict_batch(words) for lab, lab_pred, length in zip(labs, labels_pred, sequence_lengths): lab_pred = lab_pred[:length] lab = lab[:length] preds.append(lab_pred) labels.append(lab) return preds, labels
def run_epoch(self, train, dev, epoch): """Performs one complete pass over the train set and evaluate on dev Args: train: dataset that yields tuple of sentences, tags dev: dataset epoch: (int) index of the current epoch Returns: f1: (python float), score to select model on, higher is better """ # progbar stuff for logging batch_size = self.config.batch_size nbatches = (len(train) + batch_size - 1) // batch_size prog = Progbar(target=nbatches) # iterate over dataset for i, (words, labels) in enumerate(minibatches(train, batch_size)): fd, _ = self.get_feed_dict(words, labels, self.config.lr, self.config.dropout) _, train_loss, summary = self.sess.run( [self.train_op, self.loss, self.merged], feed_dict=fd) prog.update(i + 1, [("train loss", train_loss)]) # tensorboard if i % 10 == 0: self.file_writer.add_summary(summary, epoch*nbatches + i) metrics = self.run_evaluate(dev) msg = " - ".join(["{} {:04.2f}".format(k, v) for k, v in metrics.items()]) self.logger.info(msg) return metrics["f1"]
def main(): input = [["emersoN", "lAke", "aNd", "palmer"], ["i", "haVe", "a", "343yaCht123", "m%an", "2543"]] sentences_padded, _ = pad_sequences(input, '') sentences = tf.constant(sentences_padded) lowercase_sentences = lowercase(sentences) table = lookup.index_table_from_tensor(mapping=tf.constant(['']), default_value=1) sequence_lengths = tf.reduce_sum(table.lookup(sentences), 1) word_table = lookup.index_table_from_file(vocabulary_file="data/words.txt", num_oov_buckets=1) char_table = lookup.index_table_from_file(vocabulary_file="data/chars.txt", default_value=-1) sentences_shape = tf.shape(sentences, out_type=tf.int64) # We need to remove chars not in vocab removed_char_sentences = remove_unknown_chars(sentences, char_table) split_words = tf.string_split(tf.reshape(removed_char_sentences, [-1]), delimiter="") dense_split_words = tf.sparse_tensor_to_dense(split_words, default_value='') max_word_len = tf.gather_nd(split_words.dense_shape, [1]) chars_shape = tf.concat([sentences_shape, [max_word_len]], 0) chars = tf.reshape(dense_split_words, chars_shape) word_lengths = tf.reduce_sum(table.lookup(chars), 2) word_ids = word_table.lookup(sentences) char_ids = char_table.lookup(chars) word_mask = tf.sequence_mask(sequence_lengths) word_ids = tf.where(word_mask, word_ids, tf.zeros_like(word_ids)) char_mask = tf.sequence_mask(word_lengths) char_ids = tf.where(char_mask, char_ids, tf.zeros_like(char_ids)) config = Config() # build model model = NERModel(config) model.build() dev = CoNLLDataset(config.filename_dev, max_iter=config.max_iter) train = CoNLLDataset(config.filename_train, max_iter=config.max_iter) batch_size = model.config.batch_size # iterate over dataset for i, (words, labels) in enumerate(minibatches(train, batch_size)): print "Start" fd, _ = model.get_feed_dict(words, labels, model.config.lr, model.config.dropout) _, train_loss = model.sess.run([model.train_op, model.loss], feed_dict=fd) print "train loss", train_loss metrics = model.run_evaluate(dev) msg = " - ".join( ["{} {:04.2f}".format(k, v) for k, v in metrics.items()]) print msg
from model.data_utils import CoNLLDataset, minibatches from model.config import Config if __name__ == '__main__': config = Config() train = CoNLLDataset(config.filename_train, config.processing_word, config.processing_tag, config.max_iter) batch_size = config.batch_size for i, (words, labels, masks) in enumerate(minibatches(train, batch_size)): for i in words: for j in i: print(j) break