def calc_f1_batch(self, decoded_data, target_data): """ update statics for f1 score args: decoded_data (batch_size, seq_len): prediction sequence target_data (batch_size, seq_len): ground-truth """ batch_decoded = torch.unbind(decoded_data, 1) batch_targets = torch.unbind(target_data, 0) for decoded, target in zip(batch_decoded, batch_targets): gold = self.packer.convert_for_eval(target) # remove padding length = utils.find_length_from_labels(gold, self.l_map) gold = gold[:length] best_path = decoded[:length] ## filter out removed label best_path_filted, gold_filted = self.label_filter( best_path.numpy(), gold.numpy()) correct_labels_i, total_labels_i, gold_count_i, guess_count_i, overlap_count_i = self.eval_instance( best_path_filted, gold_filted) self.correct_labels += correct_labels_i self.total_labels += total_labels_i self.gold_count += gold_count_i self.guess_count += guess_count_i self.overlap_count += overlap_count_i
def check_output(self, ner_model, dataset_loader, out, f_map, emb, word_to_id, gpu, knowledge_dict, no_dict=False): ner_model.eval() self.reset() f = open('model_output.txt', 'w') for f_f, f_p, b_f, b_p, w_f, tg, mask_v, len_v in itertools.chain.from_iterable( dataset_loader): mask_v = mask_v.bool() f_f, f_p, b_f, b_p, w_f, _, mask_v = self.packer.repack_vb( f_f, f_p, b_f, b_p, w_f, tg, mask_v, len_v) w_f_word = utils.reconstruct_word_input(w_f, f_map, emb, word_to_id, gpu) prior_prob = utils.generate_prior_prob(self.r_c_map, self.l_map, f_f, knowledge_dict) scores = ner_model(f_f, f_p, b_f, b_p, w_f, w_f_word, prior_prob) decoded = self.decoder.decode(scores.data, mask_v.data, prior_prob, no_dict) self.eval_b(decoded, tg) batch_decoded = torch.unbind(decoded, 1) batch_targets = torch.unbind(tg, 0) batch_f_f = torch.unbind(f_f, 1) for decoded, target, character in zip(batch_decoded, batch_targets, batch_f_f): gold = self.packer.convert_for_eval(target) # remove padding length = utils.find_length_from_labels(gold, self.l_map) gold = gold[:length].numpy() best_path = decoded[:length].numpy() character_filted = [] for c in character.cpu().numpy(): if c != 42: character_filted.append(c) char = character_filted[:length] for i in range(len(gold)): f.write(self.r_c_map[char[i]] + ' ' + self.r_l_map[gold[i]] + ' ' + self.r_l_map[best_path[i]]) f.write('\n') f.write('\n') f.close() return self.calc_s()
def calc_predict(self, ner_model, dataset_loader, test_features, file_out, file_out_2, f_map): """ calculate score for pre-selected metrics args: ner_model: LSTM-CRF model dataset_loader: loader class for test set """ ner_model.eval() self.reset() idx2label = {v: k for k, v in self.l_map.items()} idx2word = {v: k for k, v in f_map.items()} for i in range(len(dataset_loader[0])): fea_v, tg_v, mask_v = self.packer.repack_vb( np.asarray(dataset_loader[0][i]), np.asarray(dataset_loader[1][i]), np.asarray(dataset_loader[2][i])) ner_model.zero_grad() scores, hidden = ner_model(fea_v, dataset_loader[3][i]) decoded = self.decoder.decode(scores.data, mask_v.data) gold = [d % len(self.l_map) for d in dataset_loader[1][i]] # words = [idx2word[w] for w in dataset_loader[0][i]] length = utils.find_length_from_labels(gold, self.l_map) gold = gold[:length] words = test_features[i][:length] best_path = decoded.squeeze(1).tolist()[:length] gold = [idx2label[g] for g in gold] best_path = [idx2label[g] for g in best_path] for i in range(length): file_out.write("%s %s\n" % (words[i], best_path[i])) file_out.write("\n") sent = '' pos = None word = '' for i in range(length): if best_path[i].startswith('B'): if pos != None: sent += word + '_' + pos + ' ' word = '' pos = None word += words[i] pos = best_path[i].split('-')[1] else: assert pos != None word += words[i] if len(word) > 0: sent += word + '_' + pos + ' ' file_out_2.write("%s\n" % (sent))
def calc_f1_batch(self, target_data, decoded_data_crfs, decode_data_scrfs, decode_data_jnts): """ update statics for f1 score args: decoded_data (batch_size, seq_len): prediction sequence target_data (batch_size, seq_len): ground-truth """ for target, decoded_data_crf, decode_data_scrf, decode_data_jnt in zip( target_data, decoded_data_crfs, decode_data_scrfs, decode_data_jnts): length = utils.find_length_from_labels(target, self.l_map) gold = target[:length] decoded_data_crf = decoded_data_crf[:length] decode_data_scrf = decode_data_scrf[:length] decode_data_jnt = decode_data_jnt[:length] correct_labels_i, total_labels_i, gold_count_i, guess_count_i, overlap_count_i = self.eval_instance( decoded_data_crf, gold) self.correct_labels_crf += correct_labels_i self.total_labels_crf += total_labels_i self.gold_count_crf += gold_count_i self.guess_count_crf += guess_count_i self.overlap_count_crf += overlap_count_i correct_labels_i, total_labels_i, gold_count_i, guess_count_i, overlap_count_i = self.eval_instance( decode_data_scrf, gold) self.correct_labels_scrf += correct_labels_i self.total_labels_scrf += total_labels_i self.gold_count_scrf += gold_count_i self.guess_count_scrf += guess_count_i self.overlap_count_scrf += overlap_count_i correct_labels_i, total_labels_i, gold_count_i, guess_count_i, overlap_count_i = self.eval_instance( decode_data_jnt, gold) self.correct_labels_jnt += correct_labels_i self.total_labels_jnt += total_labels_i self.gold_count_jnt += gold_count_i self.guess_count_jnt += guess_count_i self.overlap_count_jnt += overlap_count_i
def calc_acc_batch(self, decoded_data, target_data): """ update statics for accuracy args: decoded_data (batch_size, seq_len): prediction sequence target_data (batch_size, seq_len): ground-truth """ batch_decoded = torch.unbind(decoded_data, 1) batch_targets = torch.unbind(target_data, 0) for decoded, target in zip(batch_decoded, batch_targets): gold = self.packer.convert_for_eval(target) # remove padding length = utils.find_length_from_labels(gold, self.l_map) gold = gold[:length].numpy() best_path = decoded[:length].numpy() self.total_labels += length self.correct_labels += np.sum(np.equal(best_path, gold))
def calc_acc_batch(self, decoded_data, target_data): """ update statics for accuracy args: decoded_data (batch_size, seq_len): prediction sequence target_data (batch_size, seq_len): ground-truth """ batch_decoded = torch.unbind(decoded_data, 1) batch_targets = torch.unbind(target_data, 0) for decoded, target in zip(batch_decoded, batch_targets): gold = self.packer.convert_for_eval(target) # remove padding length = utils.find_length_from_labels(gold, self.l_map) gold = gold[:length].numpy() best_path = decoded[:length].numpy() self.total_labels += length self.correct_labels += np.sum(np.equal(best_path, gold))
def calc_f1_batch(self, decoded_data, target_data): """ update statics for f1 score args: decoded_data (batch_size, seq_len): prediction sequence target_data (batch_size, seq_len): ground-truth """ batch_decoded = torch.unbind(decoded_data, 1) batch_targets = torch.unbind(target_data, 0) for decoded, target in zip(batch_decoded, batch_targets): gold = self.packer.convert_for_eval(target) # remove padding length = utils.find_length_from_labels(gold, self.l_map) gold = gold[:length] best_path = decoded[:length] correct_labels_i, total_labels_i, gold_count_i, guess_count_i, overlap_count_i = self.eval_instance(best_path.numpy(), gold.numpy()) self.correct_labels += correct_labels_i self.total_labels += total_labels_i self.gold_count += gold_count_i self.guess_count += guess_count_i self.overlap_count += overlap_count_i