def inference(my_model, valid_data, entity2id, cfg, log_info=False): # Evaluation my_model.eval() eval_loss, eval_acc, eval_max_acc = [], [], [] id2entity = {idx: entity for entity, idx in entity2id.items()} valid_data.reset_batches(is_sequential=True) test_batch_size = 20 if log_info: f_pred = open(cfg['pred_file'], 'w') for iteration in tqdm(range(valid_data.num_data // test_batch_size)): batch = valid_data.get_batch(iteration, test_batch_size, fact_dropout=0.0) loss, pred, pred_dist = my_model(batch) pred = pred.data.cpu().numpy() acc, max_acc = cal_accuracy(pred, batch[-1]) if log_info: output_pred_dist(pred_dist, batch[-1], id2entity, iteration * test_batch_size, valid_data, f_pred) eval_loss.append(float(loss.data)) eval_acc.append(acc) eval_max_acc.append(max_acc) print('avg_loss', sum(eval_loss) / len(eval_loss)) print('max_acc', sum(eval_max_acc) / len(eval_max_acc)) print('avg_acc', sum(eval_acc) / len(eval_acc)) return sum(eval_acc) / len(eval_acc)
def test(model, test_data, eps): model.eval() batcher = test_data.batcher() id2entity = test_data.id2entity f1s, hits = [], [] q_to_metrics = {} questions = [] pred_answers = [] for feed in batcher: _, pred, pred_dist = model(feed) acc, max_acc = cal_accuracy(pred, feed['answers'].cpu().numpy()) batch_size = pred_dist.size(0) batch_answers = feed['answers_'] questions += feed['questions_'] batch_candidates = feed['candidate_entities'] pad_ent_id = len(id2entity) for batch_id in range(batch_size): answers = batch_answers[batch_id] candidates = batch_candidates[batch_id,:].tolist() probs = pred_dist[batch_id, :].tolist() candidate2prob = {} for c, p in zip(candidates, probs): if c == pad_ent_id: continue else: candidate2prob[c] = p f1, hit = f1_and_hits(answers, candidate2prob, eps) best_ans = get_best_ans(candidate2prob) best_ans = id2entity.get(best_ans, '') pred_answers.append(best_ans) f1s.append(f1) hits.append(hit) print('evaluation.......') print('how many eval samples......', len(f1s)) print('avg_f1', np.mean(f1s)) print('avg_hits', np.mean(hits)) model.train() return np.mean(f1s), np.mean(hits)
# gap_root_average += gap_root root_tags_one_hot = one_hot_encode(root_tags_valid, TAG_NUM) root_eval_metrics.accumulate( confidence_root, root_tags_one_hot, [0 for i in range(confidence_root.shape[0])]) video_id_all.extend(video_id_valid) predict_root_all.append(confidence_root) tags_root_all.append(root_tags_valid) if i == 0: print(confidence_root[0, :]) loss_root_average += loss_root tags_root_all, predict_root_argmax, accuracy_root = \ cal_accuracy(predict_root_all, tags_root_all) top_2_acc = top_n_accuracy( np.concatenate(predict_root_all, axis=0), tags_root_all, 2) top_3_acc = top_n_accuracy( np.concatenate(predict_root_all, axis=0), tags_root_all, 3) root_matric_result = root_eval_metrics.get() print('tags root:{}'.format(tags_root_all[:32])) print('predict root:{}'.format(predict_root_argmax[:32])) if step >= 3000 and step % 1000 == 0: cm = confusion_matrix(tags_root_all, predict_root_argmax) str_cm = ['[{}]'.format(','.join(map(str, row))) for row in cm] print(str_cm) str_error = '' for i in range(len(video_id_all)):
def train(cfg): print("training ...") # prepare data entity2id = load_dict(cfg['data_folder'] + cfg['entity2id']) word2id = load_dict(cfg['data_folder'] + cfg['word2id']) relation2id = load_dict(cfg['data_folder'] + cfg['relation2id']) train_documents = load_documents(cfg['data_folder'] + cfg['train_documents']) train_document_entity_indices, train_document_texts = index_document_entities( train_documents, word2id, entity2id, cfg['max_document_word']) train_data = DataLoader(cfg['data_folder'] + cfg['train_data'], train_documents, train_document_entity_indices, train_document_texts, word2id, relation2id, entity2id, cfg['max_query_word'], cfg['max_document_word'], cfg['use_kb'], cfg['use_doc'], cfg['use_inverse_relation']) if cfg['dev_documents'] != cfg['train_documents']: valid_documents = load_documents(cfg['data_folder'] + cfg['dev_documents']) valid_document_entity_indices, valid_document_texts = index_document_entities( valid_documents, word2id, entity2id, cfg['max_document_word']) else: valid_documents = train_documents valid_document_entity_indices, valid_document_texts = train_document_entity_indices, train_document_texts valid_data = DataLoader(cfg['data_folder'] + cfg['dev_data'], valid_documents, valid_document_entity_indices, valid_document_texts, word2id, relation2id, entity2id, cfg['max_query_word'], cfg['max_document_word'], cfg['use_kb'], cfg['use_doc'], cfg['use_inverse_relation']) if cfg['test_documents'] != cfg['dev_documents']: test_documents = load_documents(cfg['data_folder'] + cfg['test_documents']) test_document_entity_indices, test_document_texts = index_document_entities( test_documents, word2id, entity2id, cfg['max_document_word']) else: test_documents = valid_documents test_document_entity_indices, test_document_texts = valid_document_entity_indices, valid_document_texts test_data = DataLoader(cfg['data_folder'] + cfg['test_data'], test_documents, test_document_entity_indices, test_document_texts, word2id, relation2id, entity2id, cfg['max_query_word'], cfg['max_document_word'], cfg['use_kb'], cfg['use_doc'], cfg['use_inverse_relation']) # create model & set parameters my_model = get_model(cfg, train_data.num_kb_relation, len(entity2id), len(word2id)).to(device) trainable_parameters = [ p for p in my_model.parameters() if p.requires_grad ] optimizer = torch.optim.Adam(trainable_parameters, lr=cfg['learning_rate']) best_dev_acc = 0.0 for epoch in range(cfg['num_epoch']): try: print('epoch', epoch) train_data.reset_batches(is_sequential=cfg['is_debug']) # Train my_model.train() train_loss, train_acc, train_max_acc = [], [], [] for iteration in tqdm( range(train_data.num_data // cfg['batch_size'])): batch = train_data.get_batch(iteration, cfg['batch_size'], cfg['fact_dropout']) loss, pred, _ = my_model(batch) pred = pred.data.cpu().numpy() acc, max_acc = cal_accuracy(pred, batch[-1]) train_loss.append(float(loss.data)) train_acc.append(acc) train_max_acc.append(max_acc) # back propogate my_model.zero_grad() optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(my_model.parameters(), cfg['gradient_clip']) optimizer.step() print('avg_training_loss', sum(train_loss) / len(train_loss)) print('max_training_acc', sum(train_max_acc) / len(train_max_acc)) print('avg_training_acc', sum(train_acc) / len(train_acc)) print("validating ...") eval_acc = inference(my_model, valid_data, entity2id, cfg) if eval_acc > best_dev_acc and cfg['to_save_model']: print("saving model to", cfg['save_model_file']) torch.save(my_model.state_dict(), cfg['save_model_file']) best_dev_acc = eval_acc except KeyboardInterrupt: break # Test set evaluation print("evaluating on test") print('loading model from ...', cfg['save_model_file']) my_model.load_state_dict(torch.load(cfg['save_model_file'])) test_acc = inference(my_model, test_data, entity2id, cfg, log_info=True) return test_acc
def train(cfg): print("training ...") # prepare data # entities.txt, vocab.txt,relations.txt等文件 """ entity2id是一个字典,表示每个entity对应的id word2id也是一个字典,表示每个word对应的id relation2id也是,表示relation,比如has_tags这些谓词 以上三者数据量较小 train_documents 文件较大。是一个list,每个元素一个dict,包含document,title,tokens属性。 document属性下,有text, entities属性。text就是文本,entities是一个列表。表示text中每个entity以及其kb_id。 title下与document类似,只不过是title文本 token中应该是document和title的entity列表。 index_document_entities应该是 """ entity2id = load_dict(cfg['data_folder'] + cfg['entity2id']) word2id = load_dict(cfg['data_folder'] + cfg['word2id']) relation2id = load_dict(cfg['data_folder'] + cfg['relation2id']) # train_document.json train_documents = load_documents(cfg['data_folder'] + cfg['train_documents']) train_document_entity_indices, train_document_texts = index_document_entities( train_documents, word2id, entity2id, cfg['max_document_word']) train_data = DataLoader(cfg['data_folder'] + cfg['train_data'], train_documents, train_document_entity_indices, train_document_texts, word2id, relation2id, entity2id, cfg['max_query_word'], cfg['max_document_word'], cfg['use_kb'], cfg['use_doc'], cfg['use_inverse_relation']) if cfg['dev_documents'] != cfg['train_documents']: valid_documents = load_documents(cfg['data_folder'] + cfg['dev_documents']) valid_document_entity_indices, valid_document_texts = index_document_entities( valid_documents, word2id, entity2id, cfg['max_document_word']) else: valid_documents = train_documents valid_document_entity_indices, valid_document_texts = train_document_entity_indices, train_document_texts valid_data = DataLoader(cfg['data_folder'] + cfg['dev_data'], valid_documents, valid_document_entity_indices, valid_document_texts, word2id, relation2id, entity2id, cfg['max_query_word'], cfg['max_document_word'], cfg['use_kb'], cfg['use_doc'], cfg['use_inverse_relation']) if cfg['test_documents'] != cfg['dev_documents']: test_documents = load_documents(cfg['data_folder'] + cfg['test_documents']) test_document_entity_indices, test_document_texts = index_document_entities( test_documents, word2id, entity2id, cfg['max_document_word']) else: test_documents = valid_documents test_document_entity_indices, test_document_texts = valid_document_entity_indices, valid_document_texts test_data = DataLoader(cfg['data_folder'] + cfg['test_data'], test_documents, test_document_entity_indices, test_document_texts, word2id, relation2id, entity2id, cfg['max_query_word'], cfg['max_document_word'], cfg['use_kb'], cfg['use_doc'], cfg['use_inverse_relation']) # create model & set parameters my_model = get_model(cfg, train_data.num_kb_relation, len(entity2id), len(word2id), "train") trainable_parameters = [ p for p in my_model.parameters() if p.requires_grad ] optimizer = torch.optim.Adam(trainable_parameters, lr=cfg['learning_rate']) best_dev_acc = 0.0 for epoch in range(cfg['num_epoch']): try: print('epoch', epoch) train_data.reset_batches(is_sequential=cfg['is_debug']) # Train my_model.train() train_loss, train_acc, train_max_acc = [], [], [] for iteration in tqdm( range(train_data.num_data // cfg['batch_size'])): batch = train_data.get_batch(iteration, cfg['batch_size'], cfg['fact_dropout']) loss, pred, _ = my_model(batch) pred = pred.data.cpu().numpy() acc, max_acc = cal_accuracy(pred, batch[-1]) train_loss.append(loss.data) train_acc.append(acc) train_max_acc.append(max_acc) # back propogate my_model.zero_grad() optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(my_model.parameters(), cfg['gradient_clip']) optimizer.step() print('avg_training_loss', sum(train_loss) / len(train_loss)) print('max_training_acc', sum(train_max_acc) / len(train_max_acc)) print('avg_training_acc', sum(train_acc) / len(train_acc)) print("validating ...") eval_acc = inference(my_model, valid_data, entity2id, cfg) if eval_acc > best_dev_acc and cfg['to_save_model']: print("saving model to", cfg['save_model_file']) torch.save(my_model.state_dict(), cfg['save_model_file']) best_dev_acc = eval_acc except KeyboardInterrupt: break # Test set evaluation print("evaluating on test") print('loading model from ...', cfg['save_model_file']) my_model.load_state_dict(torch.load(cfg['save_model_file'])) test_acc = inference(my_model, test_data, entity2id, cfg, log_info=True) return test_acc