def valid(model, mode='all'): model.eval() with open(DATASET_DEV_CLS3, 'rb') as f: dataset_cls3 = pickle.load(f) with open(DATASET_DEV_CLS18, 'rb') as f: dataset_cls18 = pickle.load(f) dataset_summ_qa = data.ConcatDataset([dataset_cls3, dataset_cls18]) cls3_loader = torch.utils.data.DataLoader(dataset=dataset_cls3, batch_size=VALID_BATCH, shuffle=False, collate_fn=lambda x: x) cls3_iterator = iter(cls3_loader) cls18_loader = torch.utils.data.DataLoader(dataset=dataset_cls18, batch_size=VALID_BATCH, shuffle=False, collate_fn=lambda x: x) cls18_iterator = iter(cls18_loader) rouge_summ = rouge_qa = None acc_cls3 = acc_cls18 = 0 # -------------------------------------------------------------------- if mode in ['all', 'summ', 'qa']: data_val_sum_qa = [] if VALID_NUM > 0: for i in range(VALID_NUM): data_val_sum_qa.append(dataset_summ_qa[i]) else: for i in range(len(dataset_summ_qa)): data_val_sum_qa.append(dataset_summ_qa[i]) if mode in ['all', 'summ']: refs = [' '.join(data['question']) for data in data_val_sum_qa] x = [data['description'] for data in data_val_sum_qa] hyps = beam_search('summ', model, x) hyps = [' '.join(list(sent)) for sent in hyps] rouge = Rouge() try: rouge_summ = rouge.get_scores(hyps, refs, avg=True, ignore_empty=True) print_rouge(rouge_summ) except RuntimeError: print('Failed to compute Rouge!') if mode in ['all', 'qa']: refs = [' '.join(data['answer']) for data in data_val_sum_qa] x = [data['question'] for data in data_val_sum_qa] hyps = beam_search('qa', model, x) hyps = [' '.join(list(sent)) for sent in hyps] rouge = Rouge() try: rouge_qa = rouge.get_scores(hyps, refs, avg=True, ignore_empty=True) print_rouge(rouge_qa) except RuntimeError: print('Failed to compute Rouge!') # cls3 & cls18 def iter_through_cls_dev(iterator, mode): val_correct = 0 val_num = 0 for i in range(math.ceil(VALID_NUM / VALID_BATCH)): mini_batch = next(iterator) question = [data['question'] for data in mini_batch] description = [data['description'] for data in mini_batch] y_gt = torch.tensor([data['category'] for data in mini_batch]).to(device) y_pred = model(source=description, source2=question, target=None, mode=mode) y_pred_labels = torch.argmax(y_pred, dim=1) val_correct += (y_gt == y_pred_labels).sum().item() val_num += len(mini_batch) return val_correct / val_num if mode in ['all', 'cls3']: acc_cls3 = iter_through_cls_dev(cls3_iterator, 'cls3') print('Acc_cls3:', acc_cls3) if mode in ['all', 'cls18']: acc_cls18 = iter_through_cls_dev(cls18_iterator, 'cls18') print('Acc_cls18:', acc_cls18) if is_training: model.train() return rouge_summ, rouge_qa, acc_cls3, acc_cls18
import pandas as pd import numpy as np import os from rouge.rouge import Rouge # ref = 'the only thing crazier than a guy in snowbound massachusetts boxing up the powdery white stuff and offering it for sale online ? people are actually buying it . for $ 89 , self-styled entrepreneur kyle waring will ship you 6 pounds of boston-area snow in an insulated styrofoam box – enough for 10 to 15 snowballs , he says .' # cand = 'a man in suburban boston is selling snow online to customers in warmer states . for $ 89 , he will ship 6 pounds of snow in an insulated styrofoam box .' ref = 'the only thing crazier than a guy in snowbound massachusetts boxing up the powdery white stuff and offering it for sale online ? people are actually buying it . for $ 89 , self-styled entrepreneur kyle waring will ship you 6 pounds of boston-area snow in an insulated styrofoam box – enough for 10 to 15 snowballs , he says .' cand = 'in suburban boston , a man is selling snow online to customers in warmer states . he will ship 6 pounds of snow in an insulated styrofoam box for $ 89 .' rouge = Rouge() scores = rouge.get_scores(cand, ref)[0]['rouge-1']['r']