def valid(model, mode='all'):
model.eval()
with open(DATASET_DEV_CLS3, 'rb') as f:
dataset_cls3 = pickle.load(f)
with open(DATASET_DEV_CLS18, 'rb') as f:
dataset_cls18 = pickle.load(f)
dataset_summ_qa = data.ConcatDataset([dataset_cls3, dataset_cls18])
cls3_loader = torch.utils.data.DataLoader(dataset=dataset_cls3,
batch_size=VALID_BATCH,
shuffle=False,
collate_fn=lambda x: x)
cls3_iterator = iter(cls3_loader)
cls18_loader = torch.utils.data.DataLoader(dataset=dataset_cls18,
batch_size=VALID_BATCH,
shuffle=False,
collate_fn=lambda x: x)
cls18_iterator = iter(cls18_loader)
rouge_summ = rouge_qa = None
acc_cls3 = acc_cls18 = 0
# --------------------------------------------------------------------
if mode in ['all', 'summ', 'qa']:
data_val_sum_qa = []
if VALID_NUM > 0:
for i in range(VALID_NUM):
data_val_sum_qa.append(dataset_summ_qa[i])
else:
for i in range(len(dataset_summ_qa)):
data_val_sum_qa.append(dataset_summ_qa[i])
if mode in ['all', 'summ']:
refs = [' '.join(data['question']) for data in data_val_sum_qa]
x = [data['description'] for data in data_val_sum_qa]
hyps = beam_search('summ', model, x)
hyps = [' '.join(list(sent)) for sent in hyps]
rouge = Rouge()
try:
rouge_summ = rouge.get_scores(hyps,
refs,
avg=True,
ignore_empty=True)
print_rouge(rouge_summ)
except RuntimeError:
print('Failed to compute Rouge!')
if mode in ['all', 'qa']:
refs = [' '.join(data['answer']) for data in data_val_sum_qa]
x = [data['question'] for data in data_val_sum_qa]
hyps = beam_search('qa', model, x)
hyps = [' '.join(list(sent)) for sent in hyps]
rouge = Rouge()
try:
rouge_qa = rouge.get_scores(hyps,
refs,
avg=True,
ignore_empty=True)
print_rouge(rouge_qa)
except RuntimeError:
print('Failed to compute Rouge!')
# cls3 & cls18
def iter_through_cls_dev(iterator, mode):
val_correct = 0
val_num = 0
for i in range(math.ceil(VALID_NUM / VALID_BATCH)):
mini_batch = next(iterator)
question = [data['question'] for data in mini_batch]
description = [data['description'] for data in mini_batch]
y_gt = torch.tensor([data['category']
for data in mini_batch]).to(device)
y_pred = model(source=description,
source2=question,
target=None,
mode=mode)
y_pred_labels = torch.argmax(y_pred, dim=1)
val_correct += (y_gt == y_pred_labels).sum().item()
val_num += len(mini_batch)
return val_correct / val_num
if mode in ['all', 'cls3']:
acc_cls3 = iter_through_cls_dev(cls3_iterator, 'cls3')
print('Acc_cls3:', acc_cls3)
if mode in ['all', 'cls18']:
acc_cls18 = iter_through_cls_dev(cls18_iterator, 'cls18')
print('Acc_cls18:', acc_cls18)
if is_training:
model.train()
return rouge_summ, rouge_qa, acc_cls3, acc_cls18
import pandas as pd import numpy as np import os from rouge.rouge import Rouge # ref = 'the only thing crazier than a guy in snowbound massachusetts boxing up the powdery white stuff and offering it for sale online ? people are actually buying it . for $ 89 , self-styled entrepreneur kyle waring will ship you 6 pounds of boston-area snow in an insulated styrofoam box – enough for 10 to 15 snowballs , he says .' # cand = 'a man in suburban boston is selling snow online to customers in warmer states . for $ 89 , he will ship 6 pounds of snow in an insulated styrofoam box .' ref = 'the only thing crazier than a guy in snowbound massachusetts boxing up the powdery white stuff and offering it for sale online ? people are actually buying it . for $ 89 , self-styled entrepreneur kyle waring will ship you 6 pounds of boston-area snow in an insulated styrofoam box – enough for 10 to 15 snowballs , he says .' cand = 'in suburban boston , a man is selling snow online to customers in warmer states . he will ship 6 pounds of snow in an insulated styrofoam box for $ 89 .' rouge = Rouge() scores = rouge.get_scores(cand, ref)[0]['rouge-1']['r']
