def create_dataloader(self):
data = DATAMultiWOZ(
debug = False,
data_dir= self.data_dir,
)
train_examples = data.read_examples(os.path.join(self.data_dir,'train.tsv'))
train_features = data.convert_examples_to_features(train_examples, self.tokenizer, self.max_seq_length)
all_input_ids = torch.tensor(data.select_field(train_features, 'input_ids'), dtype=torch.long)
all_input_mask = torch.tensor(data.select_field(train_features, 'input_mask'), dtype=torch.long)
all_segment_ids = torch.tensor(data.select_field(train_features, 'segment_ids'), dtype=torch.long)
all_label = torch.tensor([f.label for f in train_features], dtype=torch.long)
train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=self.train_batch_size)
eval_examples = data.read_examples(os.path.join(self.data_dir, 'test.tsv'))
eval_features = data.convert_examples_to_features(eval_examples, self.tokenizer, self.max_seq_length)
all_input_ids = torch.tensor(data.select_field(eval_features, 'input_ids'), dtype=torch.long)
all_input_mask = torch.tensor(data.select_field(eval_features, 'input_mask'), dtype=torch.long)
all_segment_ids = torch.tensor(data.select_field(eval_features, 'segment_ids'), dtype=torch.long)
all_label = torch.tensor([f.label for f in eval_features], dtype=torch.long)
eval_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label)
eval_sampler = SequentialSampler(eval_data)
eval_dataloader = DataLoader(eval_data, sampler=eval_sampler, batch_size=self.eval_batch_size)
return train_dataloader,eval_dataloader,train_examples,eval_examples
def test_eval(self):
data = DATAMultiWOZ(debug=False, data_dir=self.data_dir)
test_examples = data.read_examples(
os.path.join(self.data_dir, 'test.json'))
print('eval_examples的数量', len(test_examples))
ID = [x.guid for x in test_examples]
test_features = data.convert_examples_to_features(
test_examples, self.tokenizer, self.max_seq_length)
all_input_ids = torch.tensor(data.select_field(test_features,
'input_ids'),
dtype=torch.long)
all_input_mask = torch.tensor(data.select_field(
test_features, 'input_mask'),
dtype=torch.long)
all_segment_ids = torch.tensor(data.select_field(
test_features, 'segment_ids'),
dtype=torch.long)
eval_labels_domain = torch.tensor(
[f.labels_domain for f in test_features], dtype=torch.long)
eval_labels_dependcy = torch.tensor(
[f.labels_dependcy for f in test_features], dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask,
all_segment_ids, eval_labels_domain,
eval_labels_dependcy)
# Run prediction for full data
test_sampler = SequentialSampler(test_data)
test_dataloader = DataLoader(test_data,
sampler=test_sampler,
batch_size=self.eval_batch_size)
config = BertConfig.from_pretrained(self.model_name_or_path)
model = BertForTokenClassification.from_pretrained(os.path.join(
self.output_dir, "pytorch_model.bin"),
self.args,
config=config)
model.to(self.device)
model.eval()
inference_labels = []
gold_labels_domain = []
gold_labels_dependcy = []
scores_domain = []
scores_dependcy = []
for input_ids, input_mask, segment_ids, label_domain, label_dependcy in test_dataloader:
input_ids = input_ids.to(self.device)
input_mask = input_mask.to(self.device)
segment_ids = segment_ids.to(self.device)
label_domain = label_domain.to(self.device)
label_dependcy = label_dependcy.to(self.device)
with torch.no_grad():
logits_domain = model(
input_ids=input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask,
).view(-1, self.num_labels_domain).detach().cpu().numpy()
logits_dependcy = model(
input_ids=input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask,
).view(-1, self.num_labels_dependcy).detach().cpu().numpy()
label_domain = label_domain.view(-1).to('cpu').numpy()
label_dependcy = label_dependcy.view(-1).to('cpu').numpy()
scores_domain.append(logits_domain)
scores_dependcy.append(logits_dependcy)
gold_labels_domain.append(label_domain)
gold_labels_dependcy.append(label_dependcy)
gold_labels_domain = np.concatenate(gold_labels_domain, 0)
gold_labels_depandcy = np.concatenate(gold_labels_dependcy, 0)
scores_domain = np.concatenate(scores_domain, 0)
scores_dependcy = np.concatenate(scores_dependcy, 0)
# 计算评价指标
assert scores_domain.shape[0] == scores_dependcy.shape[
0] == gold_labels_domain.shape[0] == gold_labels_depandcy.shape[0]
eval_accuracy_domain = accuracyF1(scores_domain,
gold_labels_domain,
mode='domain',
report=True)
eval_accuracy_dependcy = accuracyF1(scores_dependcy,
gold_labels_dependcy,
mode='depency',
report=True)
print('eval_accuracy_domain', eval_accuracy_domain)
print('eval_accuracy_dependcy', eval_accuracy_dependcy)
def test_eval(self):
data = DATAMultiWOZ(
debug=False,
data_dir=self.data_dir
)
test_examples = data.read_examples(os.path.join(self.data_dir, 'test.tsv'))
print('eval_examples的数量', len(test_examples))
ID = [x.guid for x in test_examples]
test_features = data.convert_examples_to_features(test_examples, self.tokenizer, self.max_seq_length)
all_input_ids = torch.tensor(data.select_field(test_features, 'input_ids'), dtype=torch.long)
all_input_mask = torch.tensor(data.select_field(test_features, 'input_mask'), dtype=torch.long)
all_segment_ids = torch.tensor(data.select_field(test_features, 'segment_ids'), dtype=torch.long)
all_utterance_mask = torch.tensor(data.select_field(test_features, 'utterance_mask'), dtype=torch.long)
all_response_mask = torch.tensor(data.select_field(test_features, 'response_mask'), dtype=torch.long)
all_history_mask = torch.tensor(data.select_field(test_features, 'history_mask'), dtype=torch.long)
all_label = torch.tensor([f.label for f in test_features], dtype=torch.long)
test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,all_utterance_mask,all_response_mask,all_history_mask, all_label)
# Run prediction for full data
test_sampler = SequentialSampler(test_data)
test_dataloader = DataLoader(test_data, sampler=test_sampler, batch_size=self.eval_batch_size)
config = BertConfig.from_pretrained(self.model_name_or_path, num_labels=self.num_labels)
model = BertForSequenceClassification.from_pretrained(
os.path.join(self.output_dir, "pytorch_model.bin"), self.args, config=config)
model.to(self.device)
model.eval()
inference_labels = []
gold_labels = []
scores = []
for input_ids, input_mask, segment_ids, label_ids in test_dataloader:
input_ids = input_ids.to(self.device)
input_mask = input_mask.to(self.device)
segment_ids = segment_ids.to(self.device)
label_ids = label_ids.to(self.device)
with torch.no_grad():
logits = model(
input_ids=input_ids,
token_type_ids=segment_ids,
attention_mask=input_mask,
).detach().cpu().numpy()
label_ids = label_ids.to('cpu').numpy()
scores.append(logits)
inference_labels.append(np.argmax(logits, axis=1))
gold_labels.append(label_ids)
gold_labels = np.concatenate(gold_labels, 0)
scores = np.concatenate(scores, 0)
logits = np.concatenate(inference_labels, 0)
# 计算评价指标
assert len(ID) == scores.shape[0]== scores.shape[0]
eval_accuracy = accuracyF1(logits, gold_labels)
# eval_DOUBAN_MRR,eval_DOUBAN_mrr,eval_DOUBAN_MAP,eval_Precision1 = compute_DOUBAN(ID,scores,gold_labels)
# print(
# 'eval_MRR',eval_DOUBAN_MRR,eval_DOUBAN_mrr,
# 'eval_MAP',eval_DOUBAN_MAP,
# 'eval_Precision1',eval_Precision1)
print('F1',eval_accuracy)