/
dataset_reader.py
138 lines (117 loc) · 5.77 KB
/
dataset_reader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
import tempfile
from typing import Dict, Iterable
from overrides import overrides
from commons import BOND_TOKEN
import torch
import copy
from allennlp.data import Instance
from allennlp.data.dataset_readers import DatasetReader
from allennlp.data.fields import SpanField, ListField, TextField, MetadataField, ArrayField, SequenceLabelField, LabelField
from allennlp.data.fields import LabelField, TextField
from allennlp.data.tokenizers import Token, Tokenizer, WhitespaceTokenizer
from parameters import Params
import glob
import os
import random
import pdb
from tqdm import tqdm
import json
from tokenizer import CustomTokenizer
import numpy as np
import pdb
import Levenshtein
class NamedEntityResolutionReader(DatasetReader):
def __init__(
self,
config,
max_tokens: int = None,
**kwargs
):
super().__init__(**kwargs)
self.custom_tokenizer_class = CustomTokenizer(config=config)
self.token_indexers = self.custom_tokenizer_class.token_indexer_returner()
self.max_tokens = max_tokens
self.config = config
self.train_mention_ids, self.dev_mention_ids, self.test_mention_ids, self.mention_id2data = \
self._mention_ids_returner()
@overrides
def _read(self, train_dev_test_flag: str) -> list:
'''
:param train_dev_test_flag: 'train', 'dev', 'test'
:return: list of instances
'''
mention_ids, instances = list(), list()
if train_dev_test_flag == 'train':
mention_ids += self.train_mention_ids
# Because Iterator(shuffle=True) has bug, we forcefully shuffle train dataset here.
random.shuffle(mention_ids)
elif train_dev_test_flag == 'dev':
mention_ids += self.dev_mention_ids
elif train_dev_test_flag == 'test':
mention_ids += self.test_mention_ids
for idx, mention_uniq_id in tqdm(enumerate(mention_ids)):
instances.append(self.text_to_instance(mention_uniq_id, data=self.mention_id2data[mention_uniq_id]))
return instances
@overrides
def text_to_instance(self, mention_uniq_id, data=None) -> Instance:
l_tokenized = [Token('[CLS]')]
l_tokenized += [Token(split_token) for split_token in self.custom_tokenizer_class.tokenize(txt=data['l'])][:self.config.max_mention_length]
l_tokenized.append(Token('[SEP]'))
r_tokenized = [Token('[CLS]')]
r_tokenized += [Token(split_token) for split_token in self.custom_tokenizer_class.tokenize(txt=data['r'])][:self.config.max_mention_length]
r_tokenized.append(Token('[SEP]'))
l_plus_r = [Token('[CLS]')]
l_plus_r += [Token(split_token) for split_token in self.custom_tokenizer_class.tokenize(txt=data['l'])][:self.config.max_mention_length]
l_plus_r += [Token(BOND_TOKEN)]
l_plus_r += [Token(split_token) for split_token in self.custom_tokenizer_class.tokenize(txt=data['r'])][:self.config.max_mention_length]
l_plus_r += [Token('[SEP]')]
context_field = TextField(l_tokenized, self.token_indexers)
fields = {"l": context_field}
fields['lev'] = ArrayField(np.array(Levenshtein.distance(data['l'], data['r'])))
# / (max(len(data['l']), len(data['r'])))))
fields['r'] = TextField(r_tokenized, self.token_indexers)
fields['l_plus_r'] = TextField(l_plus_r, self.token_indexers)
fields['label'] = ArrayField(np.array(data['label']))
fields['mention_uniq_id'] = ArrayField(np.array(mention_uniq_id))
fields['subword_match_num'] = ArrayField(np.array(len(set(self.custom_tokenizer_class.tokenize(txt=data['l'])) &
set(self.custom_tokenizer_class.tokenize(txt=data['r'])))))
return Instance(fields)
def _mention_ids_returner(self):
mention_id2data = {}
train_mention_ids, dev_mention_ids, test_mention_ids = [], [], []
dataset_dir = self.config.dataset_dir
for train_dev_test_flag in ['train', 'dev', 'test']:
with open(dataset_dir+train_dev_test_flag+'.tsv', 'r') as f:
for idx, line in enumerate(f):
if idx != 0 and line.strip() != '':
data = line.strip().split('\t')
try:
l = data[0]
r = data[1]
label = int(data[2])
mention_id = len(mention_id2data)
mention_id2data.update({mention_id: {'l': l,
'r': r,
'label': label}})
if train_dev_test_flag == 'train':
train_mention_ids.append(mention_id)
elif train_dev_test_flag == 'dev':
dev_mention_ids.append(mention_id)
elif train_dev_test_flag == 'test':
test_mention_ids.append(mention_id)
except:
print('Parse Error:', line.strip())
continue
if self.config.debug:
if idx == self.config.debug_sample_num:
break
# if self.config.debug:
# # only for debugging model
# dev_mention_ids = copy.copy(train_mention_ids)
# test_mention_ids = copy.copy(train_mention_ids)
return train_mention_ids, dev_mention_ids, test_mention_ids, mention_id2data
if __name__ == '__main__':
params = Params()
config = params.opts
dsr = NamedEntityResolutionReader(config=config)
dsr._read('train')