-
Notifications
You must be signed in to change notification settings - Fork 1
/
data_util.py
executable file
·214 lines (184 loc) · 7.72 KB
/
data_util.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Utility functions to process data.
"""
import os
import pickle
import logging
from collections import Counter
import numpy as np
from util import read_conll, read_json, one_hot, window_iterator, ConfusionMatrix, load_word_vector_mapping, load_word_vector_mapping_np
from defs import LBLS, NONE, LMAP, NUM, UNK, EMBED_SIZE
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
FDIM = 4
P_CASE = "CASE:"
CASES = ["aa", "AA", "Aa", "aA"]
START_TOKEN = "<s>"
END_TOKEN = "</s>"
def casing(word):
if len(word) == 0: return word
# all lowercase
if word.islower(): return "aa"
# all uppercase
elif word.isupper(): return "AA"
# starts with capital
elif word[0].isupper(): return "Aa"
# has non-initial capital
else: return "aA"
def normalize(word):
"""
Normalize words that are numbers or have casing.
"""
if word.isdigit(): return NUM
else: return word.lower()
def featurize(embeddings, word):
"""
Featurize a word given embeddings.
"""
case = casing(word)
word = normalize(word)
case_mapping = {c: one_hot(FDIM, i) for i, c in enumerate(CASES)}
wv = embeddings.get(word, embeddings[UNK])
fv = case_mapping[case]
return np.hstack((wv, fv))
def evaluate(model, X, Y):
cm = ConfusionMatrix(labels=LBLS)
Y_ = model.predict(X)
for i in range(Y.shape[0]):
y, y_ = np.argmax(Y[i]), np.argmax(Y_[i])
cm.update(y,y_)
cm.print_table()
return cm.summary()
class ModelHelper(object):
"""
This helper takes care of preprocessing data, constructing embeddings, etc.
"""
def __init__(self, tok2id, max_length):
self.tok2id = tok2id
self.START = [tok2id[START_TOKEN], tok2id[P_CASE + "aa"]]
self.END = [tok2id[END_TOKEN], tok2id[P_CASE + "aa"]]
self.max_length = max_length
def vectorize_example(self, sentence, labels=None):
sentence_ = [[self.tok2id.get(normalize(word), self.tok2id[UNK]), self.tok2id[P_CASE + casing(word)]] for word in sentence]
if labels:
labels_ = [LBLS.index(l) for l in labels]
return sentence_, labels_
else:
return sentence_, [LBLS[-1] for _ in sentence]
def vectorize(self, data):
return [self.vectorize_example(sentence, labels) for sentence, labels in data]
@classmethod
def build(cls, data):
# Preprocess data to construct an embedding
# Reserve 0 for the special NIL token.
tok2id = build_dict((normalize(word) for sentence, _ in data for word in sentence), offset=1, max_words=10000)
tok2id.update(build_dict([P_CASE + c for c in CASES], offset=len(tok2id)))
tok2id.update(build_dict([START_TOKEN, END_TOKEN, UNK], offset=len(tok2id)))
assert sorted(tok2id.items(), key=lambda t: t[1])[0][1] == 1
logger.info("Built dictionary for %d features.", len(tok2id))
max_length = max(len(sentence) for sentence, _ in data)
return cls(tok2id, max_length)
def build_triplet(cls, data):
# Preprocess data to construct an embedding
# Reserve 0 for the special NIL token.
qtok2id = build_dict((normalize(word) for question,_,_ in data for word in question), offset=1, max_words=10000)
qtok2id.update(build_dict([P_CASE + c for c in CASES], offset=len(qtok2id)))
qtok2id.update(build_dict([START_TOKEN, END_TOKEN, UNK], offset=len(qtok2id)))
assert sorted(qtok2id.items(), key=lambda t: t[1])[0][1] == 1
ctok2id = build_dict((normalize(word) for _,context,_ in data for word in context), offset=1, max_words=10000)
ctok2id.update(build_dict([P_CASE + c for c in CASES], offset=len(ctok2id)))
ctok2id.update(build_dict([START_TOKEN, END_TOKEN, UNK], offset=len(ctok2id)))
assert sorted(ctok2id.items(), key=lambda t: t[1])[0][1] == 1
logger.info("Built dictionary for %d features.", len(ctok2id))
max_qlength = max(len(question) for question,_,_ in data)
max_clength = max(len(context) for _,context,_ in data)
return cls(qtok2id, max_qlength, ctok2id, max_clength)
def save(self, path):
# Make sure the directory exists.
if not os.path.exists(path):
os.makedirs(path)
# Save the tok2id map.
with open(os.path.join(path, "features.pkl"), "w") as f:
pickle.dump([self.tok2id, self.max_length], f)
@classmethod
def load(cls, path):
# Make sure the directory exists.
assert os.path.exists(path) and os.path.exists(os.path.join(path, "features.pkl"))
# Save the tok2id map.
with open(os.path.join(path, "features.pkl")) as f:
tok2id, max_length = pickle.load(f)
return cls(tok2id, max_length)
def load_and_preprocess_data(args):
logger.info("Loading training data...")
train = read_json('../../data/squad/train')
logger.info("Done. Read %d sentences", len(train))
logger.info("Loading dev data...")
dev = read_json('../../data/squad/val')
logger.info("Done. Read %d sentences", len(dev))
#print('step1:',train[0])
# concat questions and context
train_concat = []
for question,context,ans in train:
if question != []:
ans_long = ["NOT"]*len(question+context)
ans_long[len(question)+ans[0]:len(question)+ans[1]] = ["ANS"]*(ans[1]-ans[0]+1)
train_concat.append((question + context, ans_long))
#print(train_concat[0])
dev_concat = []
for question,context,ans in dev:
if question != []:
ans_long = ["NOT"]*len(question+context)
ans_long[len(question)+ans[0]:len(question)+ans[1]] = ["ANS"]*(ans[1]-ans[0]+1)
dev_concat.append((question + context, ans_long))
helper = ModelHelper.build(train_concat)
# now process all the input data.
train_data = helper.vectorize(train_concat)
dev_data = helper.vectorize(dev_concat)
return helper, train_data, dev_data, train, dev
def load_embeddings(args, helper):
embeddings = np.array(np.random.randn(len(helper.tok2id) + 1, EMBED_SIZE), dtype=np.float32)
embeddings[0] = 0.
for word, vec in load_word_vector_mapping_np(args.vocab, args.vectors).items():
word = normalize(word)
if word in helper.tok2id:
embeddings[helper.tok2id[word]] = vec
logger.info("Initialized embeddings.")
return embeddings
def build_dict(words, max_words=None, offset=0):
cnt = Counter(words)
if max_words:
words = cnt.most_common(max_words)
else:
words = cnt.most_common()
return {word: offset+i for i, (word, _) in enumerate(words)}
def get_chunks(seq, default=LBLS.index(NONE)):
"""Breaks input of 4 4 4 0 0 4 0 -> (0, 4, 5), (0, 6, 7)"""
chunks = []
chunk_type, chunk_start = None, None
for i, tok in enumerate(seq):
# End of a chunk 1
if tok == default and chunk_type is not None:
# Add a chunk.
chunk = (chunk_type, chunk_start, i)
chunks.append(chunk)
chunk_type, chunk_start = None, None
# End of a chunk + start of a chunk!
elif tok != default:
if chunk_type is None:
chunk_type, chunk_start = tok, i
elif tok != chunk_type:
chunk = (chunk_type, chunk_start, i)
chunks.append(chunk)
chunk_type, chunk_start = tok, i
else:
pass
# end condition
if chunk_type is not None:
chunk = (chunk_type, chunk_start, len(seq))
chunks.append(chunk)
return chunks
def test_get_chunks():
assert get_chunks([4, 4, 4, 0, 0, 4, 1, 2, 4, 3], 4) == [(0,3,5), (1, 6, 7), (2, 7, 8), (3,9,10)]