-
Notifications
You must be signed in to change notification settings - Fork 0
/
run.py
336 lines (275 loc) · 11.9 KB
/
run.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
#!/usr/bin/python
# vim: set file-encoding=utf-8:
import sys
import math
import itertools
import operator
import cPickle
import string
import maxent
from collections import defaultdict
from maxent import MaxentModel
from optparse import OptionParser
# |iterable| should yield lines.
def read_sentences(iterable):
sentence = []
for line in iterable:
columns = line.rstrip().split()
if len(columns) == 0 and len(sentence) > 0:
yield sentence
sentence = []
if len(columns) > 0:
sentence.append(columns)
if len(sentence) > 0:
yield sentence
# Computes (local) features for word at position |i| given that label for word
# at position |i - 1| is |previous_label|. You can pass any additional data
# via |data| argument.
MIN_WORD_FREQUENCY = 3
MIN_LABEL_FREQUENCY = 10
PREV_WORDS = dict({
# "over": ["B-PER"],
# "boven": ["B-LOC"],
"tegen": ["B-LOC", "B_PER"],
"op": ["B-LOC", "B-ORG"],
# "voor": ["B-PER", "B-ORG"],
# "tussen": ["B-PER"],
"buiten": ["B-LOC"],
# "door": ["B-PER", "B-ORG"],
"tijdens": ["B-MISC"],
"uit": ["B-LOC"],
# "van": ["B-PER", "B-LOC", "B-ORG"],
"naar": ["B-LOC"],
"in": ["B-LOC"],
"aan": ["B-PER"],
"met": ["B-PER", "B-ORG"],
"volgens": ["B-PER"]
})
PROOF_MARKERS = ["minister", "dokter", "president", "koning", "prins", "directeur", "en"]
def is_up(word):
return word[0] in string.ascii_uppercase
def is_proof_marker(word):
for marker in PROOF_MARKERS:
if marker.endswith(word) or marker.startswith(word):
return True
return False
def get_prev_word(words, i, back):
return words[i - back] if i >= back else ""
def compute_features(data, words, poses, i, previous_label):
ppp_word = get_prev_word(words, i, 3)
pp_word = get_prev_word(words, i, 2)
p_word = get_prev_word(words, i, 1)
word = words[i]
marker_pos = data.get("proof_marker_pos", -1000)
if is_proof_marker(word.lower()):
marker_pos = i
uppers, downs = 0, 0
for letter in word:
if letter in string.ascii_uppercase:
uppers = uppers + 1
else:
downs = downs + 1
# In test was words like 'hello-Vasya',
# we try to find 'Vasya' and work with it
was_defis = False
if "-" in word:
was_defis = True
for w in word.split("-"):
if len(w) > 0 and is_up(w):
word = w
break
else:
yield "was-labelled-as={0}".format("O")
# Check for first letter
if not is_up(word):
yield "was-labelled-as={0}".format("O")
# ONLY IF UPPER LETTER
else:
# Check for part of speech
if not poses[i] in ["V", "N", "Int", "Art", "Prep", "Adj", "Adv"]:
yield "was-labelled-as={0}".format("O")
# Check for words with different cases 'AvsdA'
if uppers > 1 and downs > 0:
yield "was-labelled-as={0}".format("I-MISC" if previous_label[0] == "B" else "B-MISC")
# For titule + name + surname
if marker_pos + 1 == i:
yield "was-labelled-as={0}".format("B-PER")
if marker_pos + 2 == i and previous_label == "B-PER":
yield "was-labelled-as={0}".format("I-PER")
# For titule + van + name + surname
if marker_pos + 2 == i and p_word.lower() == "van":
yield "was-labelled-as={0}".format("B-ORG")
if marker_pos + 3 == i and is_up(p_word) and pp_word == "van":
yield "was-labelled-as={0}".format("B-PER")
if marker_pos + 4 == i and is_up(pp_word) and previous_label[0] == "B" and ppp_word == "van":
yield "was-labelled-as={0}".format("I-PER")
# Start of sentense
if (p_word.strip() == "" or p_word == "--") and poses[i] == "N":
yield "was-labelled-as={0}".format("B-PER")
# For seases and towns
if word.endswith("zee") or word.endswith("stad") or word.endswith("burg") or word.endswith("burgh"):
yield "was-labelled-as={0}".format("I-LOC" if previous_label[0] == "B" else "B-LOC")
if poses[i - 1] == "V" and previous_label == "O":
yield "was-labelled-as={0}".format("B-PER")
# Check for abbreviation
if word.upper() == word:
if not was_defis:
yield "was-labelled-as={0}".format("I-ORG" if previous_label[0] == "B" else "B-ORG")
else:
yield "was-labelled-as={0}".format("I-MISC" if previous_label[0] == "B" else "B-MISC")
if poses[i - 1] == "Art" and previous_label == "O":
yield "was-labelled-as={0}".format("B-MISC")
if poses[i] == "Adj":
yield "was-labelled-as={0}".format("B-MISC")
if word.endswith("ë"):
yield "was-labelled-as={0}".format("B-LOC")
# if poses[i] == "Adv":
# yield "was-labelled-as={0}".format("I" + previous_label[1:] if previous_label[0] == "B" else "O")
if poses[i] == "Num":
yield "was-labelled-as={0}".format("I" + previous_label[1:] if previous_label[0] == "B" else "B-ORG")
# Check previous word
for (pr_w, labs) in PREV_WORDS.items():
if not pr_w == p_word.lower():
continue
for l in labs:
yield "was-labelled-as={0}".format(l)
labels = data["labelled_words"].get(word, dict())
labels = filter(lambda item: item[1] > MIN_LABEL_FREQUENCY, labels.items())
#for label in labels:
# yield "was-labelled-as={0}".format(label)
if data["word_frequencies"].get(word, 0) >= MIN_WORD_FREQUENCY:
yield "word-current={0}".format(word)
# Condition on previous label.
if previous_label != "O":
yield "label-previous={0}".format(previous_label)
data["proof_marker_pos"] = marker_pos
# |iterable| should yield sentences.
# |iterable| should support multiple passes.
def train_model(options, iterable):
model = MaxentModel()
data = {}
data["feature_set"] = set()
data["word_frequencies"] = defaultdict(long)
# XXX(sandello): defaultdict(lambda: defaultdict(long)) would be
# a better choice here (for |labelled_words|) but it could not be pickled.
# C'est la vie.
data["labelled_words"] = dict()
print >>sys.stderr, "*** Training options are:"
print >>sys.stderr, " ", options
print >>sys.stderr, "*** First pass: Computing statistics..."
for n, sentence in enumerate(iterable):
if (n % 1000) == 0:
print >>sys.stderr, " {0:6d} sentences...".format(n)
for word, pos, label in sentence:
data["word_frequencies"][word] += 1
if label.startswith("B-") or label.startswith("I-"):
if word in data["labelled_words"]:
data["labelled_words"][word][label] += 1
else:
data["labelled_words"][word] = defaultdict(long)
print >>sys.stderr, "*** Second pass: Collecting features..."
model.begin_add_event()
for n, sentence in enumerate(iterable):
if (n % 1000) == 0:
print >>sys.stderr, " {0:6d} sentences...".format(n)
words, poses, labels = map(list, zip(*sentence))
for i in xrange(len(labels)):
features = compute_features(data, words, poses, i, labels[i - 1] if i >= 1 else "^")
features = list(features)
model.add_event(features, labels[i])
for feature in features:
data["feature_set"].add(feature)
model.end_add_event(options.cutoff)
print >>sys.stderr, "*** Collected {0} features.".format(len(data["feature_set"]))
print >>sys.stderr, "*** Training..."
maxent.set_verbose(1)
model.train(options.iterations, options.technique, options.gaussian)
maxent.set_verbose(0)
print >>sys.stderr, "*** Saving..."
model.save(options.model + ".maxent")
with open(options.model + ".data", "w") as handle:
cPickle.dump(data, handle)
# |iterable| should yield sentences.
def eval_model(options, iterable):
model = MaxentModel()
data = {}
print >>sys.stderr, "*** Loading..."
model.load(options.model + ".maxent")
with open(options.model + ".data", "r") as handle:
data = cPickle.load(handle)
print >>sys.stderr, "*** Evaluating..."
for n, sentence in enumerate(iterable):
if (n % 100) == 0:
print >>sys.stderr, " {0:6d} sentences...".format(n)
words, poses = map(list, zip(*sentence))
labels = eval_model_sentence(options, data, model, words, poses)
for word, pos, label in zip(words, poses, labels):
print label
print
# This is a helper method for |eval_model_sentence| and, actually,
# an implementation of Viterbi algorithm.
def eval_model_sentence(options, data, model, words, poses):
viterbi_layers = [ None for i in xrange(len(words)) ]
viterbi_backpointers = [ None for i in xrange(len(words) + 1) ]
# Compute first layer directly.
viterbi_layers[0] = model.eval_all(list(compute_features(data, words, poses, 0, "^")))
viterbi_layers[0] = dict( (k, math.log(v)) for k, v in viterbi_layers[0] )
viterbi_backpointers[0] = dict( (k, None) for k, v in viterbi_layers[0].iteritems() )
# Compute intermediate layers.
for i in xrange(1, len(words)):
viterbi_layers[i] = defaultdict(lambda: float("-inf"))
viterbi_backpointers[i] = defaultdict(lambda: None)
for prev_label, prev_logprob in viterbi_layers[i - 1].iteritems():
features = compute_features(data, words, poses, i, prev_label)
features = list(features)
for label, prob in model.eval_all(features):
logprob = math.log(prob)
if prev_logprob + logprob > viterbi_layers[i][label]:
viterbi_layers[i][label] = prev_logprob + logprob
viterbi_backpointers[i][label] = prev_label
# Most probable endpoint.
max_logprob = float("-inf")
max_label = None
for label, logprob in viterbi_layers[len(words) - 1].iteritems():
if logprob > max_logprob:
max_logprob = logprob
max_label = label
# Most probable sequence.
path = []
label = max_label
for i in reversed(xrange(len(words))):
path.insert(0, label)
label = viterbi_backpointers[i][label]
return path
################################################################################
def main():
parser = OptionParser("A sample MEMM model for NER")
parser.add_option("-T", "--train", action="store_true", dest="train",
help="Do the training, if specified; do the evaluation otherwise")
parser.add_option("-f", "--file", type="string", dest="filename",
metavar="FILE", help="File with the training data")
parser.add_option("-m", "--model", type="string", dest="model",
metavar="FILE", help="File with the model")
parser.add_option("-c", "--cutoff", type="int", default=5, dest="cutoff",
metavar="C", help="Event frequency cutoff during training")
parser.add_option("-i", "--iterations", type="int", default=100, dest="iterations",
metavar="N", help="Number of training iterations")
parser.add_option("-g", "--gaussian", type="float", default=0.0, dest="gaussian",
metavar="G", help="Gaussian smoothing penalty (sigma)")
parser.add_option("-t", "--technique", type="string", default="gis", dest="technique",
metavar="T", help="Training algorithm (either 'gis' or 'lbfgs')")
(options, args) = parser.parse_args()
if not options.filename:
parser.print_help()
sys.exit(1)
with open(options.filename, "r") as handle:
data = list(read_sentences(handle))
if options.train:
print >>sys.stderr, "*** Training model..."
train_model(options, data)
else:
print >>sys.stderr, "*** Evaluating model..."
eval_model(options, data)
print >>sys.stderr, "*** Done!"
if __name__ == "__main__":
main()