def evaluate_block(self, data_iter: DataLoader,
docs: Sequence[Document]) -> float:
"""
:param data_iter:
:param docs:
:return:
"""
self.decode_block(data_iter=data_iter, docs=docs)
if self.chunking:
acc = ChunkF1()
for doc in docs:
for sen in doc.sentences:
acc.update(labels=sen[to_gold(self.key)],
preds=sen[self.key])
else:
acc = Accuracy()
for doc in docs:
for sen in doc.sentences:
labels = nd.array([
self.label_map.cid(label)
for label in sen[to_gold(self.key)]
])
preds = nd.array(
[self.label_map.cid(pred) for pred in sen[self.key]])
acc.update(labels=labels, preds=preds)
return acc.get()[1]
def dev_evaluate(self, model, data):
correctLabels = []
predLabels = []
b = Progbar(len(data))
for i, data1 in enumerate(data):
tokens, casing, char, labels = data1
tokens = np.asarray([tokens])
casing = np.asarray([casing])
char = np.asarray([char])
pred = model.predict([tokens, casing, char], verbose=False)[0]
pred = pred.argmax(axis=-1) # Predict the classes
correctLabels.append(labels)
predLabels.append(pred)
b.update(i)
b.update(i + 1)
label_pred = []
for sentence in predLabels:
label_pred.append(
[self.idx2Label[element] for element in sentence])
label_correct = []
for sentence in correctLabels:
label_correct.append(
[self.idx2Label[element] for element in sentence])
acc = ChunkF1()
for pred, label in zip(label_pred, label_correct):
acc.update(pred, label)
print(float(acc.get()[1]))
def evaluate(self, data: List[Tuple[List[str], List[str]]],
**kwargs) -> float:
"""
:param data:
:param kwargs:
:return: the accuracy of this model.
"""
preds = self.decode(data)
labels = [y for y, _ in data]
acc = ChunkF1()
for pred, label in zip(preds, labels):
acc.update(pred, label)
return float(acc.get()[1])