def test_fmeasure(self):
'''
Test fmeasure.
'''
cm = boundary_confusion_matrix(self.masses_an1, self.masses_an2)
self.assertAlmostEquals(Decimal('0.25000000'), fmeasure(cm))
def test_precision(self):
'''
Test precision.
'''
cm = boundary_confusion_matrix(self.masses_an1, self.masses_an2)
self.assertAlmostEquals(Decimal('0.14285714'), precision(cm))
def test_recall(self):
'''
Test recall.
'''
cm = boundary_confusion_matrix(self.masses_an1, self.masses_an2)
self.assertAlmostEquals(Decimal('0.14285714'), recall(cm))
def evaluate(self, batch, preds, sent=True, word=True):
""" For a given batch and its corresponding preds, get metrics
batch: Batch instance
preds: list
Usage:
>> from loader import *
>> from modules import *
>>
>> model = TextSeg(lstm_dim=200, score_dim=200, bidir=True, num_layers=2)
>> trainer = Trainer(model=model,
train_dir='../data/wiki_727/train',
val_dir='../data/wiki_50/test',
batch_size=10,
lr=1e-3)
>> evalu = Metrics()
>>
>> batch = sample_and_batch(trainer.train_dir, trainer.batch_size, TRAIN=True)
>> preds = trainer.predict_batch(batch)
>> evalu(batch, preds)
"""
metric_dict = {}
assert (sent or
word), 'Missing: choose sent- and / or word-level evaluation.'
# Word level
if word:
w_true, w_pred = self._word(batch, preds)
metric_dict['w_pk'] = seg.pk(w_pred, w_true)
metric_dict['w_wd'] = seg.window_diff(w_pred,
w_true,
lamprier_et_al_2007_fix=True)
metric_dict['w_ss'] = seg.segmentation_similarity(w_pred, w_true)
metric_dict['w_bs'] = seg.boundary_similarity(w_pred, w_true)
w_confusion = seg.boundary_confusion_matrix(w_pred, w_true)
metric_dict['w_precision'] = seg.precision(w_confusion)
metric_dict['w_recall'] = seg.recall(w_confusion)
metric_dict['w_f1'] = seg.fmeasure(w_confusion)
# Sentence level
if sent:
s_true, s_pred = self._sent(batch, preds)
metric_dict['s_pk'] = seg.pk(s_pred, s_true)
metric_dict['s_wd'] = seg.window_diff(s_pred,
s_true,
lamprier_et_al_2007_fix=True)
metric_dict['s_ss'] = seg.segmentation_similarity(s_pred, s_true)
metric_dict['s_bs'] = seg.boundary_similarity(s_pred, s_true)
s_confusion = seg.boundary_confusion_matrix(s_pred, s_true)
metric_dict['s_precision'] = seg.precision(s_confusion)
metric_dict['s_recall'] = seg.recall(s_confusion)
metric_dict['s_f1'] = seg.fmeasure(s_confusion)
return metric_dict
