def evaluate(self, experiment, golden_instances, actual_instances):
methname = "cmp_mentions_%s_%s" % (
experiment.name,
self.use_strong_match and "strong" or "weak"
)
compare = globals()[methname]
m = Metrics()
iterable = izip_longest(golden_instances, actual_instances)
for ginstance, ainstance in iterable:
if not ainstance:
m.push(Metrics(fn=len(ginstance.mentions)))
continue
assert ginstance.instance_id == ainstance.instance_id
gm = ginstance.mentions
am = ainstance.mentions
result = compare(gm, am)
if self.print_report:
result.full_report(ginstance.instance_id)
if self.print_recap:
print "%20s\t%s" % (ginstance.instance_id, result.summary())
m.push(result.as_metric())
return m
def report_for(self, experiment, actual_instances, golden_instances):
methname = "cmp_mentions_%s_%s" % (
experiment.name,
self.use_strong_match and "strong" or "weak"
)
print "Using", methname
compare = globals()[methname]
m = Metrics()
threshold = 0.0
if self.best is not None:
if self.threshold > 0.0:
threshold = self.threshold
actual_instances = self.threshold_instances(
self.threshold, actual_instances, golden_instances
)
else:
threshold, actual_instances = self.find_best_threshold(
experiment, actual_instances, golden_instances
)
for ginstance, ainstance in izip_longest(golden_instances, actual_instances):
if not ginstance or not ainstance:
m.push(Metrics(fn=len(ginstance.mentions)))
continue
assert ginstance.instance_id == ainstance.instance_id
gm = ginstance.mentions
am = ainstance.mentions
result = compare(gm, am)
if self.print_report:
result.full_report(ginstance.instance_id)
if self.print_recap:
print "%20s\t%s" % (ginstance.instance_id, result.summary())
m.push(result.as_metric())
return (threshold, m)