예제 #1
0
    def evaluate(self, experiment, golden_instances, actual_instances):
        methname = "cmp_mentions_%s_%s" % (
            experiment.name,
            self.use_strong_match and "strong" or "weak"
        )
        compare = globals()[methname]

        m = Metrics()
        iterable = izip_longest(golden_instances, actual_instances)

        for ginstance, ainstance in iterable:
            if not ainstance:
                m.push(Metrics(fn=len(ginstance.mentions)))
                continue

            assert ginstance.instance_id == ainstance.instance_id
            gm = ginstance.mentions
            am = ainstance.mentions

            result = compare(gm, am)

            if self.print_report:
                result.full_report(ginstance.instance_id)
            if self.print_recap:
                print "%20s\t%s" % (ginstance.instance_id, result.summary())

            m.push(result.as_metric())

        return m
예제 #2
0
    def report_for(self, experiment, actual_instances, golden_instances):
        methname = "cmp_mentions_%s_%s" % (
            experiment.name,
            self.use_strong_match and "strong" or "weak"
        )
        print "Using", methname
        compare = globals()[methname]

        m = Metrics()
        threshold = 0.0

        if self.best is not None:
            if self.threshold > 0.0:
                threshold = self.threshold
                actual_instances = self.threshold_instances(
                    self.threshold, actual_instances, golden_instances
                )
            else:
                threshold, actual_instances = self.find_best_threshold(
                    experiment, actual_instances, golden_instances
                )

        for ginstance, ainstance in izip_longest(golden_instances, actual_instances):
            if not ginstance or not ainstance:
                m.push(Metrics(fn=len(ginstance.mentions)))
                continue

            assert ginstance.instance_id == ainstance.instance_id
            gm = ginstance.mentions
            am = ainstance.mentions

            result = compare(gm, am)

            if self.print_report:
                result.full_report(ginstance.instance_id)
            if self.print_recap:
                print "%20s\t%s" % (ginstance.instance_id, result.summary())

            m.push(result.as_metric())

        return (threshold, m)