def checkForAmbiguousIDs(self):
        """Check ground truth and hypotheses for multiple use of the same id per frame"""

        result = True
        
        for frame in self.groundtruth_["frames"]:
            ids = set()
            for groundtruth in frame["annotations"]:
                if not "id" in groundtruth:
                    # We should have already warned about a missing ID in checkForExistingIDs
                    # no need to raise an exception in this function by trying to access missing IDs
                    continue

                if groundtruth["id"] in ids:
                    result &= False
                    write_stderr_red("Warning:", "Ambiguous id (%s) found in ground truth, timestamp %f, frame %d!" % (str(groundtruth["id"]), frame["timestamp"], frame["num"] if "num" in frame else -1))
                else:
                    ids.add(groundtruth["id"])
        
        for frame in self.hypotheses_["frames"]:
            ids = set()
            for hypothesis in frame["hypotheses"]:
                if hypothesis["id"] in ids:
                    result &= False
                    write_stderr_red("Warning:", "Ambiguous hypothesis (%s) found in hypotheses, timestamp %f, frame %d!" % (str(hypothesis["id"]), frame["timestamp"], frame["num"] if "num" in frame else -1))
                else:
                    ids.add(hypothesis["id"])

        return result # true: OK, false: ambiguous id found
    def checkForAmbiguousIDs(self):
        """Check ground truth and hypotheses for multiple use of the same id per frame"""

        result = True

        for frame in self.groundtruth_["frames"]:
            ids = set()
            for groundtruth in frame["annotations"]:
                if not "id" in groundtruth:
                    # We should have already warned about a missing ID in checkForExistingIDs
                    # no need to raise an exception in this function by trying to access missing IDs
                    continue

                if groundtruth["id"] in ids:
                    result &= False
                    write_stderr_red("Warning:", "Ambiguous id (%s) found in ground truth, timestamp %f, frame %d!" % (str(groundtruth["id"]), frame["timestamp"], frame["num"] if "num" in frame else -1))
                else:
                    ids.add(groundtruth["id"])

        for frame in self.hypotheses_["frames"]:
            ids = set()
            for hypothesis in frame["hypotheses"]:
                if hypothesis["id"] in ids:
                    result &= False
                    write_stderr_red("Warning:", "Ambiguous hypothesis (%s) found in hypotheses, timestamp %f, frame %d!" % (str(hypothesis["id"]), frame["timestamp"], frame["num"] if "num" in frame else -1))
                else:
                    ids.add(hypothesis["id"])

        return result # true: OK, false: ambiguous id found
Пример #3
0
 def calcMOTP(abs_stats):
     num_corr = abs_stats['correspondences']
     if num_corr == 0:
         write_stderr_red("Warning",
                          "No ground truth. MOTA calculation not possible")
         return 0.0
     return float(abs_stats['total overlap']) / num_corr
    def checkForCompleteness(self):
        """Check ground truth and hypotheses for containing width, height, 
        x and y"""

        result = True
        expectedKeys = ("x", "y", "width", "height")

        for f in self.groundtruth_["frames"]:
            for g in f["annotations"]:
                for key in expectedKeys:
                    if not key in list(g.keys()):
                        write_stderr_red(
                            "Warning:", "Groundtruth without key %s found, "
                            "timestamp %f, frame %d!" %
                            (key, f["timestamp"],
                             f["num"] if "num" in f else -1))
                        result &= False

        # Hypotheses without ids or with empty ids
        for f in self.hypotheses_["frames"]:
            for h in f["hypotheses"]:
                for key in expectedKeys:
                    if not key in list(h.keys()):
                        write_stderr_red(
                            "Warning:", "Hypothesis without key %s found, "
                            "timestamp %f, frame %d!" %
                            (key, f["timestamp"],
                             f["num"] if "num" in f else -1))
                        result &= False

        return result  # true: OK, false: missing id found
Пример #5
0
    def getMOTP(self):
        motp = 0.0
        if self.total_correspondences_ == 0:
            write_stderr_red("Warning", "No correspondences found. MOTP calculation not possible")
#            raise("No correspondence found. MOTP calculation not possible")
        else:
            motp = self.total_overlap_ / self.total_correspondences_
        return motp
    def getMOTP(self):
        motp = 0.0
        if self.total_correspondences_ == 0:
            write_stderr_red("Warning", "No correspondences found. MOTP calculation not possible")
#            raise("No correspondence found. MOTP calculation not possible")
        else:
            motp = self.total_distance_ / self.total_correspondences_
        return motp
Пример #7
0
 def calcMOTA(abs_stats):
     num_gt = abs_stats['ground truths']
     if num_gt == 0:
         write_stderr_red("Warning",
                          "No ground truth. MOTA calculation not possible")
         return 0.0
     return 1.0 - float(abs_stats['misses'] + abs_stats['false positives'] +
                        abs_stats['mismatches']) / num_gt
Пример #8
0
    def getMOTA(self):
        mota = 0.0
        if self.total_groundtruths_ == 0:
            write_stderr_red("Warning", "No ground truth. MOTA calculation not possible")
#            raise("No ground truth. MOTA calculation not possible")
        else:
            mota = 1.0 - float(self.misses_ + self.false_positives_ + self.mismatches_) / float(self.total_groundtruths_)

        return mota
    def getMOTA(self):
        mota = 0.0
        if self.total_groundtruths_ == 0:
            write_stderr_red("Warning", "No ground truth. MOTA calculation not possible")
#            raise("No ground truth. MOTA calculation not possible")
        else:
            mota = 1.0 - float(self.misses_ + self.false_positives_ + self.mismatches_) / float(self.total_groundtruths_)

        return mota
Пример #10
0
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--groundtruth', type=str, required=True, help='Groundtruth json file')
    parser.add_argument('-b', '--hypothesis', type=str, required=True, help='Hypotheses json file')
    parser.add_argument('-c', '--check_format', action="store_true", default=True, help='Check if input json follow required format, default=%(default)s')
    parser.add_argument('-v', '--visual_debug_file', type=str, required=False, default=None, help='save visual debug frames to json file')
    parser.add_argument('-i', '--iou_threshold', type=float, required=False, default=0.2, help='iou threshold for bbox match, default=%(default)s')
    args = parser.parse_args()

    # Load ground truth according to format
    # Assume MOT format, if non-json
    gt = open(args.groundtruth) # gt file
    if args.groundtruth.endswith(".json"):
        groundtruth = json.load(gt)[0]
    else:
        groundtruth = MOT_groundtruth_import(gt.readlines())
    gt.close()

    # Load MOT format files
    hypo = open(args.hypothesis) # hypo file
    if args.hypothesis.endswith(".json"):
        hypotheses = json.load(hypo)[0]
    else:
        hypotheses = MOT_hypo_import(hypo.readlines())
    hypo.close()


    evaluator = MOTEvaluation(groundtruth, hypotheses, args.iou_threshold)

    if(args.check_format):
        formatChecker = FormatChecker(groundtruth, hypotheses)
        success = formatChecker.checkForExistingIDs()
        success |= formatChecker.checkForAmbiguousIDs()
        success |= formatChecker.checkForCompleteness()

        if not success:
            write_stderr_red("Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n    File: %s" % args.groundtruth)
            sys.exit()
        else:
            print("Format check pass\n")

    evaluator.evaluate()
    print("Track statistics")
    evaluator.printTrackStatistics()
    print("")
    print("Results")
    evaluator.printResults()
    #evaluator.printLegacyFormat()

    #print(json.dumps(evaluator.getAbsoluteStatistics(), indent=4, sort_keys=True))
    #print(json.dumps(evaluator.getRelativeStatistics(), indent=4, sort_keys=True))

    if(args.visual_debug_file):
        with open(args.visual_debug_file, 'w') as fp:
            json.dump(evaluator.getVisualDebug(), fp, indent=4)
    def checkForExistingIDs(self):
        """Check ground truth and hypotheses for having a valid id. Valid: existing and non-empty."""

        result = True

        for f in self.groundtruth_["frames"]:
            for g in f["annotations"]:
                
                if not "id" in g.keys():
                    write_stderr_red("Warning:", "Groundtruth without ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

                if g["id"] == "":
                    write_stderr_red("Warning:", "Groundtruth with empty ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

        # Hypotheses without ids or with empty ids
        for f in self.hypotheses_["frames"]:
            for h in f["hypotheses"]:
                if not "id" in h.keys():
                    write_stderr_red("Warning:", "Hypothesis without ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

                if h["id"] == "":
                    write_stderr_red("Warning:", "Hypothesis with empty ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

        return result # true: OK, false: missing id found
    def checkForExistingIDs(self):
        """Check ground truth and hypotheses for having a valid id. Valid: existing and non-empty."""

        result = True

        for f in self.groundtruth_["frames"]:
            for g in f["annotations"]:

                if not "id" in g.keys():
                    write_stderr_red("Warning:", "Groundtruth without ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

                if g["id"] == "":
                    write_stderr_red("Warning:", "Groundtruth with empty ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

        # Hypotheses without ids or with empty ids
        for f in self.hypotheses_["frames"]:
            for h in f["hypotheses"]:
                if not "id" in h.keys():
                    write_stderr_red("Warning:", "Hypothesis without ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

                if h["id"] == "":
                    write_stderr_red("Warning:", "Hypothesis with empty ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1))
                    result &= False
                    continue

        return result # true: OK, false: missing id found
    def checkForCompleteness(self):
        """Check ground truth and hypotheses for containing width, height, x and y"""

        result = True
        expectedKeys = ("x", "y", "width", "height")

        for f in self.groundtruth_["frames"]:
            for g in f["annotations"]:
                for key in expectedKeys:
                    if not key in g.keys():
                        write_stderr_red("Warning:", "Groundtruth without key %s found, timestamp %f, frame %d!" % (key, f["timestamp"], f["num"] if "num" in f else -1))
                        result &= False

        # Hypotheses without ids or with empty ids
        for f in self.hypotheses_["frames"]:
            for h in f["hypotheses"]:
                for key in expectedKeys:
                    if not key in h.keys():
                        write_stderr_red("Warning:", "Hypothesis without key %s found, timestamp %f, frame %d!" % (key, f["timestamp"], f["num"] if "num" in f else -1))
                        result &= False

        return result # true: OK, false: missing id found
Пример #14
0
 def calcMOTP(abs_stats):
     num_corr = abs_stats['correspondences']
     if num_corr == 0:
         write_stderr_red("Warning", "No ground truth. MOTA calculation not possible")
         return 0.0
     return float(abs_stats['total overlap']) / num_corr
Пример #15
0
 def calcMOTA(abs_stats):
     num_gt = abs_stats['ground truths']
     if num_gt == 0:
         write_stderr_red("Warning", "No ground truth. MOTA calculation not possible")
         return 0.0
     return 1.0 - float(abs_stats['misses'] + abs_stats['false positives'] + abs_stats['mismatches']) / num_gt
Пример #16
0
        hypotheses = json.load(hypo)[0]
    else:
        hypotheses = MOT_hypo_import(hypo.readlines())
    hypo.close()


    evaluator = MOTEvaluation(groundtruth, hypotheses)

    if(args.check_format):
        formatChecker = FormatChecker(groundtruth, hypotheses)
        success = formatChecker.checkForExistingIDs()
        success |= formatChecker.checkForAmbiguousIDs()
        success |= formatChecker.checkForCompleteness()

        if not success:
            write_stderr_red("Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n    File: %s" % args.groundtruth)
            sys.exit()

    evaluator.evaluate()
    print "Track statistics"
    evaluator.printTrackStatistics()
    print 
    print "Results"
    evaluator.printResults()
#    evaluator.printLegacyFormat()

#    print json.dumps(evaluator.getAbsoluteStatistics(), indent=4, sort_keys=True)
#    print json.dumps(evaluator.getRelativeStatistics(), indent=4, sort_keys=True)

    if(args.visual_debug_file):
        with open(args.visual_debug_file, 'w') as fp:
Пример #17
0
        hypotheses = json.load(hypo)[0]
    else:
        hypotheses = MOT_hypo_import(hypo.readlines())
    hypo.close()

    evaluator = MOTEvaluation(groundtruth, hypotheses)

    if (args.check_format):
        formatChecker = FormatChecker(groundtruth, hypotheses)
        success = formatChecker.checkForExistingIDs()
        success |= formatChecker.checkForAmbiguousIDs()
        success |= formatChecker.checkForCompleteness()

        if not success:
            write_stderr_red(
                "Error:",
                "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n    File: %s"
                % args.groundtruth)
            sys.exit()

    evaluator.evaluate()
    print "Track statistics"
    evaluator.printTrackStatistics()
    print
    print "Results"
    evaluator.printResults()
    #    evaluator.printLegacyFormat()

    #    print json.dumps(evaluator.getAbsoluteStatistics(), indent=4, sort_keys=True)
    #    print json.dumps(evaluator.getRelativeStatistics(), indent=4, sort_keys=True)

    if (args.visual_debug_file):