def checkForAmbiguousIDs(self): """Check ground truth and hypotheses for multiple use of the same id per frame""" result = True for frame in self.groundtruth_["frames"]: ids = set() for groundtruth in frame["annotations"]: if not "id" in groundtruth: # We should have already warned about a missing ID in checkForExistingIDs # no need to raise an exception in this function by trying to access missing IDs continue if groundtruth["id"] in ids: result &= False write_stderr_red("Warning:", "Ambiguous id (%s) found in ground truth, timestamp %f, frame %d!" % (str(groundtruth["id"]), frame["timestamp"], frame["num"] if "num" in frame else -1)) else: ids.add(groundtruth["id"]) for frame in self.hypotheses_["frames"]: ids = set() for hypothesis in frame["hypotheses"]: if hypothesis["id"] in ids: result &= False write_stderr_red("Warning:", "Ambiguous hypothesis (%s) found in hypotheses, timestamp %f, frame %d!" % (str(hypothesis["id"]), frame["timestamp"], frame["num"] if "num" in frame else -1)) else: ids.add(hypothesis["id"]) return result # true: OK, false: ambiguous id found
def calcMOTP(abs_stats): num_corr = abs_stats['correspondences'] if num_corr == 0: write_stderr_red("Warning", "No ground truth. MOTA calculation not possible") return 0.0 return float(abs_stats['total overlap']) / num_corr
def checkForCompleteness(self): """Check ground truth and hypotheses for containing width, height, x and y""" result = True expectedKeys = ("x", "y", "width", "height") for f in self.groundtruth_["frames"]: for g in f["annotations"]: for key in expectedKeys: if not key in list(g.keys()): write_stderr_red( "Warning:", "Groundtruth without key %s found, " "timestamp %f, frame %d!" % (key, f["timestamp"], f["num"] if "num" in f else -1)) result &= False # Hypotheses without ids or with empty ids for f in self.hypotheses_["frames"]: for h in f["hypotheses"]: for key in expectedKeys: if not key in list(h.keys()): write_stderr_red( "Warning:", "Hypothesis without key %s found, " "timestamp %f, frame %d!" % (key, f["timestamp"], f["num"] if "num" in f else -1)) result &= False return result # true: OK, false: missing id found
def getMOTP(self): motp = 0.0 if self.total_correspondences_ == 0: write_stderr_red("Warning", "No correspondences found. MOTP calculation not possible") # raise("No correspondence found. MOTP calculation not possible") else: motp = self.total_overlap_ / self.total_correspondences_ return motp
def getMOTP(self): motp = 0.0 if self.total_correspondences_ == 0: write_stderr_red("Warning", "No correspondences found. MOTP calculation not possible") # raise("No correspondence found. MOTP calculation not possible") else: motp = self.total_distance_ / self.total_correspondences_ return motp
def calcMOTA(abs_stats): num_gt = abs_stats['ground truths'] if num_gt == 0: write_stderr_red("Warning", "No ground truth. MOTA calculation not possible") return 0.0 return 1.0 - float(abs_stats['misses'] + abs_stats['false positives'] + abs_stats['mismatches']) / num_gt
def getMOTA(self): mota = 0.0 if self.total_groundtruths_ == 0: write_stderr_red("Warning", "No ground truth. MOTA calculation not possible") # raise("No ground truth. MOTA calculation not possible") else: mota = 1.0 - float(self.misses_ + self.false_positives_ + self.mismatches_) / float(self.total_groundtruths_) return mota
def main(): parser = argparse.ArgumentParser() parser.add_argument('-a', '--groundtruth', type=str, required=True, help='Groundtruth json file') parser.add_argument('-b', '--hypothesis', type=str, required=True, help='Hypotheses json file') parser.add_argument('-c', '--check_format', action="store_true", default=True, help='Check if input json follow required format, default=%(default)s') parser.add_argument('-v', '--visual_debug_file', type=str, required=False, default=None, help='save visual debug frames to json file') parser.add_argument('-i', '--iou_threshold', type=float, required=False, default=0.2, help='iou threshold for bbox match, default=%(default)s') args = parser.parse_args() # Load ground truth according to format # Assume MOT format, if non-json gt = open(args.groundtruth) # gt file if args.groundtruth.endswith(".json"): groundtruth = json.load(gt)[0] else: groundtruth = MOT_groundtruth_import(gt.readlines()) gt.close() # Load MOT format files hypo = open(args.hypothesis) # hypo file if args.hypothesis.endswith(".json"): hypotheses = json.load(hypo)[0] else: hypotheses = MOT_hypo_import(hypo.readlines()) hypo.close() evaluator = MOTEvaluation(groundtruth, hypotheses, args.iou_threshold) if(args.check_format): formatChecker = FormatChecker(groundtruth, hypotheses) success = formatChecker.checkForExistingIDs() success |= formatChecker.checkForAmbiguousIDs() success |= formatChecker.checkForCompleteness() if not success: write_stderr_red("Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n File: %s" % args.groundtruth) sys.exit() else: print("Format check pass\n") evaluator.evaluate() print("Track statistics") evaluator.printTrackStatistics() print("") print("Results") evaluator.printResults() #evaluator.printLegacyFormat() #print(json.dumps(evaluator.getAbsoluteStatistics(), indent=4, sort_keys=True)) #print(json.dumps(evaluator.getRelativeStatistics(), indent=4, sort_keys=True)) if(args.visual_debug_file): with open(args.visual_debug_file, 'w') as fp: json.dump(evaluator.getVisualDebug(), fp, indent=4)
def checkForExistingIDs(self): """Check ground truth and hypotheses for having a valid id. Valid: existing and non-empty.""" result = True for f in self.groundtruth_["frames"]: for g in f["annotations"]: if not "id" in g.keys(): write_stderr_red("Warning:", "Groundtruth without ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1)) result &= False continue if g["id"] == "": write_stderr_red("Warning:", "Groundtruth with empty ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1)) result &= False continue # Hypotheses without ids or with empty ids for f in self.hypotheses_["frames"]: for h in f["hypotheses"]: if not "id" in h.keys(): write_stderr_red("Warning:", "Hypothesis without ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1)) result &= False continue if h["id"] == "": write_stderr_red("Warning:", "Hypothesis with empty ID found, timestamp %f, frame %d!" % (f["timestamp"], f["num"] if "num" in f else -1)) result &= False continue return result # true: OK, false: missing id found
def checkForCompleteness(self): """Check ground truth and hypotheses for containing width, height, x and y""" result = True expectedKeys = ("x", "y", "width", "height") for f in self.groundtruth_["frames"]: for g in f["annotations"]: for key in expectedKeys: if not key in g.keys(): write_stderr_red("Warning:", "Groundtruth without key %s found, timestamp %f, frame %d!" % (key, f["timestamp"], f["num"] if "num" in f else -1)) result &= False # Hypotheses without ids or with empty ids for f in self.hypotheses_["frames"]: for h in f["hypotheses"]: for key in expectedKeys: if not key in h.keys(): write_stderr_red("Warning:", "Hypothesis without key %s found, timestamp %f, frame %d!" % (key, f["timestamp"], f["num"] if "num" in f else -1)) result &= False return result # true: OK, false: missing id found
hypotheses = json.load(hypo)[0] else: hypotheses = MOT_hypo_import(hypo.readlines()) hypo.close() evaluator = MOTEvaluation(groundtruth, hypotheses) if(args.check_format): formatChecker = FormatChecker(groundtruth, hypotheses) success = formatChecker.checkForExistingIDs() success |= formatChecker.checkForAmbiguousIDs() success |= formatChecker.checkForCompleteness() if not success: write_stderr_red("Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n File: %s" % args.groundtruth) sys.exit() evaluator.evaluate() print "Track statistics" evaluator.printTrackStatistics() print print "Results" evaluator.printResults() # evaluator.printLegacyFormat() # print json.dumps(evaluator.getAbsoluteStatistics(), indent=4, sort_keys=True) # print json.dumps(evaluator.getRelativeStatistics(), indent=4, sort_keys=True) if(args.visual_debug_file): with open(args.visual_debug_file, 'w') as fp:
hypotheses = json.load(hypo)[0] else: hypotheses = MOT_hypo_import(hypo.readlines()) hypo.close() evaluator = MOTEvaluation(groundtruth, hypotheses) if (args.check_format): formatChecker = FormatChecker(groundtruth, hypotheses) success = formatChecker.checkForExistingIDs() success |= formatChecker.checkForAmbiguousIDs() success |= formatChecker.checkForCompleteness() if not success: write_stderr_red( "Error:", "Stopping. Fix ids first. Evaluating with broken data does not make sense!\n File: %s" % args.groundtruth) sys.exit() evaluator.evaluate() print "Track statistics" evaluator.printTrackStatistics() print print "Results" evaluator.printResults() # evaluator.printLegacyFormat() # print json.dumps(evaluator.getAbsoluteStatistics(), indent=4, sort_keys=True) # print json.dumps(evaluator.getRelativeStatistics(), indent=4, sort_keys=True) if (args.visual_debug_file):