def main():
args = parseArgs()
print(args)
argv = ['', args.groundTruth, args.predictions]
print("Loading data")
gtFramesAll, prFramesAll = eval_helpers.load_data_dir(argv)
print("# gt frames :", len(gtFramesAll))
print("# pred frames:", len(prFramesAll))
if (not os.path.exists(args.outputDir)):
os.makedirs(args.outputDir)
txt = open('result.txt', 'a')
if (args.evalPoseEstimation):
#####################################################
# evaluate per-frame multi-person pose estimation (AP)
# compute AP
print("Evaluation of per-frame multi-person pose estimation")
apAll, preAll, recAll = evaluateAP(gtFramesAll, prFramesAll,
args.outputDir, True,
args.saveEvalPerSequence)
# print AP
print("Average Precision (AP) metric:")
eval_helpers.printTable(apAll)
print("Mean Precision (Pre) metric:")
eval_helpers.printTable(preAll)
print("Mean Recall (Rec) metric:")
eval_helpers.printTable(recAll)
total_AP, total_Pre, total_Rec = apAll[15][0], preAll[15][0], recAll[
15][0]
txt.write('AP:{} Pre:{} Rec:{}\n'.format(total_AP, total_Pre,
total_Rec))
txt.close()
if (args.evalPoseTracking):
#####################################################
# evaluate multi-person pose tracking in video (MOTA)
# compute MOTA
print("Evaluation of video-based multi-person pose tracking")
metricsAll = evaluateTracking(gtFramesAll, prFramesAll, args.outputDir,
True, args.saveEvalPerSequence)
metrics = np.zeros([Joint().count + 4, 1])
for i in range(Joint().count + 1):
metrics[i, 0] = metricsAll['mota'][0, i]
metrics[Joint().count + 1, 0] = metricsAll['motp'][0, Joint().count]
metrics[Joint().count + 2, 0] = metricsAll['pre'][0, Joint().count]
metrics[Joint().count + 3, 0] = metricsAll['rec'][0, Joint().count]
# print AP
print("Multiple Object Tracking (MOT) metrics:")
eval_helpers.printTable(metrics, motHeader=True)
def main():
args = parseArgs()
print args
argv = ['', args.groundTruth, args.predictions]
print "Loading data"
gtFramesAll, prFramesAll = eval_helpers.load_data_dir(argv)
print "# gt frames :", len(gtFramesAll)
print "# pred frames:", len(prFramesAll)
if (not os.path.exists(args.outputDir)):
os.makedirs(args.outputDir)
if (args.evalPoseEstimation):
#####################################################
# evaluate per-frame multi-person pose estimation (AP)
# compute AP
print "Evaluation of per-frame multi-person pose estimation"
apAll, preAll, recAll = evaluateAP(gtFramesAll, prFramesAll,
args.outputDir, True,
args.saveEvalPerSequence)
# print AP
print "Average Precision (AP) metric:"
eval_helpers.printTable(apAll)
if (args.evalPoseTracking):
#####################################################
# evaluate multi-person pose tracking in video (MOTA)
# compute MOTA
print "Evaluation of video-based multi-person pose tracking"
metricsAll = evaluateTracking(gtFramesAll, prFramesAll, args.outputDir,
True, args.saveEvalPerSequence)
metrics = np.zeros([Joint().count + 4, 1])
for i in range(Joint().count + 1):
metrics[i, 0] = metricsAll['mota'][0, i]
metrics[Joint().count + 1, 0] = metricsAll['motp'][0, Joint().count]
metrics[Joint().count + 2, 0] = metricsAll['pre'][0, Joint().count]
metrics[Joint().count + 3, 0] = metricsAll['rec'][0, Joint().count]
# print AP
print "Multiple Object Tracking (MOT) metrics:"
eval_helpers.printTable(metrics, motHeader=True)
def computeMetrics(gtFramesAll, motAll, outputDir, bSaveAll, bSaveSeq):
assert (len(gtFramesAll) == len(motAll))
nJoints = Joint().count
seqidxs = []
for imgidx in range(len(gtFramesAll)):
seqidxs += [gtFramesAll[imgidx]["seq_id"]]
seqidxs = np.array(seqidxs)
seqidxsUniq = np.unique(seqidxs)
# intermediate metrics
metricsMidNames = [
'num_misses', 'num_switches', 'num_false_positives', 'num_objects',
'num_detections'
]
# final metrics computed from intermediate metrics
metricsFinNames = ['mota', 'motp', 'pre', 'rec']
# initialize intermediate metrics
metricsMidAll = {}
for name in metricsMidNames:
metricsMidAll[name] = np.zeros([1, nJoints])
metricsMidAll['sumD'] = np.zeros([1, nJoints])
# initialize final metrics
metricsFinAll = {}
for name in metricsFinNames:
metricsFinAll[name] = np.zeros([1, nJoints + 1])
# create metrics
mh = mm.metrics.create()
imgidxfirst = 0
# iterate over tracking sequences
# seqidxsUniq = seqidxsUniq[:20]
nSeq = len(seqidxsUniq)
# initialize per-sequence metrics
metricsSeqAll = {}
for si in range(nSeq):
metricsSeqAll[si] = {}
for name in metricsFinNames:
metricsSeqAll[si][name] = np.zeros([1, nJoints + 1])
names = Joint().name
names['15'] = 'total'
for si in range(nSeq):
#for si in range(5):
print("seqidx: %d/%d" % (si + 1, nSeq))
# init per-joint metrics accumulator
accAll = {}
for i in range(nJoints):
accAll[i] = mm.MOTAccumulator(auto_id=True)
# extract frames IDs for the sequence
imgidxs = np.argwhere(seqidxs == seqidxsUniq[si])
imgidxs = imgidxs[:-1].copy()
seqName = gtFramesAll[imgidxs[0, 0]]["seq_name"]
print(seqName)
# create an accumulator that will be updated during each frame
# iterate over frames
for j in range(len(imgidxs)):
imgidx = imgidxs[j, 0]
# iterate over joints
for i in range(nJoints):
# GT tracking ID
trackidxGT = motAll[imgidx][i]["trackidxGT"]
# prediction tracking ID
trackidxPr = motAll[imgidx][i]["trackidxPr"]
# distance GT <-> pred part to compute MOT metrics
# 'NaN' means force no match
dist = motAll[imgidx][i]["dist"]
# Call update once per frame
accAll[i].update(
trackidxGT, # Ground truth objects in this frame
trackidxPr, # Detector hypotheses in this frame
dist # Distances from objects to hypotheses
)
# compute intermediate metrics per joint per sequence
for i in range(nJoints):
metricsMid = mh.compute(accAll[i],
metrics=metricsMidNames,
return_dataframe=False,
name='acc')
for name in metricsMidNames:
metricsMidAll[name][0, i] += metricsMid[name]
s = accAll[i].events['D'].sum()
if (np.isnan(s)):
s = 0
metricsMidAll['sumD'][0, i] += s
# if (bSaveSeq):
if False:
# compute metrics per joint per sequence
for i in range(nJoints):
metricsMid = mh.compute(accAll[i],
metrics=metricsMidNames,
return_dataframe=False,
name='acc')
# compute final metrics per sequence
if (metricsMid['num_objects'] > 0):
numObj = metricsMid['num_objects']
else:
numObj = np.nan
numFP = metricsMid['num_false_positives']
metricsSeqAll[si]['mota'][0, i] = 100 * (
1. - 1. * (metricsMid['num_misses'] +
metricsMid['num_switches'] + numFP) / numObj)
numDet = metricsMid['num_detections']
s = accAll[i].events['D'].sum()
if (numDet == 0 or np.isnan(s)):
metricsSeqAll[si]['motp'][0, i] = 0.0
else:
metricsSeqAll[si]['motp'][0, i] = 100 * (1. -
(1. * s / numDet))
if (numFP + numDet > 0):
totalDet = numFP + numDet
else:
totalDet = np.nan
metricsSeqAll[si]['pre'][0, i] = 100 * (1. * numDet / totalDet)
metricsSeqAll[si]['rec'][0, i] = 100 * (1. * numDet / numObj)
# average metrics over all joints per sequence
idxs = np.argwhere(
~np.isnan(metricsSeqAll[si]['mota'][0, :nJoints]))
metricsSeqAll[si]['mota'][0, nJoints] = metricsSeqAll[si]['mota'][
0, idxs].mean()
idxs = np.argwhere(
~np.isnan(metricsSeqAll[si]['motp'][0, :nJoints]))
metricsSeqAll[si]['motp'][0, nJoints] = metricsSeqAll[si]['motp'][
0, idxs].mean()
idxs = np.argwhere(
~np.isnan(metricsSeqAll[si]['pre'][0, :nJoints]))
metricsSeqAll[si]['pre'][0, nJoints] = metricsSeqAll[si]['pre'][
0, idxs].mean()
idxs = np.argwhere(
~np.isnan(metricsSeqAll[si]['rec'][0, :nJoints]))
metricsSeqAll[si]['rec'][0, nJoints] = metricsSeqAll[si]['rec'][
0, idxs].mean()
metricsSeq = metricsSeqAll[si].copy()
metricsSeq['mota'] = metricsSeq['mota'].flatten().tolist()
metricsSeq['motp'] = metricsSeq['motp'].flatten().tolist()
metricsSeq['pre'] = metricsSeq['pre'].flatten().tolist()
metricsSeq['rec'] = metricsSeq['rec'].flatten().tolist()
metricsSeq['names'] = names
filename = outputDir + '/' + seqName + '_MOT_metrics.json'
print('saving results to', filename)
#eval_helpers.writeJson(metricsSeq,filename)
writeJson(metricsSeq, filename)
# compute final metrics per joint for all sequences
for i in range(nJoints):
if (metricsMidAll['num_objects'][0, i] > 0):
numObj = metricsMidAll['num_objects'][0, i]
else:
numObj = np.nan
numFP = metricsMidAll['num_false_positives'][0, i]
metricsFinAll['mota'][0, i] = 100 * (
1. - (metricsMidAll['num_misses'][0, i] +
metricsMidAll['num_switches'][0, i] + numFP) / numObj)
numDet = metricsMidAll['num_detections'][0, i]
s = metricsMidAll['sumD'][0, i]
if (numDet == 0 or np.isnan(s)):
metricsFinAll['motp'][0, i] = 0.0
else:
metricsFinAll['motp'][0, i] = 100 * (1. - (s / numDet))
if (numFP + numDet > 0):
totalDet = numFP + numDet
else:
totalDet = np.nan
metricsFinAll['pre'][0, i] = 100 * (1. * numDet / totalDet)
metricsFinAll['rec'][0, i] = 100 * (1. * numDet / numObj)
# average metrics over all joints over all sequences
idxs = np.argwhere(~np.isnan(metricsFinAll['mota'][0, :nJoints]))
metricsFinAll['mota'][0, nJoints] = metricsFinAll['mota'][0, idxs].mean()
idxs = np.argwhere(~np.isnan(metricsFinAll['motp'][0, :nJoints]))
metricsFinAll['motp'][0, nJoints] = metricsFinAll['motp'][0, idxs].mean()
idxs = np.argwhere(~np.isnan(metricsFinAll['pre'][0, :nJoints]))
metricsFinAll['pre'][0, nJoints] = metricsFinAll['pre'][0, idxs].mean()
idxs = np.argwhere(~np.isnan(metricsFinAll['rec'][0, :nJoints]))
metricsFinAll['rec'][0, nJoints] = metricsFinAll['rec'][0, idxs].mean()
# if (bSaveAll):
if False:
metricsFin = metricsFinAll.copy()
metricsFin['mota'] = metricsFin['mota'].flatten().tolist()
metricsFin['motp'] = metricsFin['motp'].flatten().tolist()
metricsFin['pre'] = metricsFin['pre'].flatten().tolist()
metricsFin['rec'] = metricsFin['rec'].flatten().tolist()
metricsFin['names'] = names
filename = outputDir + '/total_MOT_metrics.json'
print('saving results to', filename)
# eval_helpers.writeJson(metricsFin,filename)
writeJson(metricsFin, filename)
return metricsFinAll
def evaluateAP(gtFramesAll,
prFramesAll,
outputDir,
bSaveAll=True,
bSaveSeq=False):
distThresh = 0.5
seqidxs = []
for imgidx in range(len(gtFramesAll)):
seqidxs += [gtFramesAll[imgidx]["seq_id"]]
seqidxs = np.array(seqidxs)
seqidxsUniq = np.unique(seqidxs)
nSeq = len(seqidxsUniq)
names = Joint().name
names['15'] = 'total'
if (bSaveSeq):
for si in range(nSeq):
print("seqidx: %d/%d" % (si + 1, nSeq))
# extract frames IDs for the sequence
imgidxs = np.argwhere(seqidxs == seqidxsUniq[si])
seqName = gtFramesAll[imgidxs[0, 0]]["seq_name"]
gtFrames = [
gtFramesAll[imgidx] for imgidx in imgidxs.flatten().tolist()
]
prFrames = [
prFramesAll[imgidx] for imgidx in imgidxs.flatten().tolist()
]
# assign predicted poses to GT poses
scores, labels, nGT, _ = eval_helpers.assignGTmulti(
gtFrames, prFrames, distThresh)
# compute average precision (AP), precision and recall per part
ap, pre, rec = computeMetrics(scores, labels, nGT)
metricsSeq = {
'ap': ap.flatten().tolist(),
'pre': pre.flatten().tolist(),
'rec': rec.flatten().tolist(),
'names': names
}
filename = outputDir + '/' + seqName + '_AP_metrics.json'
print('saving results to', filename)
eval_helpers.writeJson(metricsSeq, filename)
# assign predicted poses to GT poses
scoresAll, labelsAll, nGTall, _ = eval_helpers.assignGTmulti(
gtFramesAll, prFramesAll, distThresh)
# compute average precision (AP), precision and recall per part
apAll, preAll, recAll = computeMetrics(scoresAll, labelsAll, nGTall)
if (bSaveAll):
metrics = {
'ap': apAll.flatten().tolist(),
'pre': preAll.flatten().tolist(),
'rec': recAll.flatten().tolist(),
'names': names
}
filename = outputDir + '/total_AP_metrics.json'
print('saving results to', filename)
eval_helpers.writeJson(metrics, filename)
return apAll, preAll, recAll
def evaluate(gtdir,
preddir,
eval_pose=True,
eval_track=True,
eval_upper_bound=False):
gtFramesAll, prFramesAll = load_data_dir(['', gtdir, preddir])
print('# gt frames :', len(gtFramesAll))
print('# pred frames:', len(prFramesAll))
apAll = np.full((Joint().count + 1, 1), np.nan)
preAll = np.full((Joint().count + 1, 1), np.nan)
recAll = np.full((Joint().count + 1, 1), np.nan)
if eval_pose:
apAll, preAll, recAll = evaluateAP(gtFramesAll, prFramesAll)
print('Average Precision (AP) metric:')
#printTable(apAll)
cum = printTable(apAll)
metrics = np.full((Joint().count + 4, 1), np.nan)
#print(eval_track)
if eval_track:
#print(xy)
metricsAll = evaluateTracking(gtFramesAll, prFramesAll,
eval_upper_bound)
for i in range(Joint().count + 1):
metrics[i, 0] = metricsAll['mota'][0, i]
metrics[Joint().count + 1, 0] = metricsAll['motp'][0, Joint().count]
metrics[Joint().count + 2, 0] = metricsAll['pre'][0, Joint().count]
metrics[Joint().count + 3, 0] = metricsAll['rec'][0, Joint().count]
print('Multiple Object Tracking (MOT) metrics:')
track_cum = printTable(metrics, motHeader=True)
#return (apAll, preAll, recAll), metrics
#print(xy)
return cum