def produce(self, ip):
"""
Compute mAP (mean average precision) by comparing ground truth to prediction
:param ip: ground truth list, prediction list
ground truth is list of lists where each element is (frame, label, [xmin, ymin, xmax, ymax], [attributes])
prediction is list of lists where each element is (frame, label, [xmin, ymin, xmax, ymax], [attributes], confidence)
Note: Even if there are no attributes, an empty list will be received from the previous chain object
:return: mAP
"""
actualRaw, predictedRaw = ip
# filter out values where labels do not match
predicted = []
for frame in predictedRaw:
inList = []
for row in frame:
if self._is_valid(row, 'object') and self._is_valid(row, 'occlusion') and self._is_valid(row, 'activity'):
inList.append((row[1], row[2], row[-1])) # required output format: (label, [xmin, ymin, xmax, ymax], conf)
predicted.append(inList)
actual = []
for frame in actualRaw:
inList = []
for row in frame:
if self._is_valid(row, 'object') and self._is_valid(row, 'occlusion') and self._is_valid(row, 'activity'):
inList.append((row[1], row[2])) # required output format: (label, [xmin, ymin, xmax, ymax])
actual.append(inList)
assert len(actual) == len(predicted)
return mAP.evaluate_mAP(actual, predicted)
def setUp(self):
self.groundTruth = \
{0: {'person': [0, 0, 10, 10]},
1: {'person': [0, 0, 11, 11], 'dog': [20, 20, 30, 30]},
2: {'person': [0, 0, 12, 12], 'dog': [0, 0, 13, 13]},
3: {'dog': [0, 0, 12, 12], 'person': [0, 0, 13, 13]}}
self.predicted = \
{0: {'person': ([0, 0, 10, 10], 0.70), 'dog': ([6, 6, 11, 11], 0.75), 'cat': ([0, 0, 10, 10], 0.65)},
1: {'person': ([0, 0, 11, 11], 0.71)},
2: {'person': ([0, 0, 12, 12], 0.72)},
3: {'person': ([0, 0, 13, 13], 0.71), 'dog': ([0, 0, 12, 12], 0.72)}}
self.expectedIntuitiveOutDict = \
{0: {'person': 'person'},
1: {'person': 'person', 'dog': ''},
2: {'person': 'person', 'dog': ''},
3: {'person': 'person', 'dog': 'dog'}}
self.expectedIntuitiveMetric = 0.714285714286
self.actualIntuitiveMetric, self.actualIntuitiveOutDict = evaluate_intuitive(self.groundTruth, self.predicted, outputMatching=True)
# hand calulcation of MAP@2
# AP = sum(P@i * R@i - R@(i-1)) calculating for each separate set of overlapping boxes
# frame 0: (1*1) + (1/2 * 0) = 1
# : 0
# frame 1: 1
# : 0
# frame 2: 0.5
# frame 3: 1
# mAP = 3.5/6 = 0.5833
self.expectedMAP = 0.5833
self.actualMAP = evaluate_mAP(self.groundTruth, self.predicted, k=2, overlapThreshold=0.5)
def produce(self, ip): """ Computer mAP (mean average precision) by comparing ground truth to prediction :param ip: ground truth list, prediction list ground truth is list of lists where each element is (frame, label, [xmin, ymin, xmax, ymax], [attributes]) prediction is list of lists where each element is (frame, label, [xmin, ymin, xmax, ymax], [attributes], confidence) Note: Even if there are no attributes, an empty list will be received from the previous chain object Note: input of the occluded, generated, and lost flags is not currently accepted :return: mAP """ actual, predicted = ip assert len(actual) == len(predicted) # form ground truth list in desired format -- if any label or attribute matches the desired logic, mark the # create a summary output label; if not mark the label as absent gt = [] for frame in actual: inList = [] for row in frame: words2Check = [row[1]] + row[3] nLabel = '' for word in self.checkWords: if word in words2Check: nLabel += word if self.checkLogic == 'and': if word not in words2Check: nLabel = 'absent' break if not nLabel: nLabel = 'absent' inList.append(tuple([nLabel] + [row[2]])) gt.append(inList) # same process for predictions but include confidence score pd = [] for frame in predicted: inList = [] for row in frame: words2Check = [row[1]] + row[3] nLabel = '' for word in self.checkWords: if word in words2Check: nLabel += word if self.checkLogic == 'and': if word not in words2Check: nLabel = 'absent' break if not nLabel: nLabel = 'absent' inList.append(tuple([nLabel] + [row[2]] + [row[-1]])) pd.append(inList) mAP = evaluate_mAP(gt, pd) yield mAP
def produce(self, ip):
"""
Computer mAP (mean average precision) by comparing ground truth to prediction
:param ip: ground truth list, prediction list
ground truth is list of lists where each element is (frame, label, [xmin, ymin, xmax, ymax], [attributes])
prediction is list of lists where each element is (frame, label, [xmin, ymin, xmax, ymax], [attributes], confidence)
Note: Even if there are no attributes, an empty list will be received from the previous chain object
Note: input of the occluded, generated, and lost flags is not currently accepted
:return: mAP
"""
actual, predicted = ip
assert len(actual) == len(predicted)
# form ground truth list in desired format -- if any label or attribute matches the desired logic, mark the
# create a summary output label; if not mark the label as absent
gt = []
for frame in actual:
inList = []
for row in frame:
words2Check = [row[1]] + row[3]
nLabel = ''
for word in self.checkWords:
if word in words2Check:
nLabel += word
if self.checkLogic == 'and':
if word not in words2Check:
nLabel = 'absent'
break
if not nLabel:
nLabel = 'absent'
inList.append(tuple([nLabel] + [row[2]]))
gt.append(inList)
# same process for predictions but include confidence score
pd = []
for frame in predicted:
inList = []
for row in frame:
words2Check = [row[1]] + row[3]
nLabel = ''
for word in self.checkWords:
if word in words2Check:
nLabel += word
if self.checkLogic == 'and':
if word not in words2Check:
nLabel = 'absent'
break
if not nLabel:
nLabel = 'absent'
inList.append(tuple([nLabel] + [row[2]] + [row[-1]]))
pd.append(inList)
mAP = evaluate_mAP(gt, pd)
yield mAP
def setUp(self):
self.groundTruth = \
{0: {'person': [0, 0, 10, 10]},
1: {'person': [0, 0, 11, 11], 'dog': [20, 20, 30, 30]},
2: {'person': [0, 0, 12, 12], 'dog': [0, 0, 13, 13]},
3: {'dog': [0, 0, 12, 12], 'person': [0, 0, 13, 13]}}
self.predicted = \
{0: {'person': ([0, 0, 10, 10], 0.70), 'dog': ([6, 6, 11, 11], 0.75), 'cat': ([0, 0, 10, 10], 0.65)},
1: {'person': ([0, 0, 11, 11], 0.71)},
2: {'person': ([0, 0, 12, 12], 0.72)},
3: {'person': ([0, 0, 13, 13], 0.71), 'dog': ([0, 0, 12, 12], 0.72)}}
self.expectedIntuitiveOutDict = \
{0: {'person': 'person'},
1: {'person': 'person', 'dog': ''},
2: {'person': 'person', 'dog': ''},
3: {'person': 'person', 'dog': 'dog'}}
self.expectedIntuitiveMetric = 0.714285714286
self.actualIntuitiveMetric, self.actualIntuitiveOutDict = evaluate_intuitive(
self.groundTruth, self.predicted, outputMatching=True)
# hand calulcation of MAP@2
# AP = sum(P@i * R@i - R@(i-1)) calculating for each separate set of overlapping boxes
# frame 0: (1*1) + (1/2 * 0) = 1
# : 0
# frame 1: 1
# : 0
# frame 2: 0.5
# frame 3: 1
# mAP = 3.5/6 = 0.5833
self.expectedMAP = 0.5833
self.actualMAP = evaluate_mAP(self.groundTruth,
self.predicted,
k=2,
overlapThreshold=0.5)
