def scoringErrors(coco_analyze, oks, imgs_info, saveDir):
loc_dir = saveDir + '/scoring_errors'
if not os.path.exists(loc_dir):
os.makedirs(loc_dir)
f = open('%s/std_out.txt' % loc_dir, 'w')
f.write("Running Analysis: [Scoring Errors]\n\n")
tic = time.time()
paths = {}
# set parameters for the scoring errors analysis
coco_analyze.params.areaRng = [[32**2, 1e5**2]]
coco_analyze.params.areaRngLbl = ['all']
coco_analyze.params.oksThrs = [oks]
coco_analyze.cocoEval.params.useGtIgnore = 0
coco_analyze.cocoEval.params.gtIgnoreIds = []
coco_analyze.analyze(check_kpts=False,
check_scores=True,
check_bckgd=False)
coco_analyze.summarize(makeplots=True,
savedir=loc_dir,
team_name='scoring')
paths['opt_score_prc'] = \
'%s/error_prc_[scoring][%d][%s][%d].pdf'%(loc_dir, int(oks*100),
coco_analyze.params.areaRngLbl[0],
coco_analyze.params.maxDets[0])
corrected_dts = coco_analyze.corrected_dts['all']
# dictionary of all corrected detections grouped by image id
all_dts = {}
for d in coco_analyze.corrected_dts['all']:
if d['image_id'] not in all_dts:
all_dts[d['image_id']] = {}
all_dts[d['image_id']]['dts'] = [d]
else:
all_dts[d['image_id']]['dts'].append(d)
subopt_order_images = []
all_gts = {}
all_dtgt_oks = {}
for imgId in imgs_info:
if imgId in all_dts:
dts = all_dts[imgId]['dts']
all_dts[imgId]['score'] = np.argsort([-d['score'] for d in dts],
kind='mergesort')
all_dts[imgId]['opt_score'] = np.argsort(
[-d['opt_score'] for d in dts], kind='mergesort')
if list(all_dts[imgId]['score']) != list(
all_dts[imgId]['opt_score']):
subopt_order_images.append(imgId)
else:
dts = []
gts = coco_analyze.cocoGt.loadAnns(
coco_analyze.cocoGt.getAnnIds(imgIds=imgId))
not_ignore_gts = []
for g in gts:
# gt ignores are discarded
if g['ignore'] or (g['area'] < coco_analyze.params.areaRng[0][0] or
g['area'] > coco_analyze.params.areaRng[0][1]):
g['_ignore'] = 1
else:
g['_ignore'] = 0
not_ignore_gts.append(g)
# compute the oks matrix between the dts and gts of each image
image_oks_mat = utilities.compute_oks(dts, not_ignore_gts)
if len(image_oks_mat) == 0:
all_gts[imgId] = not_ignore_gts
all_dtgt_oks[imgId] = []
else:
# sort the ground truths by their max oks value with any detection
maxoksvals = [
-max(image_oks_mat[:, j]) for j in xrange(len(not_ignore_gts))
]
gtind = np.argsort(maxoksvals, kind='mergesort')
all_gts[imgId] = [not_ignore_gts[j] for j in gtind]
all_dtgt_oks[imgId] = image_oks_mat[:, gtind]
## check how many images have optimal score and original score with same order
perc = 100 * len(subopt_order_images) / float(len(all_dts))
f.write(
"Num. of imgs with sub-optimal detections order: [%d]/[%d] (%.2f%%).\n\n"
% (len(subopt_order_images), len(all_dts), perc))
## find scoring errors before and after rescoring
min_match_oks = .5
scoring_errors = {'score': [], 'opt_score': []}
for score_type in scoring_errors.keys():
for ind, imgId in enumerate(all_dts.keys()):
dind = all_dts[imgId][score_type]
sorted_dts = [all_dts[imgId]['dts'][i] for i in dind]
gtIds = [g['id'] for g in all_gts[imgId]]
if len(sorted_dts) * len(gtIds) == 0: continue
used_dts = []
for gind, gt in enumerate(all_gts[imgId]):
assert (gt['_ignore'] == 0)
oks = all_dtgt_oks[imgId][dind, gind]
dts_with_oks = np.where(oks >= min_match_oks)[0]
# remove the matched dts
dts_available = [(i,sorted_dts[i]['id'],oks[i],sorted_dts[i][score_type]) \
for i in dts_with_oks if sorted_dts[i]['id'] not in used_dts]
if len(dts_available) == 0: break
max_oks_dt = np.argmax([d[2] for d in dts_available])
used_dts.append(dts_available[max_oks_dt][1])
if len(dts_available) > 1:
# check for scoring error
max_score_dt = np.argmax([d[3] for d in dts_available])
if max_score_dt != max_oks_dt:
# this is a scoring error
error = {}
error['gt'] = gt
error['imgId'] = imgId
error['matched_dt'] = sorted_dts[
dts_available[max_score_dt][0]]
error['top_match_dt'] = sorted_dts[
dts_available[max_oks_dt][0]]
error['high_oks'] = dts_available[max_oks_dt][2]
error['low_oks'] = dts_available[max_score_dt][2]
scoring_errors[score_type].append(error)
f.write("Num. of scoring errors:\n")
f.write(" - Original Score: %d\n" % len(scoring_errors['score']))
f.write(" - Optimal Score: %d\n" % len(scoring_errors['opt_score']))
f.write("\nMost relevant scoring errors:\n")
## print the top scoring errors of the algorithm
ori_scoring_errors = scoring_errors['score']
ori_scoring_errors.sort(
key=lambda k: -np.sqrt((k['matched_dt']['score'] - k['top_match_dt'][
'score']) * (k['high_oks'] - k['low_oks'])))
for ind, err in enumerate(ori_scoring_errors[0:12]):
relevance = np.sqrt(
(err['matched_dt']['score'] - err['top_match_dt']['score']) *
(err['high_oks'] - err['low_oks']))
f.write("================================================\n")
f.write("- gt id: [%d]\n" % err['gt']['id'])
f.write("- dt id, high score, low oks: [%d][%.3f][%.3f]\n" %
(err['matched_dt']['id'], err['matched_dt']['score'],
err['low_oks']))
f.write("- dt id, low score, high oks: [%d][%.3f][%.3f]\n" %
(err['top_match_dt']['id'], err['top_match_dt']['score'],
err['high_oks']))
f.write("- Relevance: [%.3f]\n\n" % relevance)
name = 'score_err_%d_high_score' % ind
paths[name] = '%s/%s.pdf' % (loc_dir, name)
utilities.show_dets([err['matched_dt']], [err['gt']],
imgs_info[err['imgId']],
save_path=paths[name])
name = 'score_err_%d_high_oks' % ind
paths[name] = '%s/%s.pdf' % (loc_dir, name)
utilities.show_dets([err['top_match_dt']], [err['gt']],
imgs_info[err['imgId']],
save_path=paths[name])
# for all the images with dts and gts compute the following quantities
# - number of dts with oks > min_match_oks for each gt
# - histogram of oks for the detection with highest oks
# - histogram of oks for all the other detections
# - histogram of original/optimal scores for the detection with highest oks
# - histogram of original/optimal scores for all the other detections
num_dts_high_oks = []
high_oks_dt_oks_hist = []
other_dt_oks_hist = []
high_oks_dt_ori_score_hist = []
other_dt_ori_score_hist = []
high_oks_dt_opt_score_hist = []
other_dt_opt_score_hist = []
for ind, imgId in enumerate(all_dts.keys()):
dts = [(d['id'], d['score'], d['opt_score'])
for d in all_dts[imgId]['dts']]
gtIds = [g['id'] for g in all_gts[imgId]]
if len(dts) * len(gtIds) == 0: continue
for gind, gt in enumerate(all_gts[imgId]):
assert (gt['_ignore'] == 0)
dts_oks = all_dtgt_oks[imgId][:, gind]
dts_high_oks_i = np.where(dts_oks > .1)[0]
num_dts_high_oks.append(len(dts_high_oks_i))
if len(dts_high_oks_i) >= 2:
# study the case where multiple detections have high oks
# add the oks of the detections to the histogram of oks
oks_vals = sorted([(dts_oks[i], dts[i])
for i in dts_high_oks_i],
key=lambda k: -k[0])
high_oks_dt_oks_hist.append(oks_vals[0][0])
other_dt_oks_hist.extend([k[0] for k in oks_vals[1:]])
high_oks_dt_ori_score_hist.append(oks_vals[0][1][1])
other_dt_ori_score_hist.extend([k[1][1] for k in oks_vals[1:]])
high_oks_dt_opt_score_hist.append(oks_vals[0][1][2])
other_dt_opt_score_hist.extend([k[1][2] for k in oks_vals[1:]])
fig, ax = plt.subplots(figsize=(10, 10))
ax.set_facecolor('lightgray')
plt.hist(num_dts_high_oks,
bins=[i - .5 for i in xrange(max(num_dts_high_oks) + 1)],
color='green')
plt.grid()
plt.xticks([i for i in xrange(max(num_dts_high_oks))])
plt.title('Histogram of Detection Redundancy', fontsize=20)
plt.xlabel('Number of Detections with OKS > .1', fontsize=20)
plt.ylabel('Number of Ground Truth Instances', fontsize=20)
path = '%s/num_dts_high_oks.pdf' % loc_dir
paths['num_dts_high_oks'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10, 10))
y1, binEdges = np.histogram(high_oks_dt_ori_score_hist, bins=19)
bincenters1 = 0.5 * (binEdges[1:] + binEdges[:-1])
ax.plot(bincenters1,
y1,
'-',
linewidth=3,
c='b',
label='Max OKS Detection')
min_val1 = min(bincenters1)
max_val1 = max(bincenters1)
y2, binEdges = np.histogram(other_dt_ori_score_hist, bins=19)
bincenters2 = 0.5 * (binEdges[1:] + binEdges[:-1])
ax.plot(bincenters2,
y2,
'--',
linewidth=3,
c='b',
label='Lower OKS Detection(s)')
min_val2 = min(bincenters2)
max_val2 = max(bincenters2)
min_val = min(min_val1, min_val2)
max_val = max(max_val1, max_val2)
overlapbins = [min(x, y) for x, y in zip(y1, y2)]
width = (max_val - min_val) / 20.
ax.bar(np.linspace(min_val, max_val, 19),
overlapbins,
color='red',
alpha=.65,
width=width,
align='center')
plt.grid()
plt.xlim([
min_val - (max_val - min_val) / 20.,
max_val + (max_val - min_val) / 20.
])
plt.grid()
plt.legend(loc='upper center', fontsize=20)
plt.title('Histogram of Original Detection Scores', fontsize=20)
plt.xlabel('Original Confidence Score', fontsize=20)
plt.ylabel('Number of Detections', fontsize=20)
path = '%s/dts_ori_score_hist.pdf' % loc_dir
paths['dts_ori_score_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10, 10))
y1, binEdges = np.histogram(high_oks_dt_opt_score_hist, bins=19)
bincenters1 = 0.5 * (binEdges[1:] + binEdges[:-1])
ax.plot(bincenters1,
y1,
'-',
linewidth=3,
c='b',
label='Max OKS Detection')
min_val1 = min(bincenters1)
max_val1 = max(bincenters1)
y2, binEdges = np.histogram(other_dt_opt_score_hist, bins=19)
bincenters2 = 0.5 * (binEdges[1:] + binEdges[:-1])
ax.plot(bincenters2,
y2,
'--',
linewidth=3,
c='b',
label='Lower OKS Detection(s)')
min_val2 = min(bincenters2)
max_val2 = max(bincenters2)
min_val = min(min_val1, min_val2)
max_val = max(max_val1, max_val2)
overlapbins = [min(x, y) for x, y in zip(y1, y2)]
width = (max_val - min_val) / 20.
ax.bar(np.linspace(min_val, max_val, 19),
overlapbins,
color='red',
alpha=.65,
width=width,
align='center')
plt.grid()
plt.xlim([
min_val - (max_val - min_val) / 20.,
max_val + (max_val - min_val) / 20.
])
plt.grid()
plt.legend(loc='upper center', fontsize=20)
plt.title('Histogram of Optimal Detection Scores', fontsize=20)
plt.xlabel('Optimal Confidence Score', fontsize=20)
plt.ylabel('Number of Detections', fontsize=20)
path = '%s/dts_opt_score_hist.pdf' % loc_dir
paths['dts_opt_score_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
f.write("\nDone, (t=%.2fs)." % (time.time() - tic))
f.close()
return paths
def backgroundFalsePosErrors(coco_analyze, imgs_info, saveDir):
loc_dir = saveDir + '/background_errors/false_positives'
if not os.path.exists(loc_dir):
os.makedirs(loc_dir)
f = open('%s/std_out.txt' % loc_dir, 'w')
f.write("Running Analysis: [Background False Positives]\n\n")
tic = time.time()
paths = {}
oksThrs = [.5, .55, .6, .65, .7, .75, .8, .85, .9, .95]
areaRngs = [[32**2, 1e5**2]]
areaRngLbls = ['all']
coco_analyze.params.areaRng = areaRngs
coco_analyze.params.areaRngLbl = areaRngLbls
coco_analyze.params.oksThrs = oksThrs
coco_analyze.cocoEval.params.useGtIgnore = 0
coco_analyze.cocoEval.params.gtIgnoreIds = []
coco_analyze.analyze(check_kpts=False,
check_scores=False,
check_bckgd=True)
badFalsePos = coco_analyze.false_pos_dts['all', '0.5']
for tind, t in enumerate(coco_analyze.params.oksThrs):
badFalsePos = badFalsePos & coco_analyze.false_pos_dts['all', str(t)]
fp_dts = [
d for d in coco_analyze.corrected_dts['all'] if d['id'] in badFalsePos
]
f.write("Num. detections: [%d]\n" % len(coco_analyze.corrected_dts['all']))
for oks in oksThrs:
f.write("OKS thresh: [%f]\n" % oks)
f.write(" - Matches: [%d]\n" %
len(coco_analyze.bckgd_err_matches[areaRngLbls[0],
str(oks), 'dts']))
f.write(" - Bckgd. FP: [%d]\n" %
len(coco_analyze.false_pos_dts[areaRngLbls[0],
str(oks)]))
sorted_fps = sorted(fp_dts, key=lambda k: -k['score'])
show_fp = sorted_fps[0:4] + sorted_fps[-4:]
f.write("\nBackground False Positive Errors:\n")
for tind, t in enumerate(show_fp):
name = 'bckd_false_pos_%d' % tind
paths[name] = "%s/%s.pdf" % (loc_dir, name)
f.write("Image_id, detection_id, score: [%d][%d][%.3f]\n" %
(t['image_id'], t['id'], t['score']))
utilities.show_dets([t], [], imgs_info[t['image_id']], paths[name])
a = [d['score'] for d in coco_analyze.corrected_dts['all']]
p_20 = np.percentile(a, 20)
p_40 = np.percentile(a, 40)
p_60 = np.percentile(a, 60)
p_80 = np.percentile(a, 80)
f.write("\nPercentiles of the scores of all Detections:\n")
f.write(
" - 20th perc. score:[%.3f]; num. dts:[%d]\n" %
(p_20,
len([
d for d in coco_analyze.corrected_dts['all'] if d['score'] <= p_20
])))
f.write(
" - 40th perc. score:[%.3f]; num. dts:[%d]\n" %
(p_40,
len([
d for d in coco_analyze.corrected_dts['all'] if d['score'] <= p_40
])))
f.write(
" - 60th perc. score:[%.3f]; num. dts:[%d]\n" %
(p_60,
len([
d for d in coco_analyze.corrected_dts['all'] if d['score'] <= p_60
])))
f.write(
" - 80th perc. score:[%.3f]; num. dts:[%d]\n" %
(p_80,
len([
d for d in coco_analyze.corrected_dts['all'] if d['score'] <= p_80
])))
fig, ax = plt.subplots(figsize=(10, 10))
ax.set_axis_bgcolor('lightgray')
bins = [min(a), p_20, p_40, p_60, p_80, max(a)]
plt.hist([d['score'] for d in sorted_fps], bins=bins, color='b')
plt.xticks(bins, ['', '20th%', '40th%', '60th%', '80th%', ''],
rotation='vertical')
plt.grid()
plt.xlabel('All Detection Score Percentiles', fontsize=20)
plt.title('Histogram of False Positive Scores', fontsize=20)
path = "%s/bckd_false_pos_scores_histogram.pdf" % (loc_dir)
paths['false_pos_score_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
high_score_fp = [d for d in sorted_fps if p_80 <= d['score']]
max_height = max([d['bbox'][3] for d in high_score_fp])
min_height = min([d['bbox'][3] for d in high_score_fp])
max_width = max([d['bbox'][2] for d in high_score_fp])
min_width = min([d['bbox'][2] for d in high_score_fp])
f.write("\nBackground False Positives Bounding Box Dimenstions:\n")
f.write(" - Min width: [%d]\n" % min_width)
f.write(" - Max width: [%d]\n" % max_width)
f.write(" - Min height: [%d]\n" % min_height)
f.write(" - Max height: [%d]\n" % max_height)
ar_pic = np.zeros((int(max_height) + 1, int(max_width) + 1))
ar_pic_2 = np.zeros((30, 30))
ar_bins = range(10) + range(10, 100, 10) + range(100, 1000, 100) + [1000]
ar_pic_3 = np.zeros((10, 10))
ar_bins_3 = [np.power(2, x) for x in xrange(11)]
areaRngs = [[0, 32**2], [32**2, 64**2], [64**2, 96**2], [96**2, 128**2],
[128**2, 1e5**2]]
areaRngLbls = ['small', 'medium', 'large', 'xlarge', 'xxlarge']
small = 0
medium = 0
large = 0
xlarge = 0
xxlarge = 0
num_people_ranges = [[0, 0], [1, 1], [2, 4], [5, 8], [9, 100]]
num_people_labels = ['none', 'one', 'small grp.', 'large grp.', 'crowd']
no_people = 0
one = 0
small_grp = 0
large_grp = 0
crowd = 0
for t in high_score_fp:
t_width = int(t['bbox'][2])
t_height = int(t['bbox'][3])
ar_pic[0:t_height, 0:t_width] += 1
if t_width < 1024 and t_height < 1024:
col = [
i for i in xrange(len(ar_bins) - 1)
if ar_bins[i] < t_width < ar_bins[i + 1]
]
row = [
i for i in xrange(len(ar_bins) - 1)
if ar_bins[i] < t_height < ar_bins[i + 1]
]
ar_pic_2[row, col] += 1
col = [
i for i in xrange(len(ar_bins_3) - 1)
if ar_bins_3[i] < t_width < ar_bins_3[i + 1]
]
row = [
i for i in xrange(len(ar_bins_3) - 1)
if ar_bins_3[i] < t_height < ar_bins_3[i + 1]
]
ar_pic_3[row, col] += 1
else:
print "False Positive bbox has a side larger than 1024 pixels."
print "Change lists ar_bins_2 and ar_bins_3 to include larger bins."
assert (False)
area = t_width * t_height * .5
if areaRngs[0][0] <= area < areaRngs[0][1]:
small += 1
elif areaRngs[1][0] <= area < areaRngs[1][1]:
medium += 1
elif areaRngs[2][0] <= area < areaRngs[2][1]:
large += 1
elif areaRngs[3][0] <= area < areaRngs[3][1]:
xlarge += 1
elif areaRngs[4][0] <= area < areaRngs[4][1]:
xxlarge += 1
anns = coco_analyze.cocoGt.loadAnns(
coco_analyze.cocoGt.getAnnIds(t['image_id']))
iscrowd = [ann['iscrowd'] for ann in anns]
num_people = len(anns) if sum(iscrowd) == 0 else 100
if num_people_ranges[0][0] <= num_people <= num_people_ranges[0][1]:
no_people += 1
elif num_people_ranges[1][0] <= num_people <= num_people_ranges[1][1]:
one += 1
elif num_people_ranges[2][0] <= num_people <= num_people_ranges[2][1]:
small_grp += 1
elif num_people_ranges[3][0] <= num_people <= num_people_ranges[3][1]:
large_grp += 1
elif num_people_ranges[4][0] <= num_people <= num_people_ranges[4][1]:
crowd += 1
f.write("\nNumber of people in images with Background False Positives:\n")
f.write(" - No people: [%d]\n" % no_people)
f.write(" - One person: [%d]\n" % one)
f.write(" - Small group (2-4): [%d]\n" % small_grp)
f.write(" - Large Group (5-8): [%d]\n" % large_grp)
f.write(" - Crowd (>=9): [%d]\n" % crowd)
f.write("\nArea size (in pixels) of Background False Positives:\n")
f.write(" - Small (%d,%d): [%d]\n" %
(areaRngs[0][0], areaRngs[0][1], small))
f.write(" - Medium (%d,%d): [%d]\n" %
(areaRngs[1][0], areaRngs[1][1], medium))
f.write(" - Large (%d,%d): [%d]\n" %
(areaRngs[2][0], areaRngs[2][1], large))
f.write(" - X-Large (%d,%d): [%d]\n" %
(areaRngs[3][0], areaRngs[3][1], xlarge))
f.write(" - XX-Large (%d,%d): [%d]\n" %
(areaRngs[4][0], areaRngs[4][1], xxlarge))
plt.figure(figsize=(10, 10))
plt.imshow(ar_pic, origin='lower')
plt.colorbar()
plt.title('BBox Aspect Ratio', fontsize=20)
plt.xlabel('Width (px)', fontsize=20)
plt.ylabel('Height (px)', fontsize=20)
path = "%s/bckd_false_pos_bbox_aspect_ratio.pdf" % (loc_dir)
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10, 10))
plt.imshow(ar_pic_2, origin='lower')
plt.xticks(xrange(1,
len(ar_bins) + 1), ["%d" % (x) for x in ar_bins],
rotation='vertical')
plt.yticks(xrange(1, len(ar_bins) + 1), ["%d" % (x) for x in ar_bins])
plt.colorbar()
plt.grid()
plt.title('BBox Aspect Ratio', fontsize=20)
plt.xlabel('Width (px)', fontsize=20)
plt.ylabel('Height (px)', fontsize=20)
path = "%s/bckd_false_pos_bbox_aspect_ratio_2.pdf" % (loc_dir)
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10, 10))
plt.imshow(ar_pic_3, origin='lower')
plt.xticks([-.5 + x for x in range(11)], ["%d" % (x) for x in ar_bins_3])
plt.yticks([-.5 + x for x in range(11)], ["%d" % (x) for x in ar_bins_3])
plt.colorbar()
plt.grid()
plt.title('BBox Aspect Ratio', fontsize=20)
plt.xlabel('Width (px)', fontsize=20)
plt.ylabel('Height (px)', fontsize=20)
path = "%s/bckd_false_pos_bbox_aspect_ratio_3.pdf" % (loc_dir)
paths['false_pos_bbox_ar'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10, 10))
ax.set_axis_bgcolor('lightgray')
plt.bar(xrange(5), [small, medium, large, xlarge, xxlarge],
color='g',
align='center')
plt.xticks(xrange(5), areaRngLbls)
plt.grid()
plt.title('Histogram of Area Size', fontsize=20)
path = "%s/bckd_false_pos_area_histogram.pdf" % (loc_dir)
paths['false_pos_bbox_area_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10, 10))
ax.set_axis_bgcolor('lightgray')
plt.bar(xrange(5), [no_people, one, small_grp, large_grp, crowd],
color='g',
align='center')
plt.xticks(xrange(5), num_people_labels)
plt.grid()
plt.title('Histogram of Num. of People in Images', fontsize=20)
path = "%s/bckd_false_pos_num_people_histogram.pdf" % (loc_dir)
paths['false_pos_num_ppl_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
f.write("\nDone, (t=%.2fs)." % (time.time() - tic))
f.close()
return paths
def backgroundFalseNegErrors( coco_analyze, imgs_info, saveDir ):
loc_dir = saveDir + '/background_errors/false_negatives'
if not os.path.exists(loc_dir):
os.makedirs(loc_dir)
f = open('%s/std_out.txt'%loc_dir, 'w')
f.write("Running Analysis: [False Negatives]\n\n")
tic = time.time()
paths = {}
oksThrs = [.5,.55,.6,.65,.7,.75,.8,.85,.9,.95]
areaRngs = [[32**2,1e5**2]]
areaRngLbls = ['all']
coco_analyze.params.areaRng = areaRngs
coco_analyze.params.areaRngLbl = areaRngLbls
coco_analyze.params.oksThrs = oksThrs
coco_analyze.cocoEval.params.useGtIgnore = 0
coco_analyze.cocoEval.params.gtIgnoreIds = []
coco_analyze.analyze(check_kpts=False, check_scores=False, check_bckgd=True)
badFalseNeg = coco_analyze.false_neg_gts['all',str(.5)]
for tind, t in enumerate(coco_analyze.params.oksThrs):
badFalseNeg = badFalseNeg & coco_analyze.false_neg_gts['all',str(t)]
# bad false negatives are those that are false negatives at all oks thresholds
fn_gts = [coco_analyze.cocoGt.loadAnns(b)[0] for b in badFalseNeg]
f.write("Num. annotations: [%d]\n"%len(coco_analyze.cocoGt.loadAnns(coco_analyze.cocoGt.getAnnIds())))
for oks in oksThrs:
f.write("OKS thresh: [%f]\n"%oks)
f.write(" - Matches: [%d]\n"%len(coco_analyze.bckgd_err_matches[areaRngLbls[0], str(oks), 'gts']))
f.write(" - Bckgd. FN: [%d]\n"%len(coco_analyze.false_neg_gts[areaRngLbls[0],str(oks)]))
sorted_fns = sorted(fn_gts, key=lambda k: -k['num_keypoints'])
sorted_fns = [fff for fff in sorted_fns if fff['num_keypoints']>0]
show_fn = sorted_fns[0:4] + sorted_fns[-4:]
f.write("\nBackground False Negative Errors:\n")
for tind, t in enumerate(show_fn):
name = 'bckd_false_neg_%d'%tind
paths[name] = "%s/%s.pdf"%(loc_dir,name)
f.write("Image_id, ground_truth id, num_keypoints: [%d][%d][%d]\n"%(t['image_id'],t['id'],t['num_keypoints']))
utilities.show_dets([],[t],imgs_info[t['image_id']],paths[name])
max_height = max([d['bbox'][3] for d in fn_gts])
min_height = min([d['bbox'][3] for d in fn_gts])
max_width = max([d['bbox'][2] for d in fn_gts])
min_width = min([d['bbox'][2] for d in fn_gts])
f.write("\nBackground False Negatives Bounding Box Dimenstions:\n")
f.write(" - Min width: [%d]\n"%min_width)
f.write(" - Max width: [%d]\n"%max_width)
f.write(" - Min height: [%d]\n"%min_height)
f.write(" - Max height: [%d]\n"%max_height)
ar_pic = np.zeros((int(max_height)+1,int(max_width)+1))
ar_pic_2 = np.zeros((30,30))
ar_bins = range(10)+range(10,100,10)+range(100,1000,100)+[1000]
ar_pic_3 = np.zeros((10,10))
ar_bins_3 = [np.power(2,x) for x in xrange(11)]
num_fn_keypoints = {}
areaRngs = [[0, 32 ** 2],[32 ** 2, 64 ** 2],[64 ** 2, 96 ** 2],[96 ** 2, 128 ** 2],[128 ** 2, 1e5 ** 2]]
areaRngLbls = ['small','medium','large','xlarge','xxlarge']
small = 0; medium = 0; large = 0; xlarge = 0; xxlarge = 0
num_people_ranges = [[0,0],[1,1],[2,4],[5,8],[9,100]]
num_people_labels = ['none','one','small grp.','large grp.', 'crowd']
no_people = 0; one = 0; small_grp = 0; large_grp = 0; crowd = 0
segm_heatmap = np.zeros((128,128))
for i,b in enumerate(fn_gts):
if b['num_keypoints'] in num_fn_keypoints:
num_fn_keypoints[b['num_keypoints']] += 1
else:
num_fn_keypoints[b['num_keypoints']] = 1
if b['num_keypoints']==0: continue
b_width = int(b['bbox'][2])
b_height = int(b['bbox'][3])
ar_pic[0:b_height,0:b_width] += 1
if b_width < 1024 and b_height < 1024:
col = [i for i in xrange(len(ar_bins)-1) if ar_bins[i]<b_width<ar_bins[i+1]]
row = [i for i in xrange(len(ar_bins)-1) if ar_bins[i]<b_height<ar_bins[i+1]]
ar_pic_2[row,col] += 1
col = [i for i in xrange(len(ar_bins_3)-1) if ar_bins_3[i]<b_width<ar_bins_3[i+1]]
row = [i for i in xrange(len(ar_bins_3)-1) if ar_bins_3[i]<b_height<ar_bins_3[i+1]]
ar_pic_3[row,col] += 1
else:
print "False Positive bbox has a side larger than 1024 pixels."
print "Change lists ar_bins_2 and ar_bins_3 to include larger bins."
assert(False)
area = b_width * b_height * .5
if areaRngs[0][0] <= area < areaRngs[0][1]:
small += 1
elif areaRngs[1][0] <= area < areaRngs[1][1]:
medium += 1
elif areaRngs[2][0] <= area < areaRngs[2][1]:
large += 1
elif areaRngs[3][0] <= area < areaRngs[3][1]:
xlarge += 1
elif areaRngs[4][0] <= area < areaRngs[4][1]:
xxlarge += 1
anns = coco_analyze.cocoGt.loadAnns(coco_analyze.cocoGt.getAnnIds(b['image_id']))
iscrowd = [ann['iscrowd'] for ann in anns]
num_people = len(anns) if sum(iscrowd)==0 else 100
if num_people_ranges[0][0] <= num_people <= num_people_ranges[0][1]:
no_people += 1
elif num_people_ranges[1][0] <= num_people <= num_people_ranges[1][1]:
one += 1
elif num_people_ranges[2][0] <= num_people <= num_people_ranges[2][1]:
small_grp += 1
elif num_people_ranges[3][0] <= num_people <= num_people_ranges[3][1]:
large_grp += 1
elif num_people_ranges[4][0] <= num_people <= num_people_ranges[4][1]:
crowd += 1
if b['iscrowd']==1: continue
nx, ny = imgs_info[b['image_id']]['width'],imgs_info[b['image_id']]['height']
the_mask = np.zeros((ny,nx))
# Create vertex coordinates for each grid cell...
# (<0,0> is at the top left of the grid in this system)
x, y = np.meshgrid(np.arange(nx), np.arange(ny))
x, y = x.flatten(), y.flatten()
points = np.vstack((x,y)).T
for poly_verts in b['segmentation']:
path = mplPath.Path(np.array([[x,y] for x,y in zip(poly_verts[0::2],poly_verts[1::2])]))
grid = path.contains_points(points)
grid = grid.reshape((ny,nx))
the_mask += np.array(grid, dtype=int)
segm_heatmap += imresize(the_mask,(128,128))
fig = plt.figure(figsize=(10,10))
ax = plt.subplot(111)
ax.imshow(segm_heatmap)
path = "%s/bckd_false_neg_heatmaps.pdf"%(loc_dir)
paths['false_neg_hm'] = path
plt.axis('off')
plt.savefig(path, bbox_inches='tight')
plt.close()
f.write("\nNumber of people in images with Background False Negatives:\n")
f.write(" - No people: [%d]\n"%no_people)
f.write(" - One person: [%d]\n"%one)
f.write(" - Small group (2-4): [%d]\n"%small_grp)
f.write(" - Large Group (5-8): [%d]\n"%large_grp)
f.write(" - Crowd (>=9): [%d]\n"%crowd)
f.write("\nArea size (in pixels) of Background False Negatives:\n")
f.write(" - Small (%d,%d): [%d]\n"%(areaRngs[0][0],areaRngs[0][1],small))
f.write(" - Medium (%d,%d): [%d]\n"%(areaRngs[1][0],areaRngs[1][1],medium))
f.write(" - Large (%d,%d): [%d]\n"%(areaRngs[2][0],areaRngs[2][1],large))
f.write(" - X-Large (%d,%d): [%d]\n"%(areaRngs[3][0],areaRngs[3][1],xlarge))
f.write(" - XX-Large (%d,%d): [%d]\n"%(areaRngs[4][0],areaRngs[4][1],xxlarge))
f.write("\nNumber of visible keypoints for Background False Negatives:\n")
for k in num_fn_keypoints.keys():
if k == 0: continue
f.write(" - [%d] kpts: [%d] False Neg.\n"%(k,num_fn_keypoints[k]))
plt.figure(figsize=(10,10))
plt.imshow(ar_pic,origin='lower')
plt.colorbar()
plt.title('BBox Aspect Ratio',fontsize=20)
plt.xlabel('Width (px)',fontsize=20)
plt.ylabel('Height (px)',fontsize=20)
path = "%s/bckd_false_neg_bbox_aspect_ratio.pdf"%(loc_dir)
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10,10))
plt.imshow(ar_pic_2,origin='lower')
plt.xticks(xrange(1,len(ar_bins)+1),["%d"%(x) for x in ar_bins],rotation='vertical')
plt.yticks(xrange(1,len(ar_bins)+1),["%d"%(x) for x in ar_bins])
plt.colorbar()
plt.grid()
plt.title('BBox Aspect Ratio',fontsize=20)
plt.xlabel('Width (px)',fontsize=20)
plt.ylabel('Height (px)',fontsize=20)
path = "%s/bckd_false_neg_bbox_aspect_ratio_2.pdf"%(loc_dir)
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10,10))
plt.imshow(ar_pic_3,origin='lower')
plt.xticks([-.5 + x for x in range(11)],["%d"%(x) for x in ar_bins_3])
plt.yticks([-.5 + x for x in range(11)],["%d"%(x) for x in ar_bins_3])
plt.colorbar()
plt.grid()
plt.title('BBox Aspect Ratio',fontsize=20)
plt.xlabel('Width (px)',fontsize=20)
plt.ylabel('Height (px)',fontsize=20)
path = "%s/bckd_false_neg_bbox_aspect_ratio_3.pdf"%(loc_dir)
paths['false_neg_bbox_ar'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10,10))
ax.set_axis_bgcolor('lightgray')
plt.bar(xrange(5),[small,medium,large,xlarge,xxlarge],color='g',align='center')
plt.xticks(xrange(5),areaRngLbls)
plt.grid()
plt.title('Histogram of Area Size',fontsize=20)
path = "%s/bckd_false_neg_bbox_area_hist.pdf"%(loc_dir)
paths['false_neg_bbox_area_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
fig, ax = plt.subplots(figsize=(10,10))
ax.set_axis_bgcolor('lightgray')
plt.bar(xrange(5),[no_people,one,small_grp,large_grp,crowd],color='g',align='center')
plt.xticks(xrange(5),num_people_labels)
plt.grid()
plt.title('Histogram of Num. of People in Images',fontsize=20)
path = "%s/bckd_false_neg_num_people_histogram.pdf"%(loc_dir)
paths['false_neg_num_ppl_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
plt.figure(figsize=(10,10))
plt.bar([k for k in num_fn_keypoints.keys() if k!=0],[num_fn_keypoints[k] for k in num_fn_keypoints.keys() if k!= 0],align='center')
plt.title("Histogram of Number of Keypoints",fontsize=20)
path = "%s/bckd_false_neg_num_keypoints_histogram.pdf"%(loc_dir)
paths['false_neg_num_kpts_hist'] = path
plt.savefig(path, bbox_inches='tight')
plt.close()
f.write("\nDone, (t=%.2fs)."%(time.time()-tic))
f.close()
return paths