def evaluate_tracking(sequences, track_dir, gt_dir):
all_info = []
for seqname in sequences:
track_res = os.path.join(track_dir, seqname, 'res.txt')
gt_file = os.path.join(gt_dir, seqname, 'gt.txt')
assert os.path.exists(track_res) and os.path.exists(gt_file), 'Either tracking result or groundtruth directory does not exist'
trackDB = read_txt_to_struct(track_res)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
metrics, extra_info = evaluate_sequence(trackDB, gtDB, distractor_ids)
print_metrics(seqname + ' Evaluation', metrics)
all_info.append(extra_info)
all_metrics = evaluate_bm(all_info)
print_metrics('Summary Evaluation', all_metrics)
def my_main(_config):
print(_config)
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
###### PFADE ANPASSEN ##########################
results_dir = osp.join('output/tracktor/MOT17')
################################################
output_dir = osp.join(results_dir, 'eval/det_gaps')
if not osp.exists(output_dir):
os.makedirs(output_dir)
sequences_raw = [
"MOT17-13",
"MOT17-11",
"MOT17-10",
"MOT17-09",
"MOT17-05",
"MOT17-04",
"MOT17-02",
]
############### DETECTIONS ANPASSEN #############
detections = "SDP"
################################################
sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
#sequences = sequences[:1]
#tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"]
tracker = ["Tracktor", "Tracktor++", "FWT", "jCC", "MOTDT17", "MHT_DAM"]
#tracker = ["Baseline"]
# "PHD_GSDL17" does not work, error
#tracker = tracker[-4:]
results = []
det_gaps = []
for t in tracker:
print("[*] Evaluating {}".format(t))
det_gaps_coverage = []
for s in sequences:
########################################
# Get DPM / GT coverage for each track #
########################################
###### PFADE ANPASSEN ##############################################
gt_file = osp.join("data/MOT17Labels", "train", s, "gt", "gt.txt")
det_file = osp.join("data/MOT17Labels", "train", s, "det",
"det.txt")
res_file = osp.join("output/tracktor/MOT17", t, s + ".txt")
####################################################################
#gtDB = read_txt_to_struct(gt_file)
#gtDB = gtDB[gtDB[:,7] == 1]
stDB = read_txt_to_struct(res_file)
gtDB = read_txt_to_struct(gt_file)
dtDB = read_txt_to_struct(det_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
print(s)
print(len(np.unique(gtDB[:, 1])))
_, M, gtDB = evaluate_new(stDB, gtDB, distractor_ids)
print(len(np.unique(gtDB[:, 1])))
gt_ids_old = np.unique(gtDB[:, 1])
# reaload gtDB as entries not found are deleted
gtDB = read_txt_to_struct(gt_file)
# filter out so that confidence and id = 1
gtDB = gtDB[gtDB[:, 7] == 1]
gtDB = gtDB[gtDB[:, 6] == 1]
gt_frames = np.unique(gtDB[:, 0])
#st_ids = np.unique(stDB[:, 1])
#gt_ids = np.unique(gtDB[:, 1])
#dt_ids = np.unique(dtDB[:, 1]) # id always -1
f_gt = len(gt_frames)
#n_gt = len(gt_ids)
#n_st = len(st_ids)
gt_inds = [{} for i in range(f_gt)]
#st_inds = [{} for i in range(f_gt)]
dt_inds = [{} for i in range(f_gt)]
D = [{} for i in range(f_gt)] #map detections to gt
m_keys = []
for v in M:
m_keys.append(list(v.keys()))
#print("new: {}".format(np.unique(gtDB[:, 1])))
#print("old: {}".format(gt_ids_old))
#print("M: {}".format(np.unique(m_keys)))
# hash the indices to speed up indexing
#for i in range(gtDB.shape[0]):
# frame = np.where(gt_frames == gtDB[i, 0])[0][0]
# gid = np.where(gt_ids == gtDB[i, 1])[0][0]
# gt_inds[frame][gid] = i
for i in range(gtDB.shape[0]):
frame = np.where(gt_frames == gtDB[i, 0])[0][0]
gt_id = gtDB[i, 1]
gt_inds[frame][gt_id] = i
gt_frames_list = list(gt_frames)
"""
for i in range(stDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
frame = gt_frames_list.index(stDB[i, 0])
sid = np.where(st_ids == stDB[i, 1])[0][0]
st_inds[frame][sid] = i
"""
for i in range(dtDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
frame = gt_frames_list.index(dtDB[i, 0])
# id always -1, so just make up numer, not important here
did = 0
if len(dt_inds[frame]) > 0:
did = max(dt_inds[frame].keys()) + 1
#did = np.where(dt_ids == dtDB[i, 1])[0][0]
dt_inds[frame][did] = i
# find detections <-> gt
for frame in range(f_gt):
gt_id = list(gt_inds[frame].keys())
gt_i = list(gt_inds[frame].values())
gt_boxes = gtDB[gt_i, 2:6]
#print(gt_id)
dt_id = list(dt_inds[frame].keys())
dt_i = list(dt_inds[frame].values())
dt_boxes = dtDB[dt_i, 2:6]
#print(len(dt_id))
overlaps = np.zeros((len(dt_i), len(gt_i)), dtype=float)
for i in range(len(gt_i)):
overlaps[:, i] = bbox_overlap(dt_boxes, gt_boxes[i])
matched_indices = linear_assignment(1 - overlaps)
for matched in matched_indices:
if overlaps[matched[0], matched[1]] >= 0.5:
did = dt_id[matched[0]]
gid = gt_id[matched[1]]
# gid in this case is the real gt id as it comes from the new gtDB
D[frame][gid] = did
for frame in range(f_gt):
d = D[frame]
for gt_id, _ in d.items():
gid = -1
gid_ind = np.where(gt_ids_old == gt_id)[0]
if len(gid_ind) > 0:
gid = gid_ind[0]
# every gap is only handled once, at the last detection before the gap begins
# so here we are in the middle of the gap => do nothing
if gt_id not in d.keys():
continue
# OK let's check if we have a gap in the detections
next_non_empty = -1
for j in range(frame + 1, f_gt):
if gt_id in D[j].keys():
next_non_empty = j
break
# no gap as no more detections, or no gap as next frame has detection
if next_non_empty == -1 or next_non_empty == frame + 1:
continue
# now we have a gap in detections. Let's check the gap length and track coverage!
gap_length = next_non_empty - frame - 1
count_tracked = 0
for j in range(frame + 1, next_non_empty):
if gid in M[j].keys():
count_tracked += 1
if gap_length > 115 and gap_length < 135:
print(count_tracked)
det_gaps_coverage.append(
[gap_length, count_tracked / gap_length])
# also add to gt distribution
if t == tracker[0]:
det_gaps.append(gap_length)
"""
# Now check all detections if they were tracked
for frame in range(f_gt):
matched_dets = D[frame]
for did, gid in matched_dets.items():
# not matched by tracker, get visbility
line = gt_inds[frame][gid]
vis = gtDB[line, 8]
# if matched only visibility is important
if gid in M[frame].keys():
tracked_visibilities.append(vis)
continue
else:
missed_visibilities.append(vis)
# get distance to last time tracked
last_non_empty = frame
for j in range(frame, -1, -1):
if gid in M[j].keys():
last_non_empty = j
break
next_non_empty = frame
for j in range(frame, f_gt):
if gid in M[j]:
next_non_empty = j
break
dist = min(frame-last_non_empty, next_non_empty-frame)
# check if not tracked at all ...
if dist > 0:
missed_distances.append(dist)"""
det_gaps_coverage = np.array(det_gaps_coverage)
results.append(det_gaps_coverage)
#gaps = np.unique(det_gaps_coverage[:,0])
#coverage = np.zeros(gaps.shape[0])
#for i,g in enumerate(gaps):
# covs = det_gaps_coverage[det_gaps_coverage[:,0] == g, 1]
# coverage[i] = np.mean(covs)
print(" Gaps: {}".format(len(det_gaps_coverage)))
print(" Gaps>20 with coverage>0.8: {}".format(
((det_gaps_coverage[:, 1] > 0.8) *
(det_gaps_coverage[:, 0] > 20)).sum()))
#plt.figure()
#sns.regplot(x=gaps, y=coverage, ci=None, scatter=True, scatter_kws={"s": 5}, line_kws={"linewidth":1.0}, lowess=True, label=t)
#plt.legend()
#plt.ylabel('coverage of gap by tracker')
#plt.xlim((0,x_max))
#plt.xlabel('gap length in detections')
#plt.legend()
#plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('DET_GAP_COV', detections, t)), format='pdf')
#plt.plot([0,1], [0,1], 'r-')
"""
plt.figure()
plt.hist(tracked_visibilities, bins=20, density=False)
plt.ylabel('occurence')
plt.xlabel('visibility of target')
plt.title('Boxes total: {}'.format(len(tracked_visibilities)))
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'TR_VIS')), format='pdf')
#plt.plot([0,1], [0,1], 'r-')
weights = np.zeros(missed_gaps_length.shape)
for i in range(missed_gaps_length.shape[0]):
weights[i] = 1/missed_gaps_length[i]
#weights[i] = 1
plt.figure()
#plt.xlim((0, 40))
bins = list(range(40))
plt.hist(missed_gaps_length, weights=weights, bins=bins, density=False)
plt.ylabel('occurence')
plt.xlabel('gap length in tracks')
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('MISS_DIST', t, detections)), format='pdf')
x = np.arange(1,41)
y = np.zeros(len(x))
ges_occurencies = 0
for i,b in enumerate(x):
occurencies = int((missed_gaps_length==b).sum())
occurencies = occurencies/b
ges_occurencies += occurencies
y[i] = occurencies
#y = y/ges_occurencies
y_poly = np.poly1d(np.polyfit(x, y, 5))
poly_missed_gaps_length.append(y_poly)
plt.figure()
plt.hist(missed_visibilities, bins=20, density=False)
plt.ylabel('occurence')
#plt.xlim((0, xmax))
plt.xlabel('visibility of target')
plt.title('Boxes total: {}'.format(len(missed_visibilities)))
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'MISS_VIS')), format='pdf')
plt.close()
# combined plot
plt.figure()
plt.hist([tracked_visibilities, missed_visibilities], bins=20, density=True, label=['tracked', 'missed'])
plt.ylabel('occurence')
plt.xlabel('visibility of target')
plt.legend()
#plt.title('Boxes total: {}'.format(len(tracked_visibilities)))
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('VIS', t, detections)), format='pdf')
# height 0.9 <= vis <= 1.0
tracked_visibilities_heights = tracked_visibilities_heights[tracked_visibilities >= 0.9]
missed_visibilities_heights = missed_visibilities_heights[missed_visibilities >= 0.9]
plt.figure()
plt.hist([tracked_visibilities_heights, missed_visibilities_heights], bins=15, density=True, label=['tracked', 'missed'])
plt.ylabel('occurence')
plt.xlabel('height of targets')
plt.legend()
#plt.title('Boxes total: {}'.format(len(tracked_visibilities)))
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('HEIGHTS', t, detections)), format='pdf')
"""
"""
plt.figure()
x_new = np.linspace(0, x_max, num=101, endpoint=True)
for y,t in zip(y_poly_list, tracker):
plt.plot(x_new, y_poly(x_new), label=t)
plt.ylabel('coverage of gap by tracker')
plt.xlim((0,x_max))
plt.xlabel('gap length in detections')
plt.legend()
plt.savefig(osp.join(output_dir, "ALL-{}-{}.pdf".format(detections, 'DET_GAP_COV')), format='pdf')
"""
#plt.legend()
#plt.ylabel('coverage of gap by tracker')
#plt.xlim((0,x_max))
#plt.xlabel('gap length in detections')
#plt.legend()
#plt.savefig(osp.join(output_dir, "ALL-{}-{}_xmax20.pdf".format('DET_GAP_COV', detections)), format='pdf')
########### AB HIER INTERESSANT FÜR DICH ##############################
# generate bins
x_max = 30
n_bins = 8
fontsize = 16
tickfontsize = 12
plt.rc('text', usetex=True)
plt.rc('font', family='serif')
step_size = (x_max - np.min(det_gaps)) / n_bins
bins = np.arange(0, n_bins) * step_size + np.min(det_gaps)
fig, ax1 = plt.subplots()
plt.tick_params(labelsize=tickfontsize)
ax1.spines['top'].set_visible(False)
bar_width = (0.7 * step_size) / len(tracker)
det_gaps = np.array(det_gaps)
det_gaps = det_gaps[det_gaps < x_max]
for t in range(len(tracker)):
ratios = np.zeros(bins.shape[0])
data = results[t]
print("Datapoints: {}".format(data.shape[0]))
data = data[data[:, 0] < x_max]
#print("After filter: {}".format(data.shape[0]))
for i in range(bins.shape[0]):
lower = bins[i]
print("BIN: {}".format(lower))
tmp = data[data[:, 0] >= lower]
#print("After lower: {}".format(tmp.shape[0]))
if i != bins.shape[0] - 1:
upper = bins[i + 1]
tmp = tmp[tmp[:, 0] < upper]
print("Datapoints in bin: {}".format(tmp.shape[0]))
ratios[i] = np.mean(tmp[:, 1])
#ratios[i] = np.median(data[:,1])
#print(ratios)
dis = (t - (len(tracker) - 1) / 2) * bar_width
plt.bar(bins + step_size / 2 + dis,
ratios,
bar_width,
align='center',
label=tracker[t].replace('_', '\_'))
color = 'black'
if "FRCNN" in detections:
color = 'white'
if "SDP" in detections:
color = 'white'
plt.ylabel('Tracked objects [\%]', fontsize=fontsize, color=color)
plt.xlim((np.min(det_gaps), x_max))
plt.ylim((0, 1.0))
color = 'black'
if "DPM" in detections:
color = 'white'
if "SDP" in detections:
color = 'white'
plt.xlabel('Gap length in public detections',
fontsize=fontsize,
color=color)
if "FRCNN" in detections:
plt.legend(loc='upper right', fontsize=tickfontsize)
ax2 = ax1.twinx()
ax2.spines['top'].set_visible(False)
#sns.kdeplot(det_gaps, cut=0, color="red", shade=True, linewidth=0)
sns.distplot(det_gaps,
bins=8,
color="red",
norm_hist=False,
ax=ax2,
kde=False,
hist_kws={
"rwidth": 0.95,
'range': (np.min(det_gaps), x_max)
})
ax2.tick_params(labelsize=tickfontsize)
#plt.setp(ax2.get_yticklabels(), visible=False)
#ax2.tick_params(axis='y', which='both', length=0)
color = 'black'
if "DPM" in detections:
color = 'white'
if "FRCNN" in detections:
color = 'white'
ax2.set_ylabel('Occurrences in detections', fontsize=fontsize, color=color)
ax2.set_zorder(1)
ax1.set_zorder(2)
ax1.patch.set_visible(False)
plt.savefig(osp.join(output_dir,
"BAR-{}-{}.pdf".format('DET_GAP_COV', detections)),
format='pdf',
bbox_inches='tight')
def my_main(_config):
print(_config)
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
###################### PFAD ANPASSEN ###################
results_dir = osp.join('output/tracktor/MOT17')
########################################################
output_dir = osp.join(results_dir, 'eval/track_missed_found')
if not osp.exists(output_dir):
os.makedirs(output_dir)
sequences_raw = [
"MOT17-13",
"MOT17-11",
"MOT17-10",
"MOT17-09",
"MOT17-05",
"MOT17-04",
"MOT17-02",
]
######### HIER DETECTIONS ÄNDERNS (DPM, FRCNN, SDP) #
detections = "SDP"
#####################################################
sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
#sequences = sequences[:1]
#tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"]
tracker = ["Tracktor", "Tracktor++", "FWT", "jCC", "MOTDT17", "MHT_DAM"]
#tracker = tracker[:2]
# "PHD_GSDL17" does not work, error
#tracker = tracker[-4:]
poly_missed_gaps_length = []
tv = []
th = []
gt_vis = []
gt_h = []
for t in tracker:
print("[*] Evaluating {}".format(t))
missed_gaps_length = []
missed_visibilities = []
tracked_visibilities = []
tracked_visibilities_heights = []
missed_visibilities_heights = []
for s in sequences:
########################################
# Get DPM / GT coverage for each track #
########################################
################## PFADE ANPASSEN #######################
gt_file = osp.join("data/MOT17Labels", "train", s, "gt", "gt.txt")
# det_file = osp.join("../data/MOT17Labels", "train", s, "det", "det.txt")
res_file = osp.join("output/tracker/MOT17", t, s + ".txt")
# gt_file = osp.join("MOT17Labels", "train", s, "gt", "gt.txt")
# #det_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "det", "det.txt")
# res_file = osp.join("track_missed_found", t, s+".txt")
#########################################################
#gtDB = read_txt_to_struct(gt_file)
#gtDB = gtDB[gtDB[:,7] == 1]
stDB = read_txt_to_struct(res_file)
gtDB = read_txt_to_struct(gt_file)
#dtDB = read_txt_to_struct(det_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
_, M, gtDB = evaluate_new(stDB, gtDB, distractor_ids)
# reaload gtDB as entries not found are deleted
gtDB_unf = read_txt_to_struct(gt_file)
# filter out so that confidence and id = 1
gtDB_unf = gtDB_unf[gtDB_unf[:, 7] == 1]
gtDB_unf = gtDB_unf[gtDB_unf[:, 6] == 1]
if t == tracker[0]:
for vis in gtDB_unf[:, 8]:
gt_vis.append(vis)
gt_temp = gtDB_unf[gtDB_unf[:, 8] >= 0.9]
heights = gt_temp[:, 5] - gt_temp[:, 3]
if "-05-" in s:
heights *= 2.25
for h in heights:
gt_h.append(h)
gt_frames = np.unique(gtDB[:, 0])
st_ids = np.unique(stDB[:, 1])
gt_ids = np.unique(gtDB[:, 1])
#dt_ids = np.unique(dtDB[:, 1])
f_gt = len(gt_frames)
n_gt = len(gt_ids)
n_st = len(st_ids)
#gt_inds = [{} for i in range(f_gt)]
#st_inds = [{} for i in range(f_gt)]
#dt_inds = [{} for i in range(f_gt)]
gt_inds_unf = [{} for i in range(f_gt)]
#D = [{} for i in range(f_gt)] #map detections to gt
# hash the indices to speed up indexing
#for i in range(gtDB.shape[0]):
# frame = np.where(gt_frames == gtDB[i, 0])[0][0]
# gid = np.where(gt_ids == gtDB[i, 1])[0][0]
# gt_inds[frame][gid] = i
for i in range(gtDB_unf.shape[0]):
frame = np.where(gt_frames == gtDB_unf[i, 0])[0][0]
gt_id = gtDB_unf[i, 1]
gt_inds_unf[frame][gt_id] = i
#gt_frames_list = list(gt_frames)
#for i in range(stDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
# frame = gt_frames_list.index(stDB[i, 0])
# sid = np.where(st_ids == stDB[i, 1])[0][0]
# st_inds[frame][sid] = i
for frame, m in enumerate(M):
for gt_id, line in gt_inds_unf[frame].items():
vis = gtDB_unf[line, 8]
height = gtDB_unf[line, 5] - gtDB_unf[line, 3]
if "-05-" in s:
height *= 2.25
gid = -1
gid_ind = np.where(gt_ids == gt_id)[0]
if len(gid_ind) > 0:
gid = gid_ind[0]
# check if tracked
if gid in m.keys():
tracked_visibilities.append(vis)
tracked_visibilities_heights.append(height)
continue
else:
missed_visibilities.append(vis)
missed_visibilities_heights.append(height)
# get distance to last time tracked
last_non_empty = -1
for j in range(frame, -1, -1):
if gid in M[j].keys():
last_non_empty = j
break
next_non_empty = -1
for j in range(frame, f_gt):
if gid in M[j].keys():
next_non_empty = j
break
if next_non_empty != -1 and last_non_empty != -1:
gap_length = next_non_empty - last_non_empty - 1
missed_gaps_length.append(gap_length)
"""
for i in range(dtDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
frame = gt_frames_list.index(dtDB[i, 0])
did = np.where(dt_ids == dtDB[i, 1])[0][0]
dt_inds[frame][did] = i
# find track <-> gt
for frame in range(f_gt):
gt_ids = list(gt_inds[frame].keys())
gt_i = list(gt_inds[frame].values())
gt_boxes = gtDB[gt_i, 2:6]
st_ids = list(st_inds[frame].keys())
st_i = list(st_inds[frame].values())
st_boxes = stDB[st_i, 2:6]
overlaps = np.zeros((len(st_i), len(gt_i)), dtype=float)
for i in range(len(gt_i)):
overlaps[:, i] = bbox_overlap(st_boxes, gt_boxes[i])
matched_indices = linear_assignment(1 - overlaps)
for matched in matched_indices:
if overlaps[matched[0], matched[1]] > 0.5:
did = st_ids[matched[0]]
gid = gt_ids[matched[1]]
D[frame][did] = gid
# Now check all detections if they were tracked
for frame in range(f_gt):
matched_dets = D[frame]
for did, gid in matched_dets.items():
# not matched by tracker, get visbility
line = gt_inds[frame][gid]
vis = gtDB[line, 8]
# if matched only visibility is important
if gid in M[frame].keys():
tracked_visibilities.append(vis)
continue
else:
missed_visibilities.append(vis)
# get distance to last time tracked
last_non_empty = frame
for j in range(frame, -1, -1):
if gid in M[j].keys():
last_non_empty = j
break
next_non_empty = frame
for j in range(frame, f_gt):
if gid in M[j]:
next_non_empty = j
break
dist = min(frame-last_non_empty, next_non_empty-frame)
# check if not tracked at all ...
if dist > 0:
missed_distances.append(dist)"""
missed_gaps_length = np.array(missed_gaps_length)
missed_visibilities = np.array(missed_visibilities)
tracked_visibilities = np.array(tracked_visibilities)
tracked_visibilities_heights = np.array(tracked_visibilities_heights)
missed_visibilities_heights = np.array(missed_visibilities_heights)
#plt.plot([0,1], [0,1], 'r-')
"""
plt.figure()
plt.hist(tracked_visibilities, bins=20, density=False)
plt.ylabel('occurence')
plt.xlabel('visibility of target')
plt.title('Boxes total: {}'.format(len(tracked_visibilities)))
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'TR_VIS')), format='pdf')
"""
#plt.plot([0,1], [0,1], 'r-')
weights = np.zeros(missed_gaps_length.shape)
for i in range(missed_gaps_length.shape[0]):
weights[i] = 1 / missed_gaps_length[i]
#weights[i] = 1
#plt.figure()
#plt.xlim((0, 40))
#bins = list(range(40))
#plt.hist(missed_gaps_length, weights=weights, bins=bins, density=False)
#plt.ylabel('occurence')
#plt.xlabel('gap length in tracks')
#plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('MISS_DIST', t, detections)), format='pdf')
x = np.arange(1, 41)
y = np.zeros(len(x))
ges_occurencies = 0
for i, b in enumerate(x):
occurencies = int((missed_gaps_length == b).sum())
occurencies = occurencies / b
ges_occurencies += occurencies
y[i] = occurencies
#y = y/ges_occurencies
y_poly = np.poly1d(np.polyfit(x, y, 5))
poly_missed_gaps_length.append(y_poly)
"""
plt.figure()
plt.hist(missed_visibilities, bins=20, density=False)
plt.ylabel('occurence')
#plt.xlim((0, xmax))
plt.xlabel('visibility of target')
plt.title('Boxes total: {}'.format(len(missed_visibilities)))
plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'MISS_VIS')), format='pdf')
plt.close()
"""
# combined plot
# plt.figure()
# plt.hist([tracked_visibilities, missed_visibilities], bins=20, density=True, label=['tracked', 'missed'])
# plt.ylabel('occurence')
# plt.xlabel('visibility of target')
# plt.legend()
#plt.title('Boxes total: {}'.format(len(tracked_visibilities)))
# plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('VIS', t, detections)), format='pdf')
# height 0.9 <= vis <= 1.0
tracked_visibilities_heights = tracked_visibilities_heights[
tracked_visibilities >= 0.9]
missed_visibilities_heights = missed_visibilities_heights[
missed_visibilities >= 0.9]
# plt.figure()
# plt.hist([tracked_visibilities_heights, missed_visibilities_heights], bins=15, density=True, label=['tracked', 'missed'])
# plt.ylabel('occurence')
# plt.xlabel('height of targets')
# plt.legend()
#plt.title('Boxes total: {}'.format(len(tracked_visibilities)))
# plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('HEIGHTS', t, detections)), format='pdf')
tv.append(tracked_visibilities)
th.append(tracked_visibilities_heights)
h_max = 250
h_n_bins = 8
v_n_bins = 12
gt_h = np.array(gt_h)
gt_vis = np.array(gt_vis)
tv = np.array(tv)
th = np.array(th)
h_step = (h_max - np.min(gt_h)) / h_n_bins
v_step = 1.0 / v_n_bins
gt_h = gt_h[gt_h < h_max]
h_bins = np.arange(0, h_n_bins) * h_step + np.min(gt_h)
v_bins = np.arange(0, v_n_bins) * v_step
gt_h_occurencies = np.zeros(h_bins.shape[0])
for i in range(h_bins.shape[0]):
lower = h_bins[i]
tmp = gt_h[gt_h >= lower]
if i != h_bins.shape[0] - 1:
upper = h_bins[i + 1]
tmp = tmp[tmp < upper]
gt_h_occurencies[i] = len(tmp)
gt_v_occurencies = np.zeros(v_bins.shape[0])
for i in range(v_bins.shape[0]):
lower = v_bins[i]
tmp = gt_vis[gt_vis >= lower]
if i != v_bins.shape[0] - 1:
upper = v_bins[i + 1]
tmp = tmp[tmp < upper]
gt_v_occurencies[i] = len(tmp)
heights_results = []
for i in range(len(tracker)):
t = tracker[i]
heights = th[i]
heights = heights[heights < h_max]
res = []
for i in range(h_bins.shape[0]):
lower = h_bins[i]
tmp = heights[heights >= lower]
if i != h_bins.shape[0] - 1:
upper = h_bins[i + 1]
tmp = tmp[tmp < upper]
percentage = len(tmp) / gt_h_occurencies[i]
res.append(percentage)
heights_results.append(np.array(res))
vis_results = []
for i in range(len(tracker)):
t = tracker[i]
vis = tv[i]
res = []
for i in range(v_bins.shape[0]):
lower = v_bins[i]
tmp = vis[vis >= lower]
if i != v_bins.shape[0] - 1:
upper = v_bins[i + 1]
tmp = tmp[tmp < upper]
percentage = len(tmp) / gt_v_occurencies[i]
res.append(percentage)
vis_results.append(np.array(res))
plt.rc('text', usetex=True)
plt.rc('font', family='serif')
fontsize = 16
tickfontsize = 12
############ AB HIER WICHTIG FÜR DICH ##########################
# Plot heights
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
ax2.set_zorder(1)
ax1.set_zorder(2)
ax1.patch.set_visible(False)
ax2.spines['top'].set_visible(False)
#sns.distplot(gt_h)
#sns.kdeplot(gt_h, cut=0, color="red", shade=True, linewidth=0)
sns.distplot(gt_h,
bins=8,
color="red",
norm_hist=False,
ax=ax2,
kde=False,
hist_kws={
"rwidth": 0.95,
'range': (np.min(gt_h), h_max)
})
#plt.setp(ax3.get_xticklabels(), visible=False)
#plt.setp(ax2.get_yticklabels(), visible=False)
#ax2.tick_params(axis='y', which='both', length=0, labelsize=fontsize)
ax2.tick_params(labelsize=tickfontsize)
#ax2.set_ylabel('Occurrence in tracking ground truth (\%)')
color = 'black'
if "DPM" in detections:
color = 'white'
if "FRCNN" in detections:
color = 'white'
ax2.set_ylabel('Occurrences in ground truth',
fontsize=fontsize,
color=color)
#ax2.set_ylim((0,h_max))
#sns.kdeplot(gt_h, shade=True, cut=0)
ax1.spines['top'].set_visible(False)
bar_width = (0.7 * h_step) / len(tracker)
for i in range(len(tracker)):
dis = (i - (len(tracker) - 1) / 2) * bar_width
ax1.bar(h_bins + h_step / 2 + dis,
heights_results[i],
bar_width,
align='center',
label=tracker[i].replace('_', '\_'))
ax1.set_ylim((0, 1.0))
ax1.set_xlim((np.min(gt_h), h_max))
#if "DPM" in detections:
# plt.legend(fontsize=fontsize)
plt.tick_params(labelsize=tickfontsize)
color = 'black'
if "DPM" in detections:
color = 'white'
if "SDP" in detections:
color = 'white'
ax1.set_xlabel('Object height (pixels)', fontsize=fontsize, color=color)
color = 'black'
if "FRCNN" in detections:
color = 'white'
if "SDP" in detections:
color = 'white'
ax1.set_ylabel('Tracked objects (\%)', fontsize=fontsize, color=color)
#plt.title('height distribution from highly visible targets (>= 0.9)')
plt.savefig(osp.join(output_dir, "heights09-{}.pdf".format(detections)),
format='pdf',
bbox_inches='tight')
if "FRCNN" in detections:
# Plot vis
fig, ax1 = plt.subplots()
ax2 = ax1.twinx()
ax2.set_zorder(1)
ax1.set_zorder(2)
ax1.patch.set_visible(False)
ax2.spines['top'].set_visible(False)
#sns.kdeplot(gt_vis, cut=0, color="red", shade=True, linewidth=0, ax=ax2)
sns.distplot(gt_vis,
bins=12,
color="red",
norm_hist=False,
ax=ax2,
kde=False,
hist_kws={"rwidth": 0.95})
ax2.tick_params(labelsize=tickfontsize)
#plt.setp(ax2.get_yticklabels(), visible=False)
#ax2.tick_params(axis='y', which='both', length=0)
#ax2.set_ylabel('Occurrence in tracking ground truth (\%)')
ax2.set_ylabel('Occurrences in ground truth', fontsize=fontsize)
#ax2.ylim((0,0.006))
ax1.spines['top'].set_visible(False)
#plt.hist(vis_results, weights=vis_weights, label=tracker)
#sns.distplot(gt_vis)
#sns.kdeplot(gt_vis, bw=1.0, label="bw: 1.0", cut=0)
#sns.kdeplot(gt_vis, bw=0.2, label="bw: 0.2", cut=0)
#sns.kdeplot(gt_vis, bw=2, label="bw: 2", cut=0)
bar_width = (0.7 * v_step) / len(tracker)
for i in range(len(tracker)):
dis = (i - (len(tracker) - 1) / 2) * bar_width
ax1.bar(v_bins + v_step / 2 + dis,
vis_results[i],
bar_width,
align='center',
label=tracker[i].replace('_', '\_'))
ax1.set_ylim((0, 1.0))
ax1.set_xlim((0, 1.0))
ax1.tick_params(labelsize=tickfontsize)
ax1.legend(loc='upper left', fontsize=tickfontsize)
ax1.set_xlabel('Object visibility (\%)', fontsize=fontsize)
ax1.set_ylabel('Tracked objects (\%)', fontsize=fontsize)
#plt.title('visibility distribution of all targets')
plt.savefig(osp.join(output_dir, "vis-{}.pdf".format(detections)),
format='pdf',
bbox_inches='tight')
"""
def my_main(_config):
print(_config)
dataset = "mot_train_"
detections = "FRCNN"
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
module_dir = get_output_dir('MOT17')
results_dir = module_dir
module_dir = osp.join(module_dir, 'eval/video_red_green')
#output_dir = osp.join(results_dir, 'plots')
#if not osp.exists(output_dir):
# os.makedirs(output_dir)
#sequences_raw = ["MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ]
#sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
#sequences = sequences[:1]
# tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"]
# tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17", "MHT_DAM_17"]
tracker = ["FWT", "jCC", "MOTDT17"]#, "Tracktor++"]
baseline = "Tracktor"
for t in tracker:
print("[*] Evaluating {}".format(t))
for db in Datasets(dataset):
################################
# Make videos for each tracker #
################################
s = "{}-{}".format(db, detections)
gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt")
res_file = osp.join(results_dir, t, s+".txt")
base_file = osp.join(results_dir, baseline, s+".txt")
stDB = read_txt_to_struct(res_file)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
_, M_res, gtDB, stDB = evaluate_new(stDB, gtDB, distractor_ids)
gt_ids_res = np.unique(gtDB[:, 1])
bsDB = read_txt_to_struct(base_file)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
_, M_bs, gtDB, stDB = evaluate_new(bsDB, gtDB, distractor_ids)
gt_ids_base = np.unique(gtDB[:, 1])
gtDB = read_txt_to_struct(gt_file)
# filter out so that confidence and id = 1
gtDB = gtDB[gtDB[:,7] == 1]
gtDB = gtDB[gtDB[:,6] == 1]
#st_ids = np.unique(stDB[:, 1])
#gt_ids = np.unique(gtDB[:, 1])
gt_frames = np.unique(gtDB[:, 0])
f_gt = len(gt_frames)
gt_inds = [{} for i in range(f_gt)]
#st_inds = [{} for i in range(f_gt)]
# hash the indices to speed up indexing
for i in range(gtDB.shape[0]):
frame = np.where(gt_frames == gtDB[i, 0])[0][0]
#gid = np.where(gt_ids == gtDB[i, 1])[0][0]
gt_id = int(gtDB[i,1])
gt_inds[frame][gt_id] = i
#gt_frames_list = list(gt_frames)
#for i in range(stDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
# frame = gt_frames_list.index(stDB[i, 0])
# sid = np.where(st_ids == stDB[i, 1])[0][0]
# st_inds[frame][sid] = i
results = []
for frame in range(f_gt):
# get gt_ids in res
m_res = M_res[frame]
gids = list(m_res.keys())
res_gt = []
for gid in gids:
res_gt.append(gt_ids_res[gid])
# get gt_ids in base
m_bs = M_bs[frame]
gids = list(m_bs.keys())
base_gt = []
for gid in gids:
base_gt.append(gt_ids_base[gid])
# get unique gt ids
unique_gt = np.unique(res_gt + base_gt)
#print("res gt: {}".format(res_gt))
#print("base gt: {}".format(base_gt))
for gt in unique_gt:
gt = int(gt)
#print(gt)
res = np.zeros(6)
res[0] = frame+1
res[2:6] = gtDB[gt_inds[frame][gt], 2:6]
if gt in res_gt and gt in base_gt:
res[1] = 1
elif gt in base_gt:
res[1] = 2
elif gt in res_gt:
res[1] = 3
results.append(res)
results = np.array(results)
output_dir = osp.join(module_dir, t, s)
if not osp.exists(output_dir):
os.makedirs(output_dir)
print("[*] Plotting whole sequence to {}".format(output_dir))
# infinte color loop
cyl = cy('ec', colors)
loop_cy_iter = cyl()
styles = defaultdict(lambda : next(loop_cy_iter))
for frame,v in enumerate(db,1):
im_path = v['im_path']
im_name = osp.basename(im_path)
im_output = osp.join(output_dir, im_name)
im = cv2.imread(im_path)
im = im[:, :, (2, 1, 0)]
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
#fig.set_size_inches(w,h)
#fig.set_size_inches(width/height, 1, forward=False)
#fig.set_size_inches(width/100, height/100)
scale = width/640
#fig.set_size_inches(640/100, height*scale/100)
fig.set_size_inches(width/100, height/100)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
ax.imshow(im)
res_frame = results[results[:,0]==frame]
for j in range(res_frame.shape[0]):
box = res_frame[j,2:6]
gt_id = int(res_frame[j,1])
ax.add_patch(
plt.Rectangle((box[0], box[1]),
box[2] - box[0],
box[3] - box[1], fill=False,
linewidth=1.3*scale, color=colors[gt_id])
#**styles[gt_id])
)
plt.axis('off')
#plt.tight_layout()
plt.draw()
plt.savefig(im_output, dpi=100)
plt.close()
def my_main(_config):
print(_config)
dataset = "mot_train_"
detections = "SDP"
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
module_dir = get_output_dir('videos')
results_dir = osp.join(module_dir, 'results')
module_dir = osp.join(module_dir, 'normal_new')
#output_dir = osp.join(results_dir, 'plots')
#if not osp.exists(output_dir):
# os.makedirs(output_dir)
#sequences_raw = ["MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ]
#sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
#sequences = sequences[:1]
tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"]
tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17", "MHT_DAM_17"]
tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17"]
for t in tracker:
print("[*] Evaluating {}".format(t))
for db in Datasets(dataset):
################################
# Make videos for each tracker #
################################
s = "{}-{}".format(db, detections)
gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt")
res_file = osp.join(results_dir, t, s+".txt")
stDB = read_txt_to_struct(res_file)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
_, M, gtDB, stDB = evaluate_new(stDB, gtDB, distractor_ids)
st_ids = np.unique(stDB[:, 1])
gt_ids = np.unique(gtDB[:, 1])
gt_frames = np.unique(gtDB[:, 0])
f_gt = len(gt_frames)
gt_inds = [{} for i in range(f_gt)]
st_inds = [{} for i in range(f_gt)]
# hash the indices to speed up indexing
for i in range(gtDB.shape[0]):
frame = np.where(gt_frames == gtDB[i, 0])[0][0]
gid = np.where(gt_ids == gtDB[i, 1])[0][0]
gt_inds[frame][gid] = i
gt_frames_list = list(gt_frames)
for i in range(stDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
frame = gt_frames_list.index(stDB[i, 0])
sid = np.where(st_ids == stDB[i, 1])[0][0]
st_inds[frame][sid] = i
# set all ids of st to -1
#stDB[:,1] = -1
# set all results to corresponding gt id
#for frame in range(f_gt):
# m = M[frame]
# for gid, sid in m.items():
# gt_row = gt_inds[frame][gid]
# gt_id = int(gtDB[gt_row, 1])
# st_row = st_inds[frame][sid]
# stDB[st_row, 1] = gt_id
output_dir = osp.join(module_dir, t, s)
if not osp.exists(output_dir):
os.makedirs(output_dir)
print("[*] Plotting whole sequence to {}".format(output_dir))
# infinte color loop
cyl = cy('ec', colors)
loop_cy_iter = cyl()
styles = defaultdict(lambda : next(loop_cy_iter))
for frame,v in enumerate(db,1):
im_path = v['im_path']
im_name = osp.basename(im_path)
im_output = osp.join(output_dir, im_name)
im = cv2.imread(im_path)
im = im[:, :, (2, 1, 0)]
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
#fig.set_size_inches(w,h)
#fig.set_size_inches(width/height, 1, forward=False)
#fig.set_size_inches(width/100, height/100)
scale = width/640
#fig.set_size_inches(640/100, height*scale/100)
fig.set_size_inches(width/100, height/100)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
ax.imshow(im)
st_frame = stDB[stDB[:,0]==frame]
for j in range(st_frame.shape[0]):
box = st_frame[j,2:6]
gt_id = st_frame[j,1]
ax.add_patch(
plt.Rectangle((box[0], box[1]),
box[2] - box[0],
box[3] - box[1], fill=False,
linewidth=1.3*scale, **styles[gt_id])
)
plt.axis('off')
#plt.tight_layout()
plt.draw()
plt.savefig(im_output, dpi=100)
plt.close()
def my_main(_config):
print(_config)
dataset = "mot_train_"
detections = "FRCNN"
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
module_dir = get_output_dir('MOT17')
results_dir = module_dir
module_dir = osp.join(module_dir, 'eval/video_fp')
#output_dir = osp.join(results_dir, 'plots')
#if not osp.exists(output_dir):
# os.makedirs(output_dir)
#sequences_raw = ["MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ]
#sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
#sequences = sequences[:1]
# tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"]
# tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17", "MHT_DAM_17"]
tracker = ["Tracktor", "FWT", "jCC", "MOTDT17"]
#tracker = ["Baseline"]
for t in tracker:
print("[*] Evaluating {}".format(t))
if True:
#for db in Datasets(dataset):
################################
# Make videos for each tracker #
################################
db = Datasets(dataset)[2]
s = "{}-{}".format(db, detections)
gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt",
"gt.txt")
res_file = osp.join(results_dir, t, s + ".txt")
stDB = read_txt_to_struct(res_file)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
_, M, gtDB, stDB = evaluate_new(stDB, gtDB, distractor_ids)
#gt_ids_res = np.unique(gtDB[:, 1])
#gtDB = read_txt_to_struct(gt_file)
# filter out so that confidence and id = 1
#gtDB = gtDB[gtDB[:,7] == 1]
#gtDB = gtDB[gtDB[:,6] == 1]
st_ids = np.unique(stDB[:, 1])
#gt_ids = np.unique(gtDB[:, 1])
gt_frames = np.unique(gtDB[:, 0])
f_gt = len(gt_frames)
#gt_inds = [{} for i in range(f_gt)]
st_inds = [{} for i in range(f_gt)]
# hash the indices to speed up indexing
#for i in range(gtDB.shape[0]):
# frame = np.where(gt_frames == gtDB[i, 0])[0][0]
#gid = np.where(gt_ids == gtDB[i, 1])[0][0]
# gt_id = int(gtDB[i,1])
# gt_inds[frame][gt_id] = i
gt_frames_list = list(gt_frames)
for i in range(stDB.shape[0]):
# sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment
frame = gt_frames_list.index(stDB[i, 0])
sid = np.where(st_ids == stDB[i, 1])[0][0]
st_inds[frame][sid] = i
#stDB = read_txt_to_struct(res_file)
results = []
for frame in range(f_gt):
# get gt_ids in res
m = M[frame]
matched_sids = list(m.values())
#frame_sids = list(st_inds[frame].keys())
f = gt_frames_list[frame]
st_frame = stDB[stDB[:, 0] == f]
st_uniq_ids = np.unique(st_frame[:, 1])
for st_id in st_uniq_ids:
sid = -1
si = np.where(st_ids == st_id)[0]
if len(si) > 0:
sid = si[0]
if sid not in matched_sids:
res = np.zeros(6)
res[0] = frame + 1
st_track = st_frame[st_frame[:, 1] == st_id]
res[2:6] = st_track[0, 2:6]
results.append(res)
else:
matched_sids.remove(sid)
results = np.array(results)
print(results.shape[0])
output_dir = osp.join(module_dir, t, s)
if not osp.exists(output_dir):
os.makedirs(output_dir)
print("[*] Plotting whole sequence to {}".format(output_dir))
# infinte color loop
cyl = cy('ec', colors)
loop_cy_iter = cyl()
styles = defaultdict(lambda: next(loop_cy_iter))
for frame, v in enumerate(db, 1):
im_path = v['im_path']
im_name = osp.basename(im_path)
im_output = osp.join(output_dir, im_name)
im = cv2.imread(im_path)
im = im[:, :, (2, 1, 0)]
sizes = np.shape(im)
height = float(sizes[0])
width = float(sizes[1])
fig = plt.figure()
#fig.set_size_inches(w,h)
#fig.set_size_inches(width/height, 1, forward=False)
#fig.set_size_inches(width/100, height/100)
scale = width / 640
#fig.set_size_inches(640/100, height*scale/100)
fig.set_size_inches(width / 100, height / 100)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
ax.imshow(im)
res_frame = results[results[:, 0] == frame]
for j in range(res_frame.shape[0]):
box = res_frame[j, 2:6]
gt_id = int(res_frame[j, 1])
ax.add_patch(
plt.Rectangle((box[0], box[1]),
box[2] - box[0],
box[3] - box[1],
fill=False,
linewidth=1.3 * scale,
color='blue'))
ax.annotate(t, (width - 250, height - 100),
color='white',
weight='bold',
fontsize=72,
ha='center',
va='center')
plt.axis('off')
plt.draw()
plt.savefig(im_output, dpi=100)
plt.close()
def my_main(_config):
print(_config)
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
output_dir = osp.join(get_output_dir('MOT_analysis'), 'coverage')
sequences_raw = [
"MOT17-13",
"MOT17-11",
"MOT17-10",
"MOT17-09",
"MOT17-05",
"MOT17-04",
"MOT17-02",
]
detections = "DPM"
sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
tracker = [
"FRCNN", "DMAN", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17",
"MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"
]
#tracker = ["DMAN"]
for t in tracker:
print("[*] Evaluating {}".format(t))
data_points = []
for s in sequences:
########################################
# Get DPM / GT coverage for each track #
########################################
gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt",
"gt.txt")
dpm_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "det",
"det.txt")
gtDB = read_txt_to_struct(gt_file)
dpmDB = read_txt_to_struct(dpm_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
dpmDB, gtDB = preprocessingDB(dpmDB, gtDB, distractor_ids, 0.5, 0)
gt_ids = np.unique(gtDB[:, 1])
gt_ges = {int(i): 0 for i in gt_ids}
gt_matched = {int(i): 0 for i in gt_ids}
gt_tracked = {int(i): 0 for i in gt_ids}
track_frames = np.unique(dpmDB[:, 0])
gt_frames = np.unique(gtDB[:, 0])
nframes = min(len(track_frames), len(gt_frames))
res_keep = np.ones((dpmDB.shape[0], ), dtype=float)
for i in range(1, nframes + 1):
# find all result boxes in this frame
res_in_frame = np.where(dpmDB[:, 0] == i)[0]
res_in_frame_data = dpmDB[res_in_frame, :]
gt_in_frame = np.where(gtDB[:, 0] == i)[0]
gt_in_frame_data = gtDB[gt_in_frame, :]
#for gt in gt_in_frame_data:
# gt_ges[int(gt[1])] += 1
res_num = res_in_frame.shape[0]
gt_num = gt_in_frame.shape[0]
overlaps = np.zeros((res_num, gt_num), dtype=float)
for gid in range(gt_num):
overlaps[:, gid] = bbox_overlap(res_in_frame_data[:, 2:6],
gt_in_frame_data[gid, 2:6])
matched_indices = linear_assignment(1 - overlaps)
for matched in matched_indices:
# overlap lower than threshold, discard the pair
if overlaps[matched[0], matched[1]] > 0.5:
gt_id = int(gt_in_frame_data[matched[1], 1])
gt_matched[gt_id] += 1
for k in gt_ids:
gt_ges[k] = len(np.where(gtDB[:, 1] == k)[0])
gt_tracked[k] = gt_matched[k] / gt_ges[k]
res_file = osp.join(output_dir, t, s + ".txt")
gtDB = read_txt_to_struct(gt_file)
trackDB = read_txt_to_struct(res_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
trackDB, gtDB = preprocessingDB(trackDB, gtDB, distractor_ids, 0.5,
0)
tr_matched = {int(i): 0 for i in gt_ids}
tr_tracked = {int(i): 0 for i in gt_ids}
track_frames = np.unique(trackDB[:, 0])
gt_frames = np.unique(gtDB[:, 0])
nframes = min(len(track_frames), len(gt_frames))
res_keep = np.ones((trackDB.shape[0], ), dtype=float)
for i in range(1, nframes + 1):
# find all result boxes in this frame
res_in_frame = np.where(trackDB[:, 0] == i)[0]
res_in_frame_data = trackDB[res_in_frame, :]
gt_in_frame = np.where(gtDB[:, 0] == i)[0]
gt_in_frame_data = gtDB[gt_in_frame, :]
res_num = res_in_frame.shape[0]
gt_num = gt_in_frame.shape[0]
overlaps = np.zeros((res_num, gt_num), dtype=float)
for gid in range(gt_num):
overlaps[:, gid] = bbox_overlap(res_in_frame_data[:, 2:6],
gt_in_frame_data[gid, 2:6])
matched_indices = linear_assignment(1 - overlaps)
for matched in matched_indices:
# overlap lower than threshold, discard the pair
if overlaps[matched[0], matched[1]] > 0.5:
gt_id = int(gt_in_frame_data[matched[1], 1])
tr_matched[gt_id] += 1
for k in gt_ids:
data_points.append([gt_tracked[k], tr_matched[k] / gt_ges[k]])
data_points = np.array(data_points)
# add mean values
grid_step = 0.02
grid = np.arange(-grid_step / 2, 1.0 + grid_step, grid_step)
x_mean = np.arange(0.0, 1.0 + grid_step, grid_step)
bins = int(1.0 / grid_step) + 1
y_mean = np.zeros(bins)
y_std = np.zeros(bins)
for i in range(bins):
vals = (data_points[:, 0] >= grid[i]) * (data_points[:, 0] <
grid[i + 1])
mean = np.mean(data_points[vals, 1])
y_mean[i] = mean
y_std[i] = np.sqrt(np.mean((vals - mean)**2))
y_poly = np.poly1d(np.polyfit(x_mean, y_mean, 5))
x_new = np.linspace(0, 1, num=101, endpoint=True)
area = simps(y_poly(x_new), x_new)
plt.plot(x_new, y_poly(x_new), label="{} {:.3f}".format(t, area))
#plt.errorbar(x_mean, y_poly(x_mean), yerr=y_std, fmt='o')
#if t == "FRCNN":
# plt.plot(x_mean, y_mean)
#plt.plot([0,1], [0,1], 'r-')
#plt.scatter(data_points[:,0], data_points[:,1], s=2**2)
#plt.xlabel('{} coverage'.format(detections))
#plt.ylabel('tracker coverage')
#plt.savefig(osp.join(output_dir, "{}-{}.pdf".format(t, detections)), format='pdf')
#plt.close()
plt.plot([0, 1], [0, 1], 'r-')
plt.legend()
plt.xlabel('{} coverage'.format(detections))
plt.ylabel('tracker coverage')
plt.savefig(osp.join(output_dir, "coverage-{}.pdf".format(detections)),
format='pdf')
plt.close()
def my_main(_config):
print(_config)
##########################
# Initialize the modules #
##########################
print("[*] Beginning evaluation...")
output_dir = osp.join(get_output_dir('MOT_analysis'), 'occlusion')
if not osp.exists(output_dir):
os.makedirs(output_dir)
sequences_raw = [
"MOT17-13",
"MOT17-11",
"MOT17-10",
"MOT17-09",
"MOT17-05",
"MOT17-04",
"MOT17-02",
]
detections = "DPM"
sequences = ["{}-{}".format(s, detections) for s in sequences_raw]
tracker = [
"FRCNN", "DMAN", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17",
"MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"
]
#tracker = ["FRCNN"]
# "PHD_GSDL17" does not work, error
#tracker = tracker[-4:]
for t in tracker:
print("[*] Evaluating {}".format(t))
coverage = []
id_recovered = []
tr_id_recovered = []
for s in sequences:
########################################
# Get DPM / GT coverage for each track #
########################################
gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt",
"gt.txt")
det_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "det",
"det.txt")
res_file = osp.join(output_dir, t, s + ".txt")
#gtDB = read_txt_to_struct(gt_file)
#gtDB = gtDB[gtDB[:,7] == 1]
stDB = read_txt_to_struct(res_file)
gtDB = read_txt_to_struct(gt_file)
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
_, M = evaluate_new(stDB, gtDB, distractor_ids)
gt_frames = np.unique(gtDB[:, 0])
st_ids = np.unique(stDB[:, 1])
gt_ids = np.unique(gtDB[:, 1])
f_gt = len(gt_frames)
n_gt = len(gt_ids)
n_st = len(st_ids)
gt_inds = [{} for i in range(f_gt)]
st_inds = [{} for i in range(f_gt)]
# hash the indices to speed up indexing
for i in range(gtDB.shape[0]):
frame = np.where(gt_frames == gtDB[i, 0])[0][0]
gid = np.where(gt_ids == gtDB[i, 1])[0][0]
gt_inds[frame][gid] = i
# Loop thorugh all gt and find gaps (visibility < 0.5)
visible = [[0 for j in range(f_gt)] for i in range(n_gt)
] # format visible[track][frame] = {0,1}
for gid in range(n_gt):
for frame in range(f_gt):
if gid in gt_inds[frame]:
line = gt_inds[frame][gid]
vis = gtDB[line, 8]
#print(vis, frame, gid)
if vis >= 0.5:
visible[gid][frame] = 1
# Find gaps in the tracks
gt_tracked = {}
for f, v in enumerate(M):
for gt in v.keys():
if gt not in gt_tracked:
gt_tracked[gt] = []
gt_tracked[gt].append(f)
for gid, times in gt_tracked.items():
times = np.array(times)
for i in range(len(times) - 1):
t0 = times[i]
t1 = times[i + 1]
if t1 == t0 + 1:
continue
last_non_empty = -1
for j in range(t0, -1, -1):
if gid in M[j].keys():
last_non_empty = j
break
next_non_empty = -1
for j in range(t1, f_gt):
if gid in M[j]:
next_non_empty = j
break
if next_non_empty != -1 and last_non_empty != -1:
sid0 = M[last_non_empty][gid]
sid1 = M[next_non_empty][gid]
if sid1 == sid0:
tr_id_recovered.append([t1 - t0 - 1, 1])
else:
tr_id_recovered.append([t1 - t0 - 1, 0])
"""for gid in range(n_gt):
f0 = -1
count = 0
for frame in range(f_gt):
if gid in gt_inds[frame]:
vis = gtDB[gt_inds[frame][gid], 8]
if vis < 0.5 and f0 != -1:
count += 1
elif vis >= 0.5:
if count != 0:
print("Gap found {} - {} ({})".format(gid, frame, count))
count = 0
# set to current frame
f0 = frame"""
# Now iterate through the tracks and check if covered / id kept in comparison to occlusion
for gid, vis in enumerate(visible):
f0 = -1
count = 0
n_cov = 0
for frame, v in enumerate(vis):
if v == 0 and f0 != -1:
count += 1
if gid in M[frame].keys():
n_cov += 1
elif v == 1:
# gap ended
if count != 0:
coverage.append([count, n_cov])
last_non_empty = -1
for j in range(f0, -1, -1):
if gid in M[j].keys():
last_non_empty = j
break
next_non_empty = -1
for j in range(f0 + count + 1, f_gt):
if gid in M[j]:
next_non_empty = j
break
if next_non_empty != -1 and last_non_empty != -1:
sid0 = M[last_non_empty][gid]
sid1 = M[next_non_empty][gid]
if sid1 == sid0:
id_recovered.append([count, 1])
else:
id_recovered.append([count, 0])
count = 0
n_cov = 0
# set to current frame
f0 = frame
coverage = np.array(coverage)
id_recovered = np.array(id_recovered)
tr_id_recovered = np.array(tr_id_recovered)
#for c in coverage:
# print(c)
xmax = 50
# build values for plot
x_val = np.arange(1, xmax + 1)
y_val = np.zeros(xmax)
for x in x_val:
y = np.mean(coverage[coverage[:, 0] == x, 1] /
coverage[coverage[:, 0] == x, 0])
y_val[x - 1] = y
#plt.plot([0,1], [0,1], 'r-')
plt.figure()
plt.scatter(coverage[:, 0], coverage[:, 1] / coverage[:, 0], s=2**2)
plt.plot(x_val, y_val, 'rx')
plt.xlabel('gap length')
plt.xlim((0, xmax))
plt.ylabel('tracker coverage')
plt.savefig(osp.join(output_dir,
"{}-{}-{}.pdf".format(t, detections, 'GAP_COV')),
format='pdf')
# build values for plot
x_val = np.arange(1, xmax + 1)
y_val = np.zeros(xmax)
for x in x_val:
y = np.mean(id_recovered[id_recovered[:, 0] == x, 1])
y_val[x - 1] = y
plt.figure()
plt.plot(x_val, y_val, 'rx')
plt.scatter(id_recovered[:, 0], id_recovered[:, 1], s=2**2)
plt.xlabel('gt gap length')
plt.xlim((0, xmax))
plt.ylabel('part id recovered')
plt.savefig(osp.join(output_dir,
"{}-{}-{}.pdf".format(t, detections, 'GAP_ID')),
format='pdf')
plt.close()
# tr id recovered
x_val = np.arange(1, xmax + 1)
y_val = np.zeros(xmax)
for x in x_val:
y = np.mean(tr_id_recovered[tr_id_recovered[:, 0] == x, 1])
y_val[x - 1] = y
plt.figure()
plt.plot(x_val, y_val, 'rx')
plt.scatter(tr_id_recovered[:, 0], tr_id_recovered[:, 1], s=2**2)
plt.xlabel('track gap length')
plt.xlim((0, xmax))
plt.ylabel('part id recovered')
plt.savefig(osp.join(output_dir,
"{}-{}-{}.pdf".format(t, detections,
'GAP_TR_ID')),
format='pdf')
plt.close()
def evaluate_new(results, gt_file):
res = []
for i, track in results.items():
for frame, bb in track.items():
x1 = bb[0]
y1 = bb[1]
x2 = bb[2]
y2 = bb[3]
res.append([
float(frame + 1),
float(i + 1),
float(x1 + 1),
float(y1 + 1),
float(x2 + 1),
float(y2 + 1),
float(-1),
float(-1),
float(-1),
float(-1)
])
trackDB = np.array(res)
gtDB = read_txt_to_struct(gt_file)
# manipulate mot15 to fit mot16
#gtDB[:,7] = gtDB[:,6]
#gtDB[:,6] = 1
#gtDB[:,8] = 1
#gtDB = gtDB[:,:9]
gtDB, distractor_ids = extract_valid_gt_data(gtDB)
metrics, extra_info, clear_mot_info, ML_PT_MT = evaluate_sequence(
trackDB, gtDB, distractor_ids)
print_metrics(' Evaluation', metrics)
# evaluate sizes of ML_PT_MT
ML_PT_MT_sizes = [[]]
ML_PT_MT_vis = [[]]
for i, p in enumerate(ML_PT_MT, 1):
ML_PT_MT_sizes.append([])
ML_PT_MT_vis.append([])
for tr_id in p:
gt_in_person = np.where(gtDB[:, 1] == tr_id)[0]
#w = gtDB[gt_in_person, 4]
h = gtDB[gt_in_person, 5] - gtDB[gt_in_person, 3]
#sizes = (w*h).mean()
ML_PT_MT_sizes[i].append(h.mean())
ML_PT_MT_sizes[0].append(h.mean())
vis = gtDB[gt_in_person, 8]
ML_PT_MT_vis[i].append(vis.mean())
ML_PT_MT_vis[0].append(vis.mean())
print("\t\theight\tvis")
print("MT: \t\t{:.2f}\t{:.2f}".format(np.mean(ML_PT_MT_sizes[3]),
np.mean(ML_PT_MT_vis[3])))
print("PT: \t\t{:.2f}\t{:.2f}".format(np.mean(ML_PT_MT_sizes[2]),
np.mean(ML_PT_MT_vis[2])))
print("ML: \t\t{:.2f}\t{:.2f}".format(np.mean(ML_PT_MT_sizes[1]),
np.mean(ML_PT_MT_vis[1])))
print("total (mean of\t{:.2f}\t{:.2f}\ntrack means):".format(
np.mean(ML_PT_MT_sizes[0]), np.mean(ML_PT_MT_vis[0])))
return clear_mot_info
