def evaluate_tracking(sequences, track_dir, gt_dir): all_info = [] for seqname in sequences: track_res = os.path.join(track_dir, seqname, 'res.txt') gt_file = os.path.join(gt_dir, seqname, 'gt.txt') assert os.path.exists(track_res) and os.path.exists(gt_file), 'Either tracking result or groundtruth directory does not exist' trackDB = read_txt_to_struct(track_res) gtDB = read_txt_to_struct(gt_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) metrics, extra_info = evaluate_sequence(trackDB, gtDB, distractor_ids) print_metrics(seqname + ' Evaluation', metrics) all_info.append(extra_info) all_metrics = evaluate_bm(all_info) print_metrics('Summary Evaluation', all_metrics)
def my_main(_config): print(_config) ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") ###### PFADE ANPASSEN ########################## results_dir = osp.join('output/tracktor/MOT17') ################################################ output_dir = osp.join(results_dir, 'eval/det_gaps') if not osp.exists(output_dir): os.makedirs(output_dir) sequences_raw = [ "MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] ############### DETECTIONS ANPASSEN ############# detections = "SDP" ################################################ sequences = ["{}-{}".format(s, detections) for s in sequences_raw] #sequences = sequences[:1] #tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"] tracker = ["Tracktor", "Tracktor++", "FWT", "jCC", "MOTDT17", "MHT_DAM"] #tracker = ["Baseline"] # "PHD_GSDL17" does not work, error #tracker = tracker[-4:] results = [] det_gaps = [] for t in tracker: print("[*] Evaluating {}".format(t)) det_gaps_coverage = [] for s in sequences: ######################################## # Get DPM / GT coverage for each track # ######################################## ###### PFADE ANPASSEN ############################################## gt_file = osp.join("data/MOT17Labels", "train", s, "gt", "gt.txt") det_file = osp.join("data/MOT17Labels", "train", s, "det", "det.txt") res_file = osp.join("output/tracktor/MOT17", t, s + ".txt") #################################################################### #gtDB = read_txt_to_struct(gt_file) #gtDB = gtDB[gtDB[:,7] == 1] stDB = read_txt_to_struct(res_file) gtDB = read_txt_to_struct(gt_file) dtDB = read_txt_to_struct(det_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) print(s) print(len(np.unique(gtDB[:, 1]))) _, M, gtDB = evaluate_new(stDB, gtDB, distractor_ids) print(len(np.unique(gtDB[:, 1]))) gt_ids_old = np.unique(gtDB[:, 1]) # reaload gtDB as entries not found are deleted gtDB = read_txt_to_struct(gt_file) # filter out so that confidence and id = 1 gtDB = gtDB[gtDB[:, 7] == 1] gtDB = gtDB[gtDB[:, 6] == 1] gt_frames = np.unique(gtDB[:, 0]) #st_ids = np.unique(stDB[:, 1]) #gt_ids = np.unique(gtDB[:, 1]) #dt_ids = np.unique(dtDB[:, 1]) # id always -1 f_gt = len(gt_frames) #n_gt = len(gt_ids) #n_st = len(st_ids) gt_inds = [{} for i in range(f_gt)] #st_inds = [{} for i in range(f_gt)] dt_inds = [{} for i in range(f_gt)] D = [{} for i in range(f_gt)] #map detections to gt m_keys = [] for v in M: m_keys.append(list(v.keys())) #print("new: {}".format(np.unique(gtDB[:, 1]))) #print("old: {}".format(gt_ids_old)) #print("M: {}".format(np.unique(m_keys))) # hash the indices to speed up indexing #for i in range(gtDB.shape[0]): # frame = np.where(gt_frames == gtDB[i, 0])[0][0] # gid = np.where(gt_ids == gtDB[i, 1])[0][0] # gt_inds[frame][gid] = i for i in range(gtDB.shape[0]): frame = np.where(gt_frames == gtDB[i, 0])[0][0] gt_id = gtDB[i, 1] gt_inds[frame][gt_id] = i gt_frames_list = list(gt_frames) """ for i in range(stDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment frame = gt_frames_list.index(stDB[i, 0]) sid = np.where(st_ids == stDB[i, 1])[0][0] st_inds[frame][sid] = i """ for i in range(dtDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment frame = gt_frames_list.index(dtDB[i, 0]) # id always -1, so just make up numer, not important here did = 0 if len(dt_inds[frame]) > 0: did = max(dt_inds[frame].keys()) + 1 #did = np.where(dt_ids == dtDB[i, 1])[0][0] dt_inds[frame][did] = i # find detections <-> gt for frame in range(f_gt): gt_id = list(gt_inds[frame].keys()) gt_i = list(gt_inds[frame].values()) gt_boxes = gtDB[gt_i, 2:6] #print(gt_id) dt_id = list(dt_inds[frame].keys()) dt_i = list(dt_inds[frame].values()) dt_boxes = dtDB[dt_i, 2:6] #print(len(dt_id)) overlaps = np.zeros((len(dt_i), len(gt_i)), dtype=float) for i in range(len(gt_i)): overlaps[:, i] = bbox_overlap(dt_boxes, gt_boxes[i]) matched_indices = linear_assignment(1 - overlaps) for matched in matched_indices: if overlaps[matched[0], matched[1]] >= 0.5: did = dt_id[matched[0]] gid = gt_id[matched[1]] # gid in this case is the real gt id as it comes from the new gtDB D[frame][gid] = did for frame in range(f_gt): d = D[frame] for gt_id, _ in d.items(): gid = -1 gid_ind = np.where(gt_ids_old == gt_id)[0] if len(gid_ind) > 0: gid = gid_ind[0] # every gap is only handled once, at the last detection before the gap begins # so here we are in the middle of the gap => do nothing if gt_id not in d.keys(): continue # OK let's check if we have a gap in the detections next_non_empty = -1 for j in range(frame + 1, f_gt): if gt_id in D[j].keys(): next_non_empty = j break # no gap as no more detections, or no gap as next frame has detection if next_non_empty == -1 or next_non_empty == frame + 1: continue # now we have a gap in detections. Let's check the gap length and track coverage! gap_length = next_non_empty - frame - 1 count_tracked = 0 for j in range(frame + 1, next_non_empty): if gid in M[j].keys(): count_tracked += 1 if gap_length > 115 and gap_length < 135: print(count_tracked) det_gaps_coverage.append( [gap_length, count_tracked / gap_length]) # also add to gt distribution if t == tracker[0]: det_gaps.append(gap_length) """ # Now check all detections if they were tracked for frame in range(f_gt): matched_dets = D[frame] for did, gid in matched_dets.items(): # not matched by tracker, get visbility line = gt_inds[frame][gid] vis = gtDB[line, 8] # if matched only visibility is important if gid in M[frame].keys(): tracked_visibilities.append(vis) continue else: missed_visibilities.append(vis) # get distance to last time tracked last_non_empty = frame for j in range(frame, -1, -1): if gid in M[j].keys(): last_non_empty = j break next_non_empty = frame for j in range(frame, f_gt): if gid in M[j]: next_non_empty = j break dist = min(frame-last_non_empty, next_non_empty-frame) # check if not tracked at all ... if dist > 0: missed_distances.append(dist)""" det_gaps_coverage = np.array(det_gaps_coverage) results.append(det_gaps_coverage) #gaps = np.unique(det_gaps_coverage[:,0]) #coverage = np.zeros(gaps.shape[0]) #for i,g in enumerate(gaps): # covs = det_gaps_coverage[det_gaps_coverage[:,0] == g, 1] # coverage[i] = np.mean(covs) print(" Gaps: {}".format(len(det_gaps_coverage))) print(" Gaps>20 with coverage>0.8: {}".format( ((det_gaps_coverage[:, 1] > 0.8) * (det_gaps_coverage[:, 0] > 20)).sum())) #plt.figure() #sns.regplot(x=gaps, y=coverage, ci=None, scatter=True, scatter_kws={"s": 5}, line_kws={"linewidth":1.0}, lowess=True, label=t) #plt.legend() #plt.ylabel('coverage of gap by tracker') #plt.xlim((0,x_max)) #plt.xlabel('gap length in detections') #plt.legend() #plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('DET_GAP_COV', detections, t)), format='pdf') #plt.plot([0,1], [0,1], 'r-') """ plt.figure() plt.hist(tracked_visibilities, bins=20, density=False) plt.ylabel('occurence') plt.xlabel('visibility of target') plt.title('Boxes total: {}'.format(len(tracked_visibilities))) plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'TR_VIS')), format='pdf') #plt.plot([0,1], [0,1], 'r-') weights = np.zeros(missed_gaps_length.shape) for i in range(missed_gaps_length.shape[0]): weights[i] = 1/missed_gaps_length[i] #weights[i] = 1 plt.figure() #plt.xlim((0, 40)) bins = list(range(40)) plt.hist(missed_gaps_length, weights=weights, bins=bins, density=False) plt.ylabel('occurence') plt.xlabel('gap length in tracks') plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('MISS_DIST', t, detections)), format='pdf') x = np.arange(1,41) y = np.zeros(len(x)) ges_occurencies = 0 for i,b in enumerate(x): occurencies = int((missed_gaps_length==b).sum()) occurencies = occurencies/b ges_occurencies += occurencies y[i] = occurencies #y = y/ges_occurencies y_poly = np.poly1d(np.polyfit(x, y, 5)) poly_missed_gaps_length.append(y_poly) plt.figure() plt.hist(missed_visibilities, bins=20, density=False) plt.ylabel('occurence') #plt.xlim((0, xmax)) plt.xlabel('visibility of target') plt.title('Boxes total: {}'.format(len(missed_visibilities))) plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'MISS_VIS')), format='pdf') plt.close() # combined plot plt.figure() plt.hist([tracked_visibilities, missed_visibilities], bins=20, density=True, label=['tracked', 'missed']) plt.ylabel('occurence') plt.xlabel('visibility of target') plt.legend() #plt.title('Boxes total: {}'.format(len(tracked_visibilities))) plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('VIS', t, detections)), format='pdf') # height 0.9 <= vis <= 1.0 tracked_visibilities_heights = tracked_visibilities_heights[tracked_visibilities >= 0.9] missed_visibilities_heights = missed_visibilities_heights[missed_visibilities >= 0.9] plt.figure() plt.hist([tracked_visibilities_heights, missed_visibilities_heights], bins=15, density=True, label=['tracked', 'missed']) plt.ylabel('occurence') plt.xlabel('height of targets') plt.legend() #plt.title('Boxes total: {}'.format(len(tracked_visibilities))) plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('HEIGHTS', t, detections)), format='pdf') """ """ plt.figure() x_new = np.linspace(0, x_max, num=101, endpoint=True) for y,t in zip(y_poly_list, tracker): plt.plot(x_new, y_poly(x_new), label=t) plt.ylabel('coverage of gap by tracker') plt.xlim((0,x_max)) plt.xlabel('gap length in detections') plt.legend() plt.savefig(osp.join(output_dir, "ALL-{}-{}.pdf".format(detections, 'DET_GAP_COV')), format='pdf') """ #plt.legend() #plt.ylabel('coverage of gap by tracker') #plt.xlim((0,x_max)) #plt.xlabel('gap length in detections') #plt.legend() #plt.savefig(osp.join(output_dir, "ALL-{}-{}_xmax20.pdf".format('DET_GAP_COV', detections)), format='pdf') ########### AB HIER INTERESSANT FÜR DICH ############################## # generate bins x_max = 30 n_bins = 8 fontsize = 16 tickfontsize = 12 plt.rc('text', usetex=True) plt.rc('font', family='serif') step_size = (x_max - np.min(det_gaps)) / n_bins bins = np.arange(0, n_bins) * step_size + np.min(det_gaps) fig, ax1 = plt.subplots() plt.tick_params(labelsize=tickfontsize) ax1.spines['top'].set_visible(False) bar_width = (0.7 * step_size) / len(tracker) det_gaps = np.array(det_gaps) det_gaps = det_gaps[det_gaps < x_max] for t in range(len(tracker)): ratios = np.zeros(bins.shape[0]) data = results[t] print("Datapoints: {}".format(data.shape[0])) data = data[data[:, 0] < x_max] #print("After filter: {}".format(data.shape[0])) for i in range(bins.shape[0]): lower = bins[i] print("BIN: {}".format(lower)) tmp = data[data[:, 0] >= lower] #print("After lower: {}".format(tmp.shape[0])) if i != bins.shape[0] - 1: upper = bins[i + 1] tmp = tmp[tmp[:, 0] < upper] print("Datapoints in bin: {}".format(tmp.shape[0])) ratios[i] = np.mean(tmp[:, 1]) #ratios[i] = np.median(data[:,1]) #print(ratios) dis = (t - (len(tracker) - 1) / 2) * bar_width plt.bar(bins + step_size / 2 + dis, ratios, bar_width, align='center', label=tracker[t].replace('_', '\_')) color = 'black' if "FRCNN" in detections: color = 'white' if "SDP" in detections: color = 'white' plt.ylabel('Tracked objects [\%]', fontsize=fontsize, color=color) plt.xlim((np.min(det_gaps), x_max)) plt.ylim((0, 1.0)) color = 'black' if "DPM" in detections: color = 'white' if "SDP" in detections: color = 'white' plt.xlabel('Gap length in public detections', fontsize=fontsize, color=color) if "FRCNN" in detections: plt.legend(loc='upper right', fontsize=tickfontsize) ax2 = ax1.twinx() ax2.spines['top'].set_visible(False) #sns.kdeplot(det_gaps, cut=0, color="red", shade=True, linewidth=0) sns.distplot(det_gaps, bins=8, color="red", norm_hist=False, ax=ax2, kde=False, hist_kws={ "rwidth": 0.95, 'range': (np.min(det_gaps), x_max) }) ax2.tick_params(labelsize=tickfontsize) #plt.setp(ax2.get_yticklabels(), visible=False) #ax2.tick_params(axis='y', which='both', length=0) color = 'black' if "DPM" in detections: color = 'white' if "FRCNN" in detections: color = 'white' ax2.set_ylabel('Occurrences in detections', fontsize=fontsize, color=color) ax2.set_zorder(1) ax1.set_zorder(2) ax1.patch.set_visible(False) plt.savefig(osp.join(output_dir, "BAR-{}-{}.pdf".format('DET_GAP_COV', detections)), format='pdf', bbox_inches='tight')
def my_main(_config): print(_config) ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") ###################### PFAD ANPASSEN ################### results_dir = osp.join('output/tracktor/MOT17') ######################################################## output_dir = osp.join(results_dir, 'eval/track_missed_found') if not osp.exists(output_dir): os.makedirs(output_dir) sequences_raw = [ "MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] ######### HIER DETECTIONS ÄNDERNS (DPM, FRCNN, SDP) # detections = "SDP" ##################################################### sequences = ["{}-{}".format(s, detections) for s in sequences_raw] #sequences = sequences[:1] #tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"] tracker = ["Tracktor", "Tracktor++", "FWT", "jCC", "MOTDT17", "MHT_DAM"] #tracker = tracker[:2] # "PHD_GSDL17" does not work, error #tracker = tracker[-4:] poly_missed_gaps_length = [] tv = [] th = [] gt_vis = [] gt_h = [] for t in tracker: print("[*] Evaluating {}".format(t)) missed_gaps_length = [] missed_visibilities = [] tracked_visibilities = [] tracked_visibilities_heights = [] missed_visibilities_heights = [] for s in sequences: ######################################## # Get DPM / GT coverage for each track # ######################################## ################## PFADE ANPASSEN ####################### gt_file = osp.join("data/MOT17Labels", "train", s, "gt", "gt.txt") # det_file = osp.join("../data/MOT17Labels", "train", s, "det", "det.txt") res_file = osp.join("output/tracker/MOT17", t, s + ".txt") # gt_file = osp.join("MOT17Labels", "train", s, "gt", "gt.txt") # #det_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "det", "det.txt") # res_file = osp.join("track_missed_found", t, s+".txt") ######################################################### #gtDB = read_txt_to_struct(gt_file) #gtDB = gtDB[gtDB[:,7] == 1] stDB = read_txt_to_struct(res_file) gtDB = read_txt_to_struct(gt_file) #dtDB = read_txt_to_struct(det_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) _, M, gtDB = evaluate_new(stDB, gtDB, distractor_ids) # reaload gtDB as entries not found are deleted gtDB_unf = read_txt_to_struct(gt_file) # filter out so that confidence and id = 1 gtDB_unf = gtDB_unf[gtDB_unf[:, 7] == 1] gtDB_unf = gtDB_unf[gtDB_unf[:, 6] == 1] if t == tracker[0]: for vis in gtDB_unf[:, 8]: gt_vis.append(vis) gt_temp = gtDB_unf[gtDB_unf[:, 8] >= 0.9] heights = gt_temp[:, 5] - gt_temp[:, 3] if "-05-" in s: heights *= 2.25 for h in heights: gt_h.append(h) gt_frames = np.unique(gtDB[:, 0]) st_ids = np.unique(stDB[:, 1]) gt_ids = np.unique(gtDB[:, 1]) #dt_ids = np.unique(dtDB[:, 1]) f_gt = len(gt_frames) n_gt = len(gt_ids) n_st = len(st_ids) #gt_inds = [{} for i in range(f_gt)] #st_inds = [{} for i in range(f_gt)] #dt_inds = [{} for i in range(f_gt)] gt_inds_unf = [{} for i in range(f_gt)] #D = [{} for i in range(f_gt)] #map detections to gt # hash the indices to speed up indexing #for i in range(gtDB.shape[0]): # frame = np.where(gt_frames == gtDB[i, 0])[0][0] # gid = np.where(gt_ids == gtDB[i, 1])[0][0] # gt_inds[frame][gid] = i for i in range(gtDB_unf.shape[0]): frame = np.where(gt_frames == gtDB_unf[i, 0])[0][0] gt_id = gtDB_unf[i, 1] gt_inds_unf[frame][gt_id] = i #gt_frames_list = list(gt_frames) #for i in range(stDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment # frame = gt_frames_list.index(stDB[i, 0]) # sid = np.where(st_ids == stDB[i, 1])[0][0] # st_inds[frame][sid] = i for frame, m in enumerate(M): for gt_id, line in gt_inds_unf[frame].items(): vis = gtDB_unf[line, 8] height = gtDB_unf[line, 5] - gtDB_unf[line, 3] if "-05-" in s: height *= 2.25 gid = -1 gid_ind = np.where(gt_ids == gt_id)[0] if len(gid_ind) > 0: gid = gid_ind[0] # check if tracked if gid in m.keys(): tracked_visibilities.append(vis) tracked_visibilities_heights.append(height) continue else: missed_visibilities.append(vis) missed_visibilities_heights.append(height) # get distance to last time tracked last_non_empty = -1 for j in range(frame, -1, -1): if gid in M[j].keys(): last_non_empty = j break next_non_empty = -1 for j in range(frame, f_gt): if gid in M[j].keys(): next_non_empty = j break if next_non_empty != -1 and last_non_empty != -1: gap_length = next_non_empty - last_non_empty - 1 missed_gaps_length.append(gap_length) """ for i in range(dtDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment frame = gt_frames_list.index(dtDB[i, 0]) did = np.where(dt_ids == dtDB[i, 1])[0][0] dt_inds[frame][did] = i # find track <-> gt for frame in range(f_gt): gt_ids = list(gt_inds[frame].keys()) gt_i = list(gt_inds[frame].values()) gt_boxes = gtDB[gt_i, 2:6] st_ids = list(st_inds[frame].keys()) st_i = list(st_inds[frame].values()) st_boxes = stDB[st_i, 2:6] overlaps = np.zeros((len(st_i), len(gt_i)), dtype=float) for i in range(len(gt_i)): overlaps[:, i] = bbox_overlap(st_boxes, gt_boxes[i]) matched_indices = linear_assignment(1 - overlaps) for matched in matched_indices: if overlaps[matched[0], matched[1]] > 0.5: did = st_ids[matched[0]] gid = gt_ids[matched[1]] D[frame][did] = gid # Now check all detections if they were tracked for frame in range(f_gt): matched_dets = D[frame] for did, gid in matched_dets.items(): # not matched by tracker, get visbility line = gt_inds[frame][gid] vis = gtDB[line, 8] # if matched only visibility is important if gid in M[frame].keys(): tracked_visibilities.append(vis) continue else: missed_visibilities.append(vis) # get distance to last time tracked last_non_empty = frame for j in range(frame, -1, -1): if gid in M[j].keys(): last_non_empty = j break next_non_empty = frame for j in range(frame, f_gt): if gid in M[j]: next_non_empty = j break dist = min(frame-last_non_empty, next_non_empty-frame) # check if not tracked at all ... if dist > 0: missed_distances.append(dist)""" missed_gaps_length = np.array(missed_gaps_length) missed_visibilities = np.array(missed_visibilities) tracked_visibilities = np.array(tracked_visibilities) tracked_visibilities_heights = np.array(tracked_visibilities_heights) missed_visibilities_heights = np.array(missed_visibilities_heights) #plt.plot([0,1], [0,1], 'r-') """ plt.figure() plt.hist(tracked_visibilities, bins=20, density=False) plt.ylabel('occurence') plt.xlabel('visibility of target') plt.title('Boxes total: {}'.format(len(tracked_visibilities))) plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'TR_VIS')), format='pdf') """ #plt.plot([0,1], [0,1], 'r-') weights = np.zeros(missed_gaps_length.shape) for i in range(missed_gaps_length.shape[0]): weights[i] = 1 / missed_gaps_length[i] #weights[i] = 1 #plt.figure() #plt.xlim((0, 40)) #bins = list(range(40)) #plt.hist(missed_gaps_length, weights=weights, bins=bins, density=False) #plt.ylabel('occurence') #plt.xlabel('gap length in tracks') #plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('MISS_DIST', t, detections)), format='pdf') x = np.arange(1, 41) y = np.zeros(len(x)) ges_occurencies = 0 for i, b in enumerate(x): occurencies = int((missed_gaps_length == b).sum()) occurencies = occurencies / b ges_occurencies += occurencies y[i] = occurencies #y = y/ges_occurencies y_poly = np.poly1d(np.polyfit(x, y, 5)) poly_missed_gaps_length.append(y_poly) """ plt.figure() plt.hist(missed_visibilities, bins=20, density=False) plt.ylabel('occurence') #plt.xlim((0, xmax)) plt.xlabel('visibility of target') plt.title('Boxes total: {}'.format(len(missed_visibilities))) plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'MISS_VIS')), format='pdf') plt.close() """ # combined plot # plt.figure() # plt.hist([tracked_visibilities, missed_visibilities], bins=20, density=True, label=['tracked', 'missed']) # plt.ylabel('occurence') # plt.xlabel('visibility of target') # plt.legend() #plt.title('Boxes total: {}'.format(len(tracked_visibilities))) # plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('VIS', t, detections)), format='pdf') # height 0.9 <= vis <= 1.0 tracked_visibilities_heights = tracked_visibilities_heights[ tracked_visibilities >= 0.9] missed_visibilities_heights = missed_visibilities_heights[ missed_visibilities >= 0.9] # plt.figure() # plt.hist([tracked_visibilities_heights, missed_visibilities_heights], bins=15, density=True, label=['tracked', 'missed']) # plt.ylabel('occurence') # plt.xlabel('height of targets') # plt.legend() #plt.title('Boxes total: {}'.format(len(tracked_visibilities))) # plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format('HEIGHTS', t, detections)), format='pdf') tv.append(tracked_visibilities) th.append(tracked_visibilities_heights) h_max = 250 h_n_bins = 8 v_n_bins = 12 gt_h = np.array(gt_h) gt_vis = np.array(gt_vis) tv = np.array(tv) th = np.array(th) h_step = (h_max - np.min(gt_h)) / h_n_bins v_step = 1.0 / v_n_bins gt_h = gt_h[gt_h < h_max] h_bins = np.arange(0, h_n_bins) * h_step + np.min(gt_h) v_bins = np.arange(0, v_n_bins) * v_step gt_h_occurencies = np.zeros(h_bins.shape[0]) for i in range(h_bins.shape[0]): lower = h_bins[i] tmp = gt_h[gt_h >= lower] if i != h_bins.shape[0] - 1: upper = h_bins[i + 1] tmp = tmp[tmp < upper] gt_h_occurencies[i] = len(tmp) gt_v_occurencies = np.zeros(v_bins.shape[0]) for i in range(v_bins.shape[0]): lower = v_bins[i] tmp = gt_vis[gt_vis >= lower] if i != v_bins.shape[0] - 1: upper = v_bins[i + 1] tmp = tmp[tmp < upper] gt_v_occurencies[i] = len(tmp) heights_results = [] for i in range(len(tracker)): t = tracker[i] heights = th[i] heights = heights[heights < h_max] res = [] for i in range(h_bins.shape[0]): lower = h_bins[i] tmp = heights[heights >= lower] if i != h_bins.shape[0] - 1: upper = h_bins[i + 1] tmp = tmp[tmp < upper] percentage = len(tmp) / gt_h_occurencies[i] res.append(percentage) heights_results.append(np.array(res)) vis_results = [] for i in range(len(tracker)): t = tracker[i] vis = tv[i] res = [] for i in range(v_bins.shape[0]): lower = v_bins[i] tmp = vis[vis >= lower] if i != v_bins.shape[0] - 1: upper = v_bins[i + 1] tmp = tmp[tmp < upper] percentage = len(tmp) / gt_v_occurencies[i] res.append(percentage) vis_results.append(np.array(res)) plt.rc('text', usetex=True) plt.rc('font', family='serif') fontsize = 16 tickfontsize = 12 ############ AB HIER WICHTIG FÜR DICH ########################## # Plot heights fig, ax1 = plt.subplots() ax2 = ax1.twinx() ax2.set_zorder(1) ax1.set_zorder(2) ax1.patch.set_visible(False) ax2.spines['top'].set_visible(False) #sns.distplot(gt_h) #sns.kdeplot(gt_h, cut=0, color="red", shade=True, linewidth=0) sns.distplot(gt_h, bins=8, color="red", norm_hist=False, ax=ax2, kde=False, hist_kws={ "rwidth": 0.95, 'range': (np.min(gt_h), h_max) }) #plt.setp(ax3.get_xticklabels(), visible=False) #plt.setp(ax2.get_yticklabels(), visible=False) #ax2.tick_params(axis='y', which='both', length=0, labelsize=fontsize) ax2.tick_params(labelsize=tickfontsize) #ax2.set_ylabel('Occurrence in tracking ground truth (\%)') color = 'black' if "DPM" in detections: color = 'white' if "FRCNN" in detections: color = 'white' ax2.set_ylabel('Occurrences in ground truth', fontsize=fontsize, color=color) #ax2.set_ylim((0,h_max)) #sns.kdeplot(gt_h, shade=True, cut=0) ax1.spines['top'].set_visible(False) bar_width = (0.7 * h_step) / len(tracker) for i in range(len(tracker)): dis = (i - (len(tracker) - 1) / 2) * bar_width ax1.bar(h_bins + h_step / 2 + dis, heights_results[i], bar_width, align='center', label=tracker[i].replace('_', '\_')) ax1.set_ylim((0, 1.0)) ax1.set_xlim((np.min(gt_h), h_max)) #if "DPM" in detections: # plt.legend(fontsize=fontsize) plt.tick_params(labelsize=tickfontsize) color = 'black' if "DPM" in detections: color = 'white' if "SDP" in detections: color = 'white' ax1.set_xlabel('Object height (pixels)', fontsize=fontsize, color=color) color = 'black' if "FRCNN" in detections: color = 'white' if "SDP" in detections: color = 'white' ax1.set_ylabel('Tracked objects (\%)', fontsize=fontsize, color=color) #plt.title('height distribution from highly visible targets (>= 0.9)') plt.savefig(osp.join(output_dir, "heights09-{}.pdf".format(detections)), format='pdf', bbox_inches='tight') if "FRCNN" in detections: # Plot vis fig, ax1 = plt.subplots() ax2 = ax1.twinx() ax2.set_zorder(1) ax1.set_zorder(2) ax1.patch.set_visible(False) ax2.spines['top'].set_visible(False) #sns.kdeplot(gt_vis, cut=0, color="red", shade=True, linewidth=0, ax=ax2) sns.distplot(gt_vis, bins=12, color="red", norm_hist=False, ax=ax2, kde=False, hist_kws={"rwidth": 0.95}) ax2.tick_params(labelsize=tickfontsize) #plt.setp(ax2.get_yticklabels(), visible=False) #ax2.tick_params(axis='y', which='both', length=0) #ax2.set_ylabel('Occurrence in tracking ground truth (\%)') ax2.set_ylabel('Occurrences in ground truth', fontsize=fontsize) #ax2.ylim((0,0.006)) ax1.spines['top'].set_visible(False) #plt.hist(vis_results, weights=vis_weights, label=tracker) #sns.distplot(gt_vis) #sns.kdeplot(gt_vis, bw=1.0, label="bw: 1.0", cut=0) #sns.kdeplot(gt_vis, bw=0.2, label="bw: 0.2", cut=0) #sns.kdeplot(gt_vis, bw=2, label="bw: 2", cut=0) bar_width = (0.7 * v_step) / len(tracker) for i in range(len(tracker)): dis = (i - (len(tracker) - 1) / 2) * bar_width ax1.bar(v_bins + v_step / 2 + dis, vis_results[i], bar_width, align='center', label=tracker[i].replace('_', '\_')) ax1.set_ylim((0, 1.0)) ax1.set_xlim((0, 1.0)) ax1.tick_params(labelsize=tickfontsize) ax1.legend(loc='upper left', fontsize=tickfontsize) ax1.set_xlabel('Object visibility (\%)', fontsize=fontsize) ax1.set_ylabel('Tracked objects (\%)', fontsize=fontsize) #plt.title('visibility distribution of all targets') plt.savefig(osp.join(output_dir, "vis-{}.pdf".format(detections)), format='pdf', bbox_inches='tight') """
def my_main(_config): print(_config) dataset = "mot_train_" detections = "FRCNN" ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") module_dir = get_output_dir('MOT17') results_dir = module_dir module_dir = osp.join(module_dir, 'eval/video_red_green') #output_dir = osp.join(results_dir, 'plots') #if not osp.exists(output_dir): # os.makedirs(output_dir) #sequences_raw = ["MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] #sequences = ["{}-{}".format(s, detections) for s in sequences_raw] #sequences = sequences[:1] # tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"] # tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17", "MHT_DAM_17"] tracker = ["FWT", "jCC", "MOTDT17"]#, "Tracktor++"] baseline = "Tracktor" for t in tracker: print("[*] Evaluating {}".format(t)) for db in Datasets(dataset): ################################ # Make videos for each tracker # ################################ s = "{}-{}".format(db, detections) gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt") res_file = osp.join(results_dir, t, s+".txt") base_file = osp.join(results_dir, baseline, s+".txt") stDB = read_txt_to_struct(res_file) gtDB = read_txt_to_struct(gt_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) _, M_res, gtDB, stDB = evaluate_new(stDB, gtDB, distractor_ids) gt_ids_res = np.unique(gtDB[:, 1]) bsDB = read_txt_to_struct(base_file) gtDB = read_txt_to_struct(gt_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) _, M_bs, gtDB, stDB = evaluate_new(bsDB, gtDB, distractor_ids) gt_ids_base = np.unique(gtDB[:, 1]) gtDB = read_txt_to_struct(gt_file) # filter out so that confidence and id = 1 gtDB = gtDB[gtDB[:,7] == 1] gtDB = gtDB[gtDB[:,6] == 1] #st_ids = np.unique(stDB[:, 1]) #gt_ids = np.unique(gtDB[:, 1]) gt_frames = np.unique(gtDB[:, 0]) f_gt = len(gt_frames) gt_inds = [{} for i in range(f_gt)] #st_inds = [{} for i in range(f_gt)] # hash the indices to speed up indexing for i in range(gtDB.shape[0]): frame = np.where(gt_frames == gtDB[i, 0])[0][0] #gid = np.where(gt_ids == gtDB[i, 1])[0][0] gt_id = int(gtDB[i,1]) gt_inds[frame][gt_id] = i #gt_frames_list = list(gt_frames) #for i in range(stDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment # frame = gt_frames_list.index(stDB[i, 0]) # sid = np.where(st_ids == stDB[i, 1])[0][0] # st_inds[frame][sid] = i results = [] for frame in range(f_gt): # get gt_ids in res m_res = M_res[frame] gids = list(m_res.keys()) res_gt = [] for gid in gids: res_gt.append(gt_ids_res[gid]) # get gt_ids in base m_bs = M_bs[frame] gids = list(m_bs.keys()) base_gt = [] for gid in gids: base_gt.append(gt_ids_base[gid]) # get unique gt ids unique_gt = np.unique(res_gt + base_gt) #print("res gt: {}".format(res_gt)) #print("base gt: {}".format(base_gt)) for gt in unique_gt: gt = int(gt) #print(gt) res = np.zeros(6) res[0] = frame+1 res[2:6] = gtDB[gt_inds[frame][gt], 2:6] if gt in res_gt and gt in base_gt: res[1] = 1 elif gt in base_gt: res[1] = 2 elif gt in res_gt: res[1] = 3 results.append(res) results = np.array(results) output_dir = osp.join(module_dir, t, s) if not osp.exists(output_dir): os.makedirs(output_dir) print("[*] Plotting whole sequence to {}".format(output_dir)) # infinte color loop cyl = cy('ec', colors) loop_cy_iter = cyl() styles = defaultdict(lambda : next(loop_cy_iter)) for frame,v in enumerate(db,1): im_path = v['im_path'] im_name = osp.basename(im_path) im_output = osp.join(output_dir, im_name) im = cv2.imread(im_path) im = im[:, :, (2, 1, 0)] sizes = np.shape(im) height = float(sizes[0]) width = float(sizes[1]) fig = plt.figure() #fig.set_size_inches(w,h) #fig.set_size_inches(width/height, 1, forward=False) #fig.set_size_inches(width/100, height/100) scale = width/640 #fig.set_size_inches(640/100, height*scale/100) fig.set_size_inches(width/100, height/100) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.set_axis_off() fig.add_axes(ax) ax.imshow(im) res_frame = results[results[:,0]==frame] for j in range(res_frame.shape[0]): box = res_frame[j,2:6] gt_id = int(res_frame[j,1]) ax.add_patch( plt.Rectangle((box[0], box[1]), box[2] - box[0], box[3] - box[1], fill=False, linewidth=1.3*scale, color=colors[gt_id]) #**styles[gt_id]) ) plt.axis('off') #plt.tight_layout() plt.draw() plt.savefig(im_output, dpi=100) plt.close()
def my_main(_config): print(_config) dataset = "mot_train_" detections = "SDP" ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") module_dir = get_output_dir('videos') results_dir = osp.join(module_dir, 'results') module_dir = osp.join(module_dir, 'normal_new') #output_dir = osp.join(results_dir, 'plots') #if not osp.exists(output_dir): # os.makedirs(output_dir) #sequences_raw = ["MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] #sequences = ["{}-{}".format(s, detections) for s in sequences_raw] #sequences = sequences[:1] tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"] tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17", "MHT_DAM_17"] tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17"] for t in tracker: print("[*] Evaluating {}".format(t)) for db in Datasets(dataset): ################################ # Make videos for each tracker # ################################ s = "{}-{}".format(db, detections) gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt") res_file = osp.join(results_dir, t, s+".txt") stDB = read_txt_to_struct(res_file) gtDB = read_txt_to_struct(gt_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) _, M, gtDB, stDB = evaluate_new(stDB, gtDB, distractor_ids) st_ids = np.unique(stDB[:, 1]) gt_ids = np.unique(gtDB[:, 1]) gt_frames = np.unique(gtDB[:, 0]) f_gt = len(gt_frames) gt_inds = [{} for i in range(f_gt)] st_inds = [{} for i in range(f_gt)] # hash the indices to speed up indexing for i in range(gtDB.shape[0]): frame = np.where(gt_frames == gtDB[i, 0])[0][0] gid = np.where(gt_ids == gtDB[i, 1])[0][0] gt_inds[frame][gid] = i gt_frames_list = list(gt_frames) for i in range(stDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment frame = gt_frames_list.index(stDB[i, 0]) sid = np.where(st_ids == stDB[i, 1])[0][0] st_inds[frame][sid] = i # set all ids of st to -1 #stDB[:,1] = -1 # set all results to corresponding gt id #for frame in range(f_gt): # m = M[frame] # for gid, sid in m.items(): # gt_row = gt_inds[frame][gid] # gt_id = int(gtDB[gt_row, 1]) # st_row = st_inds[frame][sid] # stDB[st_row, 1] = gt_id output_dir = osp.join(module_dir, t, s) if not osp.exists(output_dir): os.makedirs(output_dir) print("[*] Plotting whole sequence to {}".format(output_dir)) # infinte color loop cyl = cy('ec', colors) loop_cy_iter = cyl() styles = defaultdict(lambda : next(loop_cy_iter)) for frame,v in enumerate(db,1): im_path = v['im_path'] im_name = osp.basename(im_path) im_output = osp.join(output_dir, im_name) im = cv2.imread(im_path) im = im[:, :, (2, 1, 0)] sizes = np.shape(im) height = float(sizes[0]) width = float(sizes[1]) fig = plt.figure() #fig.set_size_inches(w,h) #fig.set_size_inches(width/height, 1, forward=False) #fig.set_size_inches(width/100, height/100) scale = width/640 #fig.set_size_inches(640/100, height*scale/100) fig.set_size_inches(width/100, height/100) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.set_axis_off() fig.add_axes(ax) ax.imshow(im) st_frame = stDB[stDB[:,0]==frame] for j in range(st_frame.shape[0]): box = st_frame[j,2:6] gt_id = st_frame[j,1] ax.add_patch( plt.Rectangle((box[0], box[1]), box[2] - box[0], box[3] - box[1], fill=False, linewidth=1.3*scale, **styles[gt_id]) ) plt.axis('off') #plt.tight_layout() plt.draw() plt.savefig(im_output, dpi=100) plt.close()
def my_main(_config): print(_config) dataset = "mot_train_" detections = "FRCNN" ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") module_dir = get_output_dir('MOT17') results_dir = module_dir module_dir = osp.join(module_dir, 'eval/video_fp') #output_dir = osp.join(results_dir, 'plots') #if not osp.exists(output_dir): # os.makedirs(output_dir) #sequences_raw = ["MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] #sequences = ["{}-{}".format(s, detections) for s in sequences_raw] #sequences = sequences[:1] # tracker = ["FRCNN_Base", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17"] # tracker = ["Baseline", "BnW", "FWT_17", "jCC", "MOTDT17", "MHT_DAM_17"] tracker = ["Tracktor", "FWT", "jCC", "MOTDT17"] #tracker = ["Baseline"] for t in tracker: print("[*] Evaluating {}".format(t)) if True: #for db in Datasets(dataset): ################################ # Make videos for each tracker # ################################ db = Datasets(dataset)[2] s = "{}-{}".format(db, detections) gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt") res_file = osp.join(results_dir, t, s + ".txt") stDB = read_txt_to_struct(res_file) gtDB = read_txt_to_struct(gt_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) _, M, gtDB, stDB = evaluate_new(stDB, gtDB, distractor_ids) #gt_ids_res = np.unique(gtDB[:, 1]) #gtDB = read_txt_to_struct(gt_file) # filter out so that confidence and id = 1 #gtDB = gtDB[gtDB[:,7] == 1] #gtDB = gtDB[gtDB[:,6] == 1] st_ids = np.unique(stDB[:, 1]) #gt_ids = np.unique(gtDB[:, 1]) gt_frames = np.unique(gtDB[:, 0]) f_gt = len(gt_frames) #gt_inds = [{} for i in range(f_gt)] st_inds = [{} for i in range(f_gt)] # hash the indices to speed up indexing #for i in range(gtDB.shape[0]): # frame = np.where(gt_frames == gtDB[i, 0])[0][0] #gid = np.where(gt_ids == gtDB[i, 1])[0][0] # gt_id = int(gtDB[i,1]) # gt_inds[frame][gt_id] = i gt_frames_list = list(gt_frames) for i in range(stDB.shape[0]): # sometimes detection missed in certain frames, thus should be assigned to groundtruth frame id for alignment frame = gt_frames_list.index(stDB[i, 0]) sid = np.where(st_ids == stDB[i, 1])[0][0] st_inds[frame][sid] = i #stDB = read_txt_to_struct(res_file) results = [] for frame in range(f_gt): # get gt_ids in res m = M[frame] matched_sids = list(m.values()) #frame_sids = list(st_inds[frame].keys()) f = gt_frames_list[frame] st_frame = stDB[stDB[:, 0] == f] st_uniq_ids = np.unique(st_frame[:, 1]) for st_id in st_uniq_ids: sid = -1 si = np.where(st_ids == st_id)[0] if len(si) > 0: sid = si[0] if sid not in matched_sids: res = np.zeros(6) res[0] = frame + 1 st_track = st_frame[st_frame[:, 1] == st_id] res[2:6] = st_track[0, 2:6] results.append(res) else: matched_sids.remove(sid) results = np.array(results) print(results.shape[0]) output_dir = osp.join(module_dir, t, s) if not osp.exists(output_dir): os.makedirs(output_dir) print("[*] Plotting whole sequence to {}".format(output_dir)) # infinte color loop cyl = cy('ec', colors) loop_cy_iter = cyl() styles = defaultdict(lambda: next(loop_cy_iter)) for frame, v in enumerate(db, 1): im_path = v['im_path'] im_name = osp.basename(im_path) im_output = osp.join(output_dir, im_name) im = cv2.imread(im_path) im = im[:, :, (2, 1, 0)] sizes = np.shape(im) height = float(sizes[0]) width = float(sizes[1]) fig = plt.figure() #fig.set_size_inches(w,h) #fig.set_size_inches(width/height, 1, forward=False) #fig.set_size_inches(width/100, height/100) scale = width / 640 #fig.set_size_inches(640/100, height*scale/100) fig.set_size_inches(width / 100, height / 100) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.set_axis_off() fig.add_axes(ax) ax.imshow(im) res_frame = results[results[:, 0] == frame] for j in range(res_frame.shape[0]): box = res_frame[j, 2:6] gt_id = int(res_frame[j, 1]) ax.add_patch( plt.Rectangle((box[0], box[1]), box[2] - box[0], box[3] - box[1], fill=False, linewidth=1.3 * scale, color='blue')) ax.annotate(t, (width - 250, height - 100), color='white', weight='bold', fontsize=72, ha='center', va='center') plt.axis('off') plt.draw() plt.savefig(im_output, dpi=100) plt.close()
def my_main(_config): print(_config) ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") output_dir = osp.join(get_output_dir('MOT_analysis'), 'coverage') sequences_raw = [ "MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] detections = "DPM" sequences = ["{}-{}".format(s, detections) for s in sequences_raw] tracker = [ "FRCNN", "DMAN", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17" ] #tracker = ["DMAN"] for t in tracker: print("[*] Evaluating {}".format(t)) data_points = [] for s in sequences: ######################################## # Get DPM / GT coverage for each track # ######################################## gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt") dpm_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "det", "det.txt") gtDB = read_txt_to_struct(gt_file) dpmDB = read_txt_to_struct(dpm_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) dpmDB, gtDB = preprocessingDB(dpmDB, gtDB, distractor_ids, 0.5, 0) gt_ids = np.unique(gtDB[:, 1]) gt_ges = {int(i): 0 for i in gt_ids} gt_matched = {int(i): 0 for i in gt_ids} gt_tracked = {int(i): 0 for i in gt_ids} track_frames = np.unique(dpmDB[:, 0]) gt_frames = np.unique(gtDB[:, 0]) nframes = min(len(track_frames), len(gt_frames)) res_keep = np.ones((dpmDB.shape[0], ), dtype=float) for i in range(1, nframes + 1): # find all result boxes in this frame res_in_frame = np.where(dpmDB[:, 0] == i)[0] res_in_frame_data = dpmDB[res_in_frame, :] gt_in_frame = np.where(gtDB[:, 0] == i)[0] gt_in_frame_data = gtDB[gt_in_frame, :] #for gt in gt_in_frame_data: # gt_ges[int(gt[1])] += 1 res_num = res_in_frame.shape[0] gt_num = gt_in_frame.shape[0] overlaps = np.zeros((res_num, gt_num), dtype=float) for gid in range(gt_num): overlaps[:, gid] = bbox_overlap(res_in_frame_data[:, 2:6], gt_in_frame_data[gid, 2:6]) matched_indices = linear_assignment(1 - overlaps) for matched in matched_indices: # overlap lower than threshold, discard the pair if overlaps[matched[0], matched[1]] > 0.5: gt_id = int(gt_in_frame_data[matched[1], 1]) gt_matched[gt_id] += 1 for k in gt_ids: gt_ges[k] = len(np.where(gtDB[:, 1] == k)[0]) gt_tracked[k] = gt_matched[k] / gt_ges[k] res_file = osp.join(output_dir, t, s + ".txt") gtDB = read_txt_to_struct(gt_file) trackDB = read_txt_to_struct(res_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) trackDB, gtDB = preprocessingDB(trackDB, gtDB, distractor_ids, 0.5, 0) tr_matched = {int(i): 0 for i in gt_ids} tr_tracked = {int(i): 0 for i in gt_ids} track_frames = np.unique(trackDB[:, 0]) gt_frames = np.unique(gtDB[:, 0]) nframes = min(len(track_frames), len(gt_frames)) res_keep = np.ones((trackDB.shape[0], ), dtype=float) for i in range(1, nframes + 1): # find all result boxes in this frame res_in_frame = np.where(trackDB[:, 0] == i)[0] res_in_frame_data = trackDB[res_in_frame, :] gt_in_frame = np.where(gtDB[:, 0] == i)[0] gt_in_frame_data = gtDB[gt_in_frame, :] res_num = res_in_frame.shape[0] gt_num = gt_in_frame.shape[0] overlaps = np.zeros((res_num, gt_num), dtype=float) for gid in range(gt_num): overlaps[:, gid] = bbox_overlap(res_in_frame_data[:, 2:6], gt_in_frame_data[gid, 2:6]) matched_indices = linear_assignment(1 - overlaps) for matched in matched_indices: # overlap lower than threshold, discard the pair if overlaps[matched[0], matched[1]] > 0.5: gt_id = int(gt_in_frame_data[matched[1], 1]) tr_matched[gt_id] += 1 for k in gt_ids: data_points.append([gt_tracked[k], tr_matched[k] / gt_ges[k]]) data_points = np.array(data_points) # add mean values grid_step = 0.02 grid = np.arange(-grid_step / 2, 1.0 + grid_step, grid_step) x_mean = np.arange(0.0, 1.0 + grid_step, grid_step) bins = int(1.0 / grid_step) + 1 y_mean = np.zeros(bins) y_std = np.zeros(bins) for i in range(bins): vals = (data_points[:, 0] >= grid[i]) * (data_points[:, 0] < grid[i + 1]) mean = np.mean(data_points[vals, 1]) y_mean[i] = mean y_std[i] = np.sqrt(np.mean((vals - mean)**2)) y_poly = np.poly1d(np.polyfit(x_mean, y_mean, 5)) x_new = np.linspace(0, 1, num=101, endpoint=True) area = simps(y_poly(x_new), x_new) plt.plot(x_new, y_poly(x_new), label="{} {:.3f}".format(t, area)) #plt.errorbar(x_mean, y_poly(x_mean), yerr=y_std, fmt='o') #if t == "FRCNN": # plt.plot(x_mean, y_mean) #plt.plot([0,1], [0,1], 'r-') #plt.scatter(data_points[:,0], data_points[:,1], s=2**2) #plt.xlabel('{} coverage'.format(detections)) #plt.ylabel('tracker coverage') #plt.savefig(osp.join(output_dir, "{}-{}.pdf".format(t, detections)), format='pdf') #plt.close() plt.plot([0, 1], [0, 1], 'r-') plt.legend() plt.xlabel('{} coverage'.format(detections)) plt.ylabel('tracker coverage') plt.savefig(osp.join(output_dir, "coverage-{}.pdf".format(detections)), format='pdf') plt.close()
def my_main(_config): print(_config) ########################## # Initialize the modules # ########################## print("[*] Beginning evaluation...") output_dir = osp.join(get_output_dir('MOT_analysis'), 'occlusion') if not osp.exists(output_dir): os.makedirs(output_dir) sequences_raw = [ "MOT17-13", "MOT17-11", "MOT17-10", "MOT17-09", "MOT17-05", "MOT17-04", "MOT17-02", ] detections = "DPM" sequences = ["{}-{}".format(s, detections) for s in sequences_raw] tracker = [ "FRCNN", "DMAN", "HAM_SADF17", "MOTDT17", "EDMT17", "IOU17", "MHT_bLSTM", "FWT_17", "jCC", "MHT_DAM_17" ] #tracker = ["FRCNN"] # "PHD_GSDL17" does not work, error #tracker = tracker[-4:] for t in tracker: print("[*] Evaluating {}".format(t)) coverage = [] id_recovered = [] tr_id_recovered = [] for s in sequences: ######################################## # Get DPM / GT coverage for each track # ######################################## gt_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "gt", "gt.txt") det_file = osp.join(cfg.DATA_DIR, "MOT17Labels", "train", s, "det", "det.txt") res_file = osp.join(output_dir, t, s + ".txt") #gtDB = read_txt_to_struct(gt_file) #gtDB = gtDB[gtDB[:,7] == 1] stDB = read_txt_to_struct(res_file) gtDB = read_txt_to_struct(gt_file) gtDB, distractor_ids = extract_valid_gt_data(gtDB) _, M = evaluate_new(stDB, gtDB, distractor_ids) gt_frames = np.unique(gtDB[:, 0]) st_ids = np.unique(stDB[:, 1]) gt_ids = np.unique(gtDB[:, 1]) f_gt = len(gt_frames) n_gt = len(gt_ids) n_st = len(st_ids) gt_inds = [{} for i in range(f_gt)] st_inds = [{} for i in range(f_gt)] # hash the indices to speed up indexing for i in range(gtDB.shape[0]): frame = np.where(gt_frames == gtDB[i, 0])[0][0] gid = np.where(gt_ids == gtDB[i, 1])[0][0] gt_inds[frame][gid] = i # Loop thorugh all gt and find gaps (visibility < 0.5) visible = [[0 for j in range(f_gt)] for i in range(n_gt) ] # format visible[track][frame] = {0,1} for gid in range(n_gt): for frame in range(f_gt): if gid in gt_inds[frame]: line = gt_inds[frame][gid] vis = gtDB[line, 8] #print(vis, frame, gid) if vis >= 0.5: visible[gid][frame] = 1 # Find gaps in the tracks gt_tracked = {} for f, v in enumerate(M): for gt in v.keys(): if gt not in gt_tracked: gt_tracked[gt] = [] gt_tracked[gt].append(f) for gid, times in gt_tracked.items(): times = np.array(times) for i in range(len(times) - 1): t0 = times[i] t1 = times[i + 1] if t1 == t0 + 1: continue last_non_empty = -1 for j in range(t0, -1, -1): if gid in M[j].keys(): last_non_empty = j break next_non_empty = -1 for j in range(t1, f_gt): if gid in M[j]: next_non_empty = j break if next_non_empty != -1 and last_non_empty != -1: sid0 = M[last_non_empty][gid] sid1 = M[next_non_empty][gid] if sid1 == sid0: tr_id_recovered.append([t1 - t0 - 1, 1]) else: tr_id_recovered.append([t1 - t0 - 1, 0]) """for gid in range(n_gt): f0 = -1 count = 0 for frame in range(f_gt): if gid in gt_inds[frame]: vis = gtDB[gt_inds[frame][gid], 8] if vis < 0.5 and f0 != -1: count += 1 elif vis >= 0.5: if count != 0: print("Gap found {} - {} ({})".format(gid, frame, count)) count = 0 # set to current frame f0 = frame""" # Now iterate through the tracks and check if covered / id kept in comparison to occlusion for gid, vis in enumerate(visible): f0 = -1 count = 0 n_cov = 0 for frame, v in enumerate(vis): if v == 0 and f0 != -1: count += 1 if gid in M[frame].keys(): n_cov += 1 elif v == 1: # gap ended if count != 0: coverage.append([count, n_cov]) last_non_empty = -1 for j in range(f0, -1, -1): if gid in M[j].keys(): last_non_empty = j break next_non_empty = -1 for j in range(f0 + count + 1, f_gt): if gid in M[j]: next_non_empty = j break if next_non_empty != -1 and last_non_empty != -1: sid0 = M[last_non_empty][gid] sid1 = M[next_non_empty][gid] if sid1 == sid0: id_recovered.append([count, 1]) else: id_recovered.append([count, 0]) count = 0 n_cov = 0 # set to current frame f0 = frame coverage = np.array(coverage) id_recovered = np.array(id_recovered) tr_id_recovered = np.array(tr_id_recovered) #for c in coverage: # print(c) xmax = 50 # build values for plot x_val = np.arange(1, xmax + 1) y_val = np.zeros(xmax) for x in x_val: y = np.mean(coverage[coverage[:, 0] == x, 1] / coverage[coverage[:, 0] == x, 0]) y_val[x - 1] = y #plt.plot([0,1], [0,1], 'r-') plt.figure() plt.scatter(coverage[:, 0], coverage[:, 1] / coverage[:, 0], s=2**2) plt.plot(x_val, y_val, 'rx') plt.xlabel('gap length') plt.xlim((0, xmax)) plt.ylabel('tracker coverage') plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'GAP_COV')), format='pdf') # build values for plot x_val = np.arange(1, xmax + 1) y_val = np.zeros(xmax) for x in x_val: y = np.mean(id_recovered[id_recovered[:, 0] == x, 1]) y_val[x - 1] = y plt.figure() plt.plot(x_val, y_val, 'rx') plt.scatter(id_recovered[:, 0], id_recovered[:, 1], s=2**2) plt.xlabel('gt gap length') plt.xlim((0, xmax)) plt.ylabel('part id recovered') plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'GAP_ID')), format='pdf') plt.close() # tr id recovered x_val = np.arange(1, xmax + 1) y_val = np.zeros(xmax) for x in x_val: y = np.mean(tr_id_recovered[tr_id_recovered[:, 0] == x, 1]) y_val[x - 1] = y plt.figure() plt.plot(x_val, y_val, 'rx') plt.scatter(tr_id_recovered[:, 0], tr_id_recovered[:, 1], s=2**2) plt.xlabel('track gap length') plt.xlim((0, xmax)) plt.ylabel('part id recovered') plt.savefig(osp.join(output_dir, "{}-{}-{}.pdf".format(t, detections, 'GAP_TR_ID')), format='pdf') plt.close()
def evaluate_new(results, gt_file): res = [] for i, track in results.items(): for frame, bb in track.items(): x1 = bb[0] y1 = bb[1] x2 = bb[2] y2 = bb[3] res.append([ float(frame + 1), float(i + 1), float(x1 + 1), float(y1 + 1), float(x2 + 1), float(y2 + 1), float(-1), float(-1), float(-1), float(-1) ]) trackDB = np.array(res) gtDB = read_txt_to_struct(gt_file) # manipulate mot15 to fit mot16 #gtDB[:,7] = gtDB[:,6] #gtDB[:,6] = 1 #gtDB[:,8] = 1 #gtDB = gtDB[:,:9] gtDB, distractor_ids = extract_valid_gt_data(gtDB) metrics, extra_info, clear_mot_info, ML_PT_MT = evaluate_sequence( trackDB, gtDB, distractor_ids) print_metrics(' Evaluation', metrics) # evaluate sizes of ML_PT_MT ML_PT_MT_sizes = [[]] ML_PT_MT_vis = [[]] for i, p in enumerate(ML_PT_MT, 1): ML_PT_MT_sizes.append([]) ML_PT_MT_vis.append([]) for tr_id in p: gt_in_person = np.where(gtDB[:, 1] == tr_id)[0] #w = gtDB[gt_in_person, 4] h = gtDB[gt_in_person, 5] - gtDB[gt_in_person, 3] #sizes = (w*h).mean() ML_PT_MT_sizes[i].append(h.mean()) ML_PT_MT_sizes[0].append(h.mean()) vis = gtDB[gt_in_person, 8] ML_PT_MT_vis[i].append(vis.mean()) ML_PT_MT_vis[0].append(vis.mean()) print("\t\theight\tvis") print("MT: \t\t{:.2f}\t{:.2f}".format(np.mean(ML_PT_MT_sizes[3]), np.mean(ML_PT_MT_vis[3]))) print("PT: \t\t{:.2f}\t{:.2f}".format(np.mean(ML_PT_MT_sizes[2]), np.mean(ML_PT_MT_vis[2]))) print("ML: \t\t{:.2f}\t{:.2f}".format(np.mean(ML_PT_MT_sizes[1]), np.mean(ML_PT_MT_vis[1]))) print("total (mean of\t{:.2f}\t{:.2f}\ntrack means):".format( np.mean(ML_PT_MT_sizes[0]), np.mean(ML_PT_MT_vis[0]))) return clear_mot_info