def track(self, detections): self.frame += 1 self.tracks_active[self.frame] = {} #Clear the tracks in old frame self.clean_old_tracks() dets = [det for det in detections if det['score'] >= self.sigma_l] for id_, track in self.retrieve_tracks(): if len(dets) > 0: # get det with highest iou best_match = max(dets, key=lambda x: iou(track['bbox'], x['bbox'])) if iou(track['bbox'], best_match['bbox']) >= self.sigma_iou: self.tracks_active[self.frame][id_] = best_match # remove from best matching detection from detections del dets[dets.index(best_match)] #Create new tracks for det in dets: self.id_count += 1 self.tracks_active[self.frame][self.id_count] = det #Return the current tracks return self.tracks_active[self.frame]
def track_iou(detections, sigma_l, sigma_h, sigma_iou, t_min): """ Simple IOU based tracker. See "High-Speed Tracking-by-Detection Without Using Image Information by E. Bochinski, V. Eiselein, T. Sikora" for more information. Args: detections (list): list of detections per frame, usually generated by util.load_mot sigma_l (float): low detection threshold. sigma_h (float): high detection threshold. sigma_iou (float): IOU threshold. t_min (float): minimum track length in frames. Returns: list: list of tracks. """ tracks_active = [] tracks_finished = [] for frame_num, detections_frame in enumerate(detections, start=1): # apply low threshold to detections dets = [det for det in detections_frame if det['score'] >= sigma_l] updated_tracks = [] for track in tracks_active: if len(dets) > 0: # get det with highest iou best_match = max(dets, key=lambda x: iou(track['bboxes'][-1], x['bbox'])) if iou(track['bboxes'][-1], best_match['bbox']) >= sigma_iou: track['bboxes'].append(best_match['bbox']) track['max_score'] = max(track['max_score'], best_match['score']) track['class'] = best_match['class'] updated_tracks.append(track) # remove from best matching detection from detections del dets[dets.index(best_match)] # if track was not updated if len(updated_tracks) == 0 or track is not updated_tracks[-1]: # finish track when the conditions are met if track['max_score'] >= sigma_h and len(track['bboxes']) >= t_min: tracks_finished.append(track) # create new tracks new_tracks = [{'bboxes': [det['bbox']], 'max_score': det['score'], 'start_frame': frame_num,'class' : det['class']} for det in dets] tracks_active = updated_tracks + new_tracks # finish all remaining active tracks tracks_finished += [track for track in tracks_active if track['max_score'] >= sigma_h and len(track['bboxes']) >= t_min] return tracks_finished
def track_iou(detections, sigma_l, sigma_h, sigma_iou, t_min): """ Simple IOU based tracker. See "High-Speed Tracking-by-Detection Without Using Image Information by E. Bochinski, V. Eiselein, T. Sikora" for more information. Args: detections (list): list of detections per frame, usually generated by util.load_mot sigma_l (float): low detection threshold. sigma_h (float): high detection threshold. sigma_iou (float): IOU threshold. t_min (float): minimum track length in frames. Returns: list: list of tracks. """ tracks_active = [] tracks_finished = [] for frame_num, detections_frame in enumerate(detections, start=1): # apply low threshold to detections dets = [det for det in detections_frame if det['score'] >= sigma_l] updated_tracks = [] for track in tracks_active: if len(dets) > 0: # get det with highest iou best_match = max(dets, key=lambda x: iou(track['bboxes'][-1], x['bbox'])) if iou(track['bboxes'][-1], best_match['bbox']) >= sigma_iou: track['bboxes'].append(best_match['bbox']) track['max_score'] = max(track['max_score'], best_match['score']) updated_tracks.append(track) # remove from best matching detection from detections del dets[dets.index(best_match)] # if track was not updated if len(updated_tracks) == 0 or track is not updated_tracks[-1]: # finish track when the conditions are met if track['max_score'] >= sigma_h and len(track['bboxes']) >= t_min: tracks_finished.append(track) # create new tracks new_tracks = [{'bboxes': [det['bbox']], 'max_score': det['score'], 'start_frame': frame_num} for det in dets] tracks_active = updated_tracks + new_tracks # finish all remaining active tracks tracks_finished += [track for track in tracks_active if track['max_score'] >= sigma_h and len(track['bboxes']) >= t_min] return tracks_finished
def track_iou(pre_tracker, detections, sigma_iou): current_tracker = [] number = 0 best_data = 0.0 # current_tracker = detections # print("current") # print(current_tracker) # for i in range(len(current_tracker)): # print("current_tracker["+str(i)+"]") # print(current_tracker[i]) # print("bbox") # print(current_tracker[i]['bbox']) # print("bbox number") # print(current_tracker[i]['bbox'][0]) if not pre_tracker and not detections: return pre_tracker, "wait" elif not pre_tracker: print("pre") pre_tracker = detections[0] return pre_tracker, None elif not detections: return pre_tracker, "wait" else: print("curr") current_tracker = detections for k in range(len(current_tracker)): if iou(pre_tracker['bbox'], current_tracker[k]['bbox']) >= best_data: best_data = iou(pre_tracker['bbox'], current_tracker[k]['bbox']) number = k print("pre print") print(pre_tracker) print("curr print[number]") print(current_tracker[number]) if best_data >= sigma_iou: finish = current_tracker[number] pre_tracker = current_tracker[number] else: finish = None return pre_tracker, finish
def get_bg_proposals(object_proposals, annot): annot_rect = util.get_annot_rect(annot) bg_proposals = [] for obj_proposal in object_proposals: if util.iou(obj_proposal, annot_rect) <= 0.5: bg_proposals.append(obj_proposal) return bg_proposals
def ratio_tracker(current_tracker, p_tracker): best_iou_ratio = 0.0 # 최상의 적합성 비율은 얼마인가 best_data = [] # 최상의 비율을 가지는 데이터 for c_tracker in current_tracker: # 현재 트래커에서 전에 맞는 최적을 찾아냄 iou_ratio = iou(p_tracker['bbox'], c_tracker['bbox']) if iou_ratio >= best_iou_ratio: best_iou_ratio = iou_ratio best_data = c_tracker return best_data, best_iou_ratio
def active_criteria(x, tracks): """ Take matching candidate and track, offset the track's last bounding box by the predicted offset, and calculate IOU. Args: x (list [roi, bbox, score]): a detection from this frame. tracks (list [[frames], Kalman_filter]): a track containing all frames and a Kalman filter associated with it. """ ofdx, ofdy, _, _ = tracks[0][-1]['pred_state'] - tracks[0][-1]['cur_state'] offset_vector = np.array([ofdy, ofdx, ofdy, ofdx]) offset_roi = tracks[0][-1]['roi'] + offset_vector th = iou(x['roi'], offset_roi) return th
def _pair_bboxes(self, bboxes_true, bboxes_new, change_colors=False): pairs = [] for i_true, bb_true in enumerate(bboxes_true): match = None # Calculate overlap of bounding boxes. iou_max = 0.0 for i_new, bb_new in enumerate(bboxes_new): iou = util.iou(bb_true, bb_new) if iou > iou_max: iou_max = iou match = i_new if match is not None: pairs.append((i_true, match)) # Check for missing. if len(pairs) < len(bboxes_true): matched_true = [x[0] for x in pairs] matched_new = [x[1] for x in pairs] for i_true, bb_true in enumerate(bboxes_true): if i_true in matched_true: continue match = None d_min = numpy.Inf for i_new, bb_new in enumerate(bboxes_new): # Skip already matched new bounding boxes. if i_new in matched_new: continue d = util.bbox_distance(bb_true, bb_new) if d < d_min: d_min = d match = i_new if match is not None: pairs.append((i_true, match)) matched_true.append(i_true) # Update colors if still not all matched. if change_colors and len(pairs) < len(bboxes_true): colors_new = [] for ci in [x[0] for x in sorted(pairs, key=lambda x: x[1])]: colors_new.append(self._bbox_color[ci]) self._bbox_color = numpy.array(colors_new) return pairs
def associate(tracks, detections, sigma_iou): """ perform association between tracks and detections in a frame. Args: tracks (list): input tracks detections (list): input detections sigma_iou (float): minimum intersection-over-union of a valid association Returns: (tuple): tuple containing: track_ids (numpy.array): 1D array with indexes of the tracks det_ids (numpy.array): 1D array of the associated indexes of the detections """ costs = np.empty(shape=(len(tracks), len(detections)), dtype=np.float32) for row, track in enumerate(tracks): for col, detection in enumerate(detections): costs[row, col] = 1 - iou(track['bboxes'][-1], detection['bbox']) np.nan_to_num(costs) costs[costs > 1 - sigma_iou] = np.nan track_ids, det_ids = solve_dense(costs) return track_ids, det_ids
def dist_func(v1, v2, epsilon=1e-6, mode='iou'): """ :param v1: n-d numpy array :param v2: n-d numpy array, same shape with v1 :param epsilon: :param mode: :return: """ if mode == 'euclid': sm = np.sum((v1 - v2)**2) return (sm + epsilon)**0.5 elif mode == 'iou': x1, y1 = v1 x2, y2 = v2 box1 = [960 - x1 / 2, 600 - y1 / 2, 960 + x1 / 2, 600 + y1 / 2] box2 = [960 - x2 / 2, 600 - y2 / 2, 960 + x2 / 2, 600 + y2 / 2] return 1 - iou(box1, box2, encode=False) else: raise Exception('Unrecognized mode {}'.format(mode))
def main(): img_list = [] param_list = [] results = [] for _ in range(1000): params, img = noisy_circle(200, 50, 2) #normalize img_list.append(img / img.max()) param_list.append(params) param_list = np.array(param_list) img_list = np.array(img_list) # pass all samples as one batch for runtime detected = find_circle(img_list[:, :, :, np.newaxis]) # z-transform prediction into data range detected[:, :2] = detected[:, :2] * 200 detected[:, 2] = 10 + detected[:, 2] * 40 for i in range(1000): results.append(iou(param_list[i], detected[i])) results = np.array(results) print((results > 0.7).mean())
def run(self): self.vc = cv2.VideoCapture(self.mot_video_dir) sigma_l = 0.4 sigma_h = 0.5 sigma_iou = 0.2 t_min = 2 # load bounding boxes. detections = load_mot(self.mot_det_dir) tracks_finished = [] # set the color of the object randomly. color_for_boundingbox = [(13 * i % 255, (255 - 5 * i) % 255, (240 + 10 * i) % 255) for i in range(0, 51)] # run algorithm. for frame_num, detections_frame in enumerate(detections, start=1): # apply low threshold to detections dets = [det for det in detections_frame if det['score'] >= sigma_l] updated_tracks = [] for track in self.tracks_active: if len(dets) > 0: # get det with highest iou best_match = max(dets, key=lambda x: iou(track['bboxes'][-1], x['bbox'])) if iou(track['bboxes'][-1], best_match['bbox']) >= sigma_iou: track['bboxes'].append(best_match['bbox']) track['max_score'] = max(track['max_score'], best_match['score']) updated_tracks.append(track) # remove from best matching detection from detections del dets[dets.index(best_match)] # if track was not updated if len(updated_tracks) == 0 or track is not updated_tracks[-1]: # finish track when the conditions are met if track['max_score'] >= sigma_h and len(track['bboxes']) >= t_min: tracks_finished.append(track) # create new tracks new_tracks = [{'bboxes': [det['bbox']], 'max_score': det['score'], 'start_frame': frame_num, 'color': color_for_boundingbox[(len(self.tracks_active) + random.randint(0, 51)) % 51]} for i, det in enumerate(dets)] self.tracks_active = updated_tracks + new_tracks self.retval, current_frame = self.vc.read() labeled_frame = show_tracking_results(current_frame, self.tracks_active) rgb_frame = convert_cvimage_to_qimage(labeled_frame) self.mot_signal.emit(rgb_frame) # finish all remaining active tracks tracks_finished += [track for track in self.tracks_active if track['max_score'] >= sigma_h and len(track['bboxes']) >= t_min] output_path = os.path.dirname(self.mot_video_dir) save_to_csv(output_path, tracks_finished) self.vc.release()
def track_viou(frames_path, detections, sigma_l, sigma_h, sigma_iou, t_min, ttl, tracker_type, keep_upper_height_ratio): """ V-IOU Tracker. See "Extending IOU Based Multi-Object Tracking by Visual Information by E. Bochinski, T. Senst, T. Sikora" for more information. Args: frames_path (str): path to ALL frames. string must contain a placeholder like {:07d} to be replaced with the frame numbers. detections (list): list of detections per frame, usually generated by util.load_mot sigma_l (float): low detection threshold. sigma_h (float): high detection threshold. sigma_iou (float): IOU threshold. t_min (float): minimum track length in frames. ttl (float): maximum number of frames to perform visual tracking. this can fill 'gaps' of up to 2*ttl frames (ttl times forward and backward). tracker_type (str): name of the visual tracker to use. see VisTracker for more details. keep_upper_height_ratio (float): float between 0.0 and 1.0 that determines the ratio of height of the object to track to the total height of the object used for visual tracking. Returns: list: list of tracks. """ if tracker_type == 'NONE': assert ttl == 1, "ttl should not be larger than 1 if no visual tracker is selected" tracks_active = [] tracks_extendable = [] tracks_finished = [] frame_buffer = [] for frame_num, detections_frame in enumerate(tqdm(detections), start=1): # load frame and put into buffer frame_path = frames_path.format(frame_num) frame = cv2.imread(frame_path) assert frame is not None, "could not read '{}'".format(frame_path) frame_buffer.append(frame) if len(frame_buffer) > ttl + 1: frame_buffer.pop(0) # apply low threshold to detections dets = [det for det in detections_frame if det['score'] >= sigma_l] track_ids, det_ids = associate(tracks_active, dets, sigma_iou) updated_tracks = [] for track_id, det_id in zip(track_ids, det_ids): tracks_active[track_id]['bboxes'].append(dets[det_id]['bbox']) tracks_active[track_id]['max_score'] = max( tracks_active[track_id]['max_score'], dets[det_id]['score']) tracks_active[track_id]['classes'].append(dets[det_id]['class']) tracks_active[track_id]['det_counter'] += 1 if tracks_active[track_id]['ttl'] != ttl: # reset visual tracker if active tracks_active[track_id]['ttl'] = ttl tracks_active[track_id]['visual_tracker'] = None updated_tracks.append(tracks_active[track_id]) tracks_not_updated = [ tracks_active[idx] for idx in set(range(len( tracks_active))).difference(set(track_ids)) ] for track in tracks_not_updated: if track['ttl'] > 0: if track['ttl'] == ttl: # init visual tracker track['visual_tracker'] = VisTracker( tracker_type, track['bboxes'][-1], frame_buffer[-2], keep_upper_height_ratio) # viou forward update ok, bbox = track['visual_tracker'].update(frame) if not ok: # visual update failed, track can still be extended tracks_extendable.append(track) continue track['ttl'] -= 1 track['bboxes'].append(bbox) updated_tracks.append(track) else: tracks_extendable.append(track) # update the list of extendable tracks. tracks that are too old are moved to the finished_tracks. this should # not be necessary but may improve the performance for large numbers of tracks (eg. for mot19) tracks_extendable_updated = [] for track in tracks_extendable: if track['start_frame'] + len( track['bboxes']) + ttl - track['ttl'] >= frame_num: tracks_extendable_updated.append(track) elif track['max_score'] >= sigma_h and track[ 'det_counter'] >= t_min: tracks_finished.append(track) tracks_extendable = tracks_extendable_updated new_dets = [ dets[idx] for idx in set(range(len(dets))).difference(set(det_ids)) ] dets_for_new = [] for det in new_dets: finished = False # go backwards and track visually boxes = [] vis_tracker = VisTracker(tracker_type, det['bbox'], frame, keep_upper_height_ratio) for f in reversed(frame_buffer[:-1]): ok, bbox = vis_tracker.update(f) if not ok: # can not go further back as the visual tracker failed break boxes.append(bbox) # sorting is not really necessary but helps to avoid different behaviour for different orderings # preferring longer tracks for extension seems intuitive, LAP solving might be better for track in sorted(tracks_extendable, key=lambda x: len(x['bboxes']), reverse=True): offset = track['start_frame'] + len( track['bboxes']) + len(boxes) - frame_num # association not optimal (LAP solving might be better) # association is performed at the same frame, not adjacent ones if 1 <= offset <= ttl - track['ttl'] and iou( track['bboxes'][-offset], bbox) >= sigma_iou: if offset > 1: # remove existing visually tracked boxes behind the matching frame track['bboxes'] = track['bboxes'][:-offset + 1] track['bboxes'] += list(reversed(boxes))[1:] track['bboxes'].append(det['bbox']) track['max_score'] = max(track['max_score'], det['score']) track['classes'].append(det['class']) track['ttl'] = ttl track['visual_tracker'] = None tracks_extendable.remove(track) if track in tracks_finished: del tracks_finished[tracks_finished.index(track)] updated_tracks.append(track) finished = True break if finished: break if not finished: dets_for_new.append(det) # create new tracks new_tracks = [{ 'bboxes': [det['bbox']], 'max_score': det['score'], 'start_frame': frame_num, 'ttl': ttl, 'classes': [det['class']], 'det_counter': 1, 'visual_tracker': None } for det in dets_for_new] tracks_active = [] for track in updated_tracks + new_tracks: if track['ttl'] == 0: tracks_extendable.append(track) else: tracks_active.append(track) # finish all remaining active and extendable tracks tracks_finished = tracks_finished + \ [track for track in tracks_active + tracks_extendable if track['max_score'] >= sigma_h and track['det_counter'] >= t_min] # remove last visually tracked frames and compute the track classes for track in tracks_finished: if ttl != track['ttl']: track['bboxes'] = track['bboxes'][:-(ttl - track['ttl'])] track['class'] = max(set(track['classes']), key=track['classes'].count) del track['visual_tracker'] return tracks_finished
def main(): args = tracker_args() frame_count = 0 vid_loc = './data/video.mov' print('Track-before-Detect with Neural Networks') print('[1] Creating Frame Data') input_data(vid_loc) #create bootstrap detection print('[2] Creating Bootstrap Detection') detections = detection('./data/img_frames/1' + '.png', FAR, 0) #enter loop where tracker is fed detections, detector fed tracks, and threshold evaluated as this changes print('[3] Run T-b-D over frames') while (frame_count < 100): #grab a set amt. of frames from ./data/frames and move it to args.frames_path #delete args.frame_path first print('[*] Stage Frame Cluster') for root, dirs, files in os.walk(args.frames_path): for file in files: os.remove(os.path.join(root, file)) #grab next 100 frames from ./data/img_frames offset by frame_count for i in range(100): src = './data/img_frames/' + str(i + frame_count) + '.png' copyfile(src, args.frames_path + '/' + str(i + frame_count) + '.png') frame_count += 100 #call tracks now print('[*] Track Frame Cluster') #testing here print(detections) tracks = tracker(args, detections) #call detection() on last image in args.frames_path print('[*] Detect Frame Cluster') detections = detection( args.frames_path + '/' + str(frame_count - 1) + '.png', FAR, frame_count - 1) #call detect and then mesh detect coord. over track coord. and compare, avoid detect() interepret time. #array of detection bounding boxes print('[*] Evaluate T-b-D Performance') dt_boxes = [] for dt in range(len(detections)): temp = [] temp.append(detections[dt][2]) temp.append(detections[dt][3]) temp.append(detections[dt][4]) temp.append(detections[dt][5]) dt_boxes.append(temp) ious = [] if (len(tracks) == 0): print('There are no tracks') else: for track in range(len(tracks)): track_cmp = [] #track_cmp.append(tracks[track][2]) print tracks[track] track_cmp.append(tracks[track]['x']) track_cmp.append(tracks[track]['y']) track_cmp.append(tracks[track]['w']) track_cmp.append(tracks[track]['h']) for dt in range(len(dt_boxes)): #compare current track against all boxes tx1, ty1, tx2, ty2 = track_cmp[0], track_cmp[1], track_cmp[ 0] + track_cmp[2], track_cmp[1] + track_cmp[3] bbox1 = [tx1, ty1, tx2, ty2] bbox1 = np.asarray(bbox1) dx1, dy1, dx2, dy2 = float(dt_boxes[dt][0]), float( dt_boxes[dt][1]), float(dt_boxes[dt][0]) + float( dt_boxes[dt][2]), float(dt_boxes[dt][1]) + float( dt_boxes[dt][3]) bbox2 = [dx1, dy1, dx2, dy2] bbox2 = np.asarray(bbox2) intovunion = iou(bbox1, bbox2) if (intovunion > args.sigma_l): ious.append(intovunion) else: pass #call ev_thresh() to see if FAR shoud be updated print('[*] Update False Alarm Rate Threshold') ev_thresh(len(ious), len(tracks)) print('Track-before-Detect Complete!')