def assign_detections_to_tracks(self, detections): # A. no instances now: initial—>将detection的结果加入instances # B. we have instances now: (prediction/track<—>detection matching) # 1. predict/track—> kalman filter # 2. 找到detection和prediction相互的匹配 # 2.a Munkres Algorithm # det: b e f // b->b e->f f->f # pre: a b c f // a->b b->b c->b f->f # matched: b, f # 2.b bi-matching: 以predict/track为基准,先匹配det;再由det为基准,匹配pre/track # 3. 对于正确匹配的物体,进行更新:correction # 4. 处理未正确匹配的物体 # 4.1 未检测到的物体:a.即时移除(不建议) b.继续更新(单纯pred/track,不考虑detection) # c.长久未匹配,移除 # 4.2 检测到有新物体:a. 即时加入 b.加入track(试用期),几帧后(通过适用期后),加入instances if len(self.instances) == 0: for det in detections: # det: 检测结果 # det: {'tag': [bbox_left, bbox_right, bbox_top, bbox_bottom]} instance = Instance(self.config, self.video_helper) tag = list(det.keys())[0] bbox = det[tag] # instance.add_to_track(tag, bbox) # 辅助 self.instances.append(instance) # B. # B.1 # 算距离:检测框与预测框之间的距离 # costs[i, j]: 每一个预测框与检测框之间的距离 costs = np.zeros(shape=(len(self.instances), len(detections))) for i, instance in enumerate(self.instances): # Here, by using Kalman Filter, we predict an bbx for each instance predicted_bbx = instance.get_predicted_bbx() for j, det in enumerate(detections): detected_bbx = list(det.values())[0] dist = util.dist_btwn_bbx_centroids(predicted_bbx, detected_bbx) max_dist = self.config.MAX_PIXELS_DIST_BETWEEN_PREDICTED_AND_DETECTED if dist > max_dist: dist = 1000 # sys.maxsize costs[i, j] = dist # set all tracked instances as unassigned for instance in self.instances: instance.has_match = False # B.2 # 利用cost矩阵寻找匹配 # Munkres Algorithm # Instructions and C# version can be found here: http://csclab.murraystate.edu/~bob.pilgrim/445/munkres.html # while in Python we can solve the problem by using method imported as below. # Descriptions for Python version can be found in # https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.optimize.linear_sum_assignment.html assigned_instances, assigned_detections = linear_sum_assignment(costs) # B.3 # 更新assigned bbox assigned_detection_id = [] for idx, instance_id in enumerate(assigned_instances): detection_id = assigned_detections[idx] # if assignment for this instance and detection is sys.maxsize, discard it if costs[instance_id, detection_id] != 1000: # sys.maxsize: assigned_detection_id.append(detection_id) self.instances[instance_id].has_match = True self.instances[instance_id].correct_track( detections[detection_id]) self.instances[instance_id].num_misses = 0 # B.4 # keep track of how many times a track has gone unassigned for instance in self.instances: if instance.has_match is False: instance.num_misses += 1 # The function shown below can only remove those instances which has already been # added to tracks but CAN NOT remove detected bbx which has a huge IOU with # existed tracks. So we need another remove function to dual with that self.remove_dead_instances() # get unassigned detection ids unassigned_detection_id = list( set(range(0, len(detections))) - set(assigned_detection_id)) for idx in range(0, len(detections)): if idx in unassigned_detection_id: # det: {'tag' : [bbx_left, bbx_right, bbx_up, bbx_bottom]} tag = list(detections[idx].keys())[0] bbx = detections[idx][tag] # then we need to confirm whether the detection is a good one if self.is_good_detection(bbx): instance = Instance(self.config, self.video_helper) # instance.add_to_track(tag, bbx) self.instances.append(instance)
def assign_detections_to_tracks(self, detections, frame_id, frame): """ This function aims to assign detections to their corresponding tracks by minimizing the cost matrix using Munkres Algorithm. Args: param 1. detections: [{'tag1' : [bbx1]}, {'tag2' : [bbx2]}, ..., {'tagn' : [bbxn]}] where bbx = [bbx_left, bbx_right, bbx_up, bbx_down] param 2. frame_id: current frame ID starting from 0 """ """ Step 0: got measurements (detections) """ """ Step 1: get predicted states """ # find distance from all tracked instances to all detected boxes # if there are no instances, then all detections are new tracks if (len(self.instances)) == 0: for det in detections: # det: {'tag' : [bbx_left, bbx_right, bbx_up, bbx_bottom]} instance = Instance(self.config, self.video_helper, frame) tag = list(det.keys())[0] bbx = det[tag] instance.add_to_track(tag, bbx, frame) self.instances.append(instance) return True """ Step 2: assign detections to correspondiong instances """ # iou match between detected bboxes and tracked bboxes track_det_iou, det_track_iou = {}, {} for t, instance in enumerate(self.instances): if not track_det_iou.__contains__(t): track_det_iou[t] = [] # creat key predicted_bbx = instance.get_predicted_bbx( frame ) # Here, by using KCF, we predict an bbx for each instance for d, det in enumerate(detections): if not det_track_iou.__contains__(d): det_track_iou[d] = [] detected_bbx = list(det.values())[0] iou = util.get_iou( predicted_bbx, detected_bbx ) # get IOU between all detected bboxes and tracked bboxes track_det_iou[t].append([d, iou]) det_track_iou[d].append([t, iou]) # set all tracked instances as unassigned for instance in self.instances: instance.has_match = False assigned_instances, assigned_detections = [], [] for i, id_iou in track_det_iou.items(): match_detid = util.get_maxiou_id(id_iou) if match_detid != None: match_trackid = util.get_maxiou_id(det_track_iou[match_detid]) if match_trackid == i: # match assigned_instances.append(match_trackid) assigned_detections.append(match_detid) """ Step 3: correct an instance if it's matched by correponsding instance and detection """ assigned_detection_id = [] if assigned_instances != None and assigned_detections != None: # sys.maxsize: for idx, instance_id in enumerate(assigned_instances): detection_id = assigned_detections[idx] # if assignment for this instance and detection is sys.maxsize, discard it # record this detection is assigned assigned_detection_id.append(detection_id) # record this tracked instance which is assigned self.instances[instance_id].has_match = True # correct states by using kalman filter self.instances[instance_id].correct_track( detections[detection_id], frame) # means this instance is detected self.instances[instance_id].num_misses = 0 """ Step 4: remove instances which should be """ # keep track of how many times a track has gone unassigned for instance in self.instances: if instance.has_match is False: instance.num_misses += 1 # The function shown below can only remove those instances which has already been # added to tracks but CAN NOT remove detected bbx which has a huge IOU with # existed tracks. So we need another remove function to dual with that self.remove_dead_instances() """ Step 5: create new instances to track if no detections are matched """ # get unassigned detection ids unassigned_detection_id = list( set(range(0, len(detections))) - set(assigned_detection_id)) for idx in range(0, len(detections)): if idx in unassigned_detection_id: # det: {'tag' : [bbx_left, bbx_right, bbx_up, bbx_bottom]} tag = list(detections[idx].keys())[0] bbx = detections[idx][tag] # then we need to confirm whether the detection is a good one if self.is_good_detection(bbx): instance = Instance(self.config, self.video_helper, frame) instance.add_to_track(tag, bbx, frame) self.instances.append(instance)