class Track_And_Detect(object): effective_track_thresh = 0.5 effective_detection_thresh = 0.5 effective_keypoints_thresh = 0.6 effective_keypoints_number = 8 iou_match_thresh = 0.5 nms_thresh = 0.5 oks_thresh = 0.8 embedding_match_thresh = 2 feature_length = 2048 oks_flag = True tracker_flag = True tracker_update_flag = True new_embedding_flag = True descrease_tracker_flag = True #descrease_tracker_flag = False def __init__(self, gpu_id=0, track_model=None, pose_model=None, embedding_model=None): if self.tracker_flag: self.tracker = SiamFCTracker(gpu_id, track_model) self.posenet = PoseNet(gpu_id, pose_model) self.matcher = Matcher() print('----------------------------------------') print('Flag parameters are set as follow:') print('Tracker flag: {}'.format(self.tracker_flag)) print('Tracker update flag: {}'.format(self.tracker_update_flag)) print('Decrease tracker flag: {}'.format(self.descrease_tracker_flag)) print('New embedding(with pose) flag: {}'.format( self.new_embedding_flag)) print('----------------------------------------') # bbox must be format of x1y1x2y2 def update_tracker(self, rgb_frame, bbox, track_id): self.tracker.update_data_dict(rgb_frame, bbox, track_id) def pose_detect(self, im, bbox): return self.posenet.detect_pose(im, bbox) def embedding(self, frame, bbox): feature = self.posenet.embedding(frame, bbox) return feature # initialize the first frame of this video def init_tracker(self, frame, bbox_list): self.new_id_flag = 0 self.track_id_dict = dict() if self.tracker_flag: self.tracker.clear_data() # conver bgr(opencv) to rgb rgb_frame = frame[:, :, ::-1] bbox_list, keypoint_list = self.oks_filter(bbox_list, frame) for bbox in bbox_list: self.create_id(frame, rgb_frame, bbox) bbox_list = [] for id, item in self.track_id_dict.items(): bbox = item['bbox_and_score'] + [id] bbox_list.append(bbox) return bbox_list def oks_filter(self, det_list, frame): keypoint_list = [] for bbox in det_list: center, scale = self.posenet.x1y1x2y2_to_cs(bbox[0:4]) area = np.prod(scale * 200, 1) pred = np.zeros((15, 3), dtype=np.float32) pose_positions, pose_vals, pose_heatmaps = self.pose_detect( frame, bbox) pred[:, 0:2] = pose_positions pred[:, 2] = pose_vals score_all, valid_num = 0, 0 for i in range(15): score_i = pose_vals[i] if score_i >= 0.2: score_all += score_i valid_num += 1 if valid_num != 0: new_score = score_all / valid_num * bbox[4] else: new_score = 0 keypoint_dict = { 'score': new_score, 'area': area, 'keypoints': pred } keypoint_list.append(keypoint_dict) keep = self.matcher.oks_nms(keypoint_list, thresh=self.oks_thresh) new_det_list = [det_list[i] for i in keep] new_keypoint_list = [keypoint_list[i] for i in keep] return new_det_list, new_keypoint_list def create_id(self, frame, rgb_frame, bbox): score = bbox[4] bbox = bbox[0:4] track_id = self.new_id_flag if self.new_embedding_flag: feature = self.embedding(frame, bbox) else: feature = self.embedder.embedding(frame, bbox) self.track_id_dict[track_id] = { 'bbox_and_score': bbox + [score], 'feature': feature, 'frame_flag': 1, 'exist': True } if self.tracker_flag: self.update_tracker(rgb_frame, bbox, track_id) self.new_id_flag += 1 def update_id(self, frame, rgb_frame, det_bbox, track_id): bbox, score = det_bbox[0:4], det_bbox[4] if self.new_embedding_flag: feature = np.array(self.embedding(frame, bbox)) else: feature = np.array(self.embedder.embedding(frame, bbox)) former_track_dict = self.track_id_dict[track_id] former_frame_flag, former_feature = former_track_dict[ 'frame_flag'], np.array(former_track_dict['feature']) now_frame_flag = former_frame_flag + 1 now_feature = feature.tolist() self.track_id_dict[track_id] = { 'bbox_and_score': det_bbox, 'feature': now_feature, 'frame_flag': now_frame_flag, 'exist': True } if self.tracker_flag and self.tracker_update_flag: self.update_tracker(rgb_frame, bbox, track_id) def multi_track(self, frame): rgb_frame = frame[:, :, ::-1] bbox_list = [] for id in self.track_id_dict: if self.track_id_dict[id]['exist'] == False: continue bbox, score = self.tracker.track_id(rgb_frame, id) bbox_list.append(bbox + [score] + [id]) self.track_id_dict[id]['bbox'] = bbox return bbox_list def match_detection_tracking_oks_iou_embedding(self, detections, track_list, frame): rgb_frame = frame[:, :, ::-1] #print(detections) for track in track_list: track_score = track[4] if track_score >= self.effective_track_thresh: if self.descrease_tracker_flag: track_score -= 0.35 detections.append(track[0:4] + [track_score]) #print(detections) if self.oks_flag: detections, keypoint_list = self.oks_filter(detections, frame) #decrease the tracking score #get feature for former trackers database_bbox_list = [] for database_id in self.track_id_dict: database_id_bbox = self.track_id_dict[database_id][ 'bbox_and_score'] + [database_id] database_bbox_list.append(database_id_bbox) matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, database_bbox_list, iou_threshold=self.iou_match_thresh) #update the matched trackers with detection bbox for match in matches: det_index, track_index = match det_bbox = detections[det_index] update_id = database_bbox_list[track_index][5] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new index for unmatched_detections det_feature_list = [] for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] if self.new_embedding_flag: det_id_feature = self.embedding(frame, det_bbox) + det_bbox else: det_id_feature = self.embedder.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) track_feature_list = [] for delete_index in unmatched_trackers: track_bbox = database_bbox_list[delete_index] track_score, delete_id = track_bbox[4], track_bbox[5] delete_id_feature = self.track_id_dict[delete_id]['feature'] + [ delete_id ] track_feature_list.append(delete_id_feature) embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, track_feature_list, distance_threshold = self.embedding_match_thresh) #update matched embedding detections and former tracking feature for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = track_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new id for unmatched detections for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] del self.track_id_dict[delete_id] if self.tracker_flag: self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) return bbox_list
class Track_And_Detect: effective_track_thresh = 0.55 effective_detection_thresh = 0.4 effective_keypoints_thresh = 0.6 effective_keypoints_number = 8 iou_match_thresh = 0.5 embedding_match_thresh = 2 nms_thresh = 0.5 oks_thresh = 0.8 def __init__( self, gpu_id=[0, 0, 0, 0], flag=[True, False, True, False], #track_model='/export/home/zby/SiamFC/models/output/siamfc_35.pth', track_model='/export/home/zby/SiamFC/models/output/siamfc_20.pth', detection_model='/export/home/zby/SiamFC/models/res101_old/pascal_voc/faster_rcnn_1_25_4379.pth', pose_model='/export/home/zby/SiamFC/data/models/final_new.pth.tar', embedding_model='/export/home/zby/SiamFC/models/embedding_model.pth' ): if flag[0]: self.tracker = SiamFCTracker(gpu_id[0], track_model) #input RGB if flag[1]: self.detector = Detector(gpu_id[1], detection_model) #input BGR if flag[2]: self.posenet = PoseNet(gpu_id[2], pose_model) #input BGR if flag[3]: self.embedder = EmbeddingNet(gpu_id[3], embedding_model) #self.tracker = SiamFCTracker(gpu_id[0], track_model) self.matcher = Matcher() print('----------------------------------------') #initialize the first frame of this video def init_tracker(self, frame, bbox_list): self.new_id_flag = 0 self.track_id_dict = dict() #self.tracker.clear_data() #conver bgr(opencv) to rgb rgb_frame = frame[:, :, ::-1] bbox_list, keypoint_list = self.oks_filter(bbox_list, frame) #print(bbox_list) for bbox in bbox_list: self.create_id(frame, rgb_frame, bbox) bbox_list = [] #pose_list=[] for id, item in self.track_id_dict.items(): bbox = item['bbox_and_score'] + [id] bbox_list.append(bbox) # pose_position, pose_value, pose_heatmap = self.pose_detect(frame, bbox) # pose_info = np.hstack(pose_postion, pose_value) return bbox_list def oks_filter(self, det_list, frame): keypoint_list = [] for bbox in det_list: center, scale = self.posenet.x1y1x2y2_to_cs(bbox[0:4]) area = np.prod(scale * 200, 1) pred = np.zeros((15, 3), dtype=np.float32) pose_positions, pose_vals, pose_heatmaps = self.pose_detect( frame, bbox) #print(pose_vals) #posa_vals = np.expand_dims(pose_vals, axis=1) pred[:, 0:2] = pose_positions pred[:, 2] = pose_vals score_all, valid_num = 0, 0 for i in range(15): score_i = pose_vals[i] if score_i >= 0.2: score_all += score_i valid_num += 1 if valid_num != 0: new_score = score_all / valid_num * bbox[4] else: new_score = 0 keypoint_dict = { 'score': new_score, 'area': area, 'keypoints': pred } keypoint_list.append(keypoint_dict) keep = self.matcher.oks_nms(keypoint_list, thresh=self.oks_thresh) new_det_list = [det_list[i] for i in keep] new_keypoint_list = [keypoint_list[i] for i in keep] return new_det_list, new_keypoint_list def create_id(self, frame, rgb_frame, bbox): score = bbox[4] bbox = bbox[0:4] track_id = self.new_id_flag feature = self.embedding(frame, bbox) #self.track_id_dict[track_id]={'bbox_and_score':bbox+[score]} self.track_id_dict[track_id] = { 'bbox_and_score': bbox + [score], 'feature': feature, 'frame_flag': 1, 'exist': True } #self.track_id_dict[track_id]={'bbox_and_score':bbox+[score],'feature':[feature], 'frame_flag':1, 'exist':True} #pose_position, pose_value, pose_heatmap = self.pose_detect(frame, bbox) #self.update_tracker(rgb_frame, bbox, track_id) #print('Track id {} has been initinized'.format(track_id)) self.new_id_flag += 1 def update_id(self, frame, rgb_frame, det_bbox, track_id): bbox, score = det_bbox[0:4], det_bbox[4] feature = np.array(self.embedding(frame, bbox)) former_track_dict = self.track_id_dict[track_id] former_frame_flag, former_feature = former_track_dict[ 'frame_flag'], np.array(former_track_dict['feature']) now_frame_flag = former_frame_flag + 1 #calculate the average feature #now_feature = ((former_feature*former_frame_flag+feature)/now_frame_flag).tolist() #former_feature = former_feature.tolist() now_feature = feature.tolist() #former_feature.append(now_feature) self.track_id_dict[track_id] = { 'bbox_and_score': det_bbox, 'feature': now_feature, 'frame_flag': now_frame_flag, 'exist': True } #pose_position, pose_value, pose_heatmap = self.pose_detect(frame, bbox) #self.update_tracker(rgb_frame, bbox, track_id) def multi_track(self, frame): rgb_frame = frame[:, :, ::-1] bbox_list = [] for id in self.track_id_dict: if self.track_id_dict[id]['exist'] == False: continue bbox, score = self.tracker.track_id(rgb_frame, id) bbox_list.append(bbox + [score] + [id]) self.track_id_dict[id]['bbox'] = bbox return bbox_list def match_and_track_embedding(self, detections, track_list, frame): # print(self.track_id_dict.keys()) # exist_flag = [] # for id in self.track_id_dict: # exist_flag.append(self.track_id_dict[id]['exist']) # print(exist_flag) rgb_frame = frame[:, :, ::-1] matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, track_list, iou_threshold=self.iou_match_thresh) has_tracked_id = set() for match in matches: det_index, track_index = match has_tracked_id.add(track_index) det_bbox = detections[det_index] update_id = track_list[track_index][5] self.update_id(frame, rgb_frame, det_bbox, update_id) #get feature unmatched_detections det_feature_list = [] for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] det_id_feature = self.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) #get feature for unmatched_trackers database_feature_list = [] for database_id in self.track_id_dict: if database_id in has_tracked_id: continue database_id_feature = self.track_id_dict[database_id][ 'feature'] + [database_id] database_feature_list.append(database_id_feature) # for delete_index in unmatched_trackers: # track_bbox = track_list[delete_index] # track_score, delete_id = track_bbox[4], track_bbox[5] # delete_id_feature = self.track_id_dict[delete_id]['feature'] + [delete_id] # track_feature_list.append(delete_id_feature) #match the detection and tracklist embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, database_feature_list, distance_threshold = self.embedding_match_thresh) for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = database_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #change status for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] self.track_id_dict[delete_id]['exist'] = False #delete unuseful index for unmatched_trackers # for delete_index in embedding_unmatched_trackers: # delete_id = track_feature_list[delete_index][2048] # del self.track_id_dict[delete_id] # self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list def match_detection_embedding(self, detections, frame): rgb_frame = frame[:, :, ::-1] #get detection feature det_feature_list = [] for det_bbox in detections: det_id_feature = self.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) #get feature for former trackers database_feature_list = [] for database_id in self.track_id_dict: database_id_feature = self.track_id_dict[database_id][ 'feature'] + [database_id] database_feature_list.append(database_id_feature) #match the detection and tracklist embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, database_feature_list, distance_threshold = self.embedding_match_thresh) for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = database_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful trackers for delete_index in embedding_unmatched_trackers: delete_id = database_feature_list[delete_index][2048] del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list def match_detection_iou(self, detections, frame): rgb_frame = frame[:, :, ::-1] #get feature for former trackers database_bbox_list = [] for database_id in self.track_id_dict: database_id_bbox = self.track_id_dict[database_id][ 'bbox_and_score'] + [database_id] database_bbox_list.append(database_id_bbox) matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, database_bbox_list, iou_threshold=self.iou_match_thresh) #update the matched trackers with detection bbox for match in matches: det_index, track_index = match det_bbox = detections[det_index] update_id = database_bbox_list[track_index][5] self.track_id_dict[update_id]['bbox_and_score'] = det_bbox[0:5] #create new index for unmatched_detections for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in unmatched_trackers: delete_id = database_bbox_list[delete_index][5] del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list def match_detection_iou_embedding(self, detections, frame): rgb_frame = frame[:, :, ::-1] #final_bbox = [] detections, keypoint_list = self.oks_filter(detections, frame) #get feature for former trackers database_bbox_list = [] for database_id in self.track_id_dict: database_id_bbox = self.track_id_dict[database_id][ 'bbox_and_score'] + [database_id] database_bbox_list.append(database_id_bbox) matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, database_bbox_list, iou_threshold=self.iou_match_thresh) #update the matched trackers with detection bbox for match in matches: det_index, track_index = match det_bbox = detections[det_index] update_id = database_bbox_list[track_index][5] self.update_id(frame, rgb_frame, det_bbox, update_id) #final_bbox.append(det_bbox+[update_id]) #create new index for unmatched_detections det_feature_list = [] for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] det_id_feature = self.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) track_feature_list = [] for delete_index in unmatched_trackers: track_bbox = database_bbox_list[delete_index] track_score, delete_id = track_bbox[4], track_bbox[5] delete_id_feature = self.track_id_dict[delete_id]['feature'] + [ delete_id ] track_feature_list.append(delete_id_feature) embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, track_feature_list, distance_threshold = self.embedding_match_thresh) #update matched embedding detections and former tracking feature for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = track_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new id for unmatched detections for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] del self.track_id_dict[delete_id] #self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) return bbox_list def match_detection_tracking_iou_embedding(self, detections, track_list, frame): rgb_frame = frame[:, :, ::-1] #print(detections) for track in track_list: track_score = track[4] if track_score >= self.effective_track_thresh: detections.append(track[0:5]) #print(detections) detections, keypoint_list = self.oks_filter(detections, frame) #get feature for former trackers database_bbox_list = [] for database_id in self.track_id_dict: database_id_bbox = self.track_id_dict[database_id][ 'bbox_and_score'] + [database_id] database_bbox_list.append(database_id_bbox) matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, database_bbox_list, iou_threshold=self.iou_match_thresh) #update the matched trackers with detection bbox for match in matches: det_index, track_index = match det_bbox = detections[det_index] update_id = database_bbox_list[track_index][5] self.update_id(frame, rgb_frame, det_bbox, update_id) #final_bbox.append(det_bbox+[update_id]) #create new index for unmatched_detections det_feature_list = [] for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] det_id_feature = self.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) track_feature_list = [] for delete_index in unmatched_trackers: track_bbox = database_bbox_list[delete_index] track_score, delete_id = track_bbox[4], track_bbox[5] delete_id_feature = self.track_id_dict[delete_id]['feature'] + [ delete_id ] track_feature_list.append(delete_id_feature) embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, track_feature_list, distance_threshold = self.embedding_match_thresh) #update matched embedding detections and former tracking feature for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = track_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new id for unmatched detections for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) return bbox_list def match_and_track_embedding_no_database(self, detections, track_list, frame): # print(self.track_id_dict.keys()) rgb_frame = frame[:, :, ::-1] matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, track_list, iou_threshold=self.iou_match_thresh) has_tracked_id = set() for match in matches: det_index, track_index = match has_tracked_id.add(track_index) det_bbox = detections[det_index] update_id = track_list[track_index][5] self.update_id(frame, rgb_frame, det_bbox, update_id) #get feature unmatched_detections det_feature_list = [] for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] det_id_feature = self.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) #get feature for unmatched_trackers track_feature_list = [] for delete_index in unmatched_trackers: track_bbox = track_list[delete_index] track_score, delete_id = track_bbox[4], track_bbox[5] delete_id_feature = self.track_id_dict[delete_id]['feature'] + [ delete_id ] track_feature_list.append(delete_id_feature) #match the detection and tracklist embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, track_feature_list, distance_threshold = self.embedding_match_thresh) #update matched embedding detections and former tracking feature for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = track_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new id for unmatched detections for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list def match_and_track_embedding_temporal_database(self, detections, track_list, frame): # print(self.track_id_dict.keys()) rgb_frame = frame[:, :, ::-1] matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, track_list, iou_threshold=self.iou_match_thresh) has_tracked_id = set() for match in matches: det_index, track_index = match has_tracked_id.add(track_index) det_bbox = detections[det_index] update_id = track_list[track_index][5] self.update_id(frame, rgb_frame, det_bbox, update_id) #get feature unmatched_detections det_feature_list = [] for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] det_id_feature = self.embedding(frame, det_bbox) + det_bbox det_feature_list.append(det_id_feature) #get feature for unmatched_trackers track_feature_list = [] for delete_index in unmatched_trackers: track_bbox = track_list[delete_index] track_score, delete_id = track_bbox[4], track_bbox[5] delete_id_features = self.track_id_dict[delete_id]['feature'] for delete_id_feature in delete_id_features: track_feature_list.append(delete_id_feature + [delete_id]) #match the detection and tracklist embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, track_feature_list, distance_threshold = self.embedding_match_thresh) #update matched embedding detections and former tracking feature update_ids = dict() for match in embedding_matches: det_index, track_index = match update_id = track_feature_list[track_index][2048] if not update_id in update_ids: update_ids[update_id] = det_index else: score_former = det_feature_list[update_ids[update_id]][-1] score_now = det_feature_list[det_index][-1] if score_now >= score_former: update_ids[update_id] = det_index for update_id, det_index in update_ids.items(): det_bbox = det_feature_list[det_index][2048:] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new id for unmatched detections for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] if delete_id not in update_ids and delete_id in self.track_id_dict: del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list def match_and_track_embedding_no_database_nms(self, detections, track_list, frame): # print(self.track_id_dict.keys()) rgb_frame = frame[:, :, ::-1] all_bbox = [] for det in detections: all_bbox.append(det) for track in track_list: if not track[4] >= self.effective_track_thresh: continue all_bbox.append(track[0:4] + [track[5]]) all_bbox = np.array(all_bbox) #print(all_bbox) keep = self.matcher.nms(all_bbox, self.nms_thresh) keep_bboxes = all_bbox[keep].tolist() #print(len(all_bbox), len(keep_bboxes)) #print(keep_bboxes) #get feature unmatched_detections det_feature_list = [] for keep_bbox in keep_bboxes: det_id_feature = self.embedding(frame, keep_bbox) + keep_bbox det_feature_list.append(det_id_feature) #get feature for unmatched_trackers track_feature_list = [] for former_id in self.track_id_dict: former_id_feature = self.track_id_dict[former_id]['feature'] + [ former_id ] track_feature_list.append(former_id_feature) #match the detection and tracklist embedding_matches, \ embedding_unmatched_detections,\ embedding_unmatched_trackers = self.matcher.associate_detections_to_trackers_embedding(det_feature_list, track_feature_list, distance_threshold = self.embedding_match_thresh) #update matched embedding detections and former tracking feature for match in embedding_matches: det_index, track_index = match det_bbox = det_feature_list[det_index][2048:] update_id = track_feature_list[track_index][2048] self.update_id(frame, rgb_frame, det_bbox, update_id) #create new id for unmatched detections for new_index in embedding_unmatched_detections: det_bbox = det_feature_list[new_index][2048:] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in embedding_unmatched_trackers: delete_id = track_feature_list[delete_index][2048] del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): if item['exist'] == True: bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list def match_and_track_iou(self, detections, track_list, frame): rgb_frame = frame[:, :, ::-1] matches, unmatched_detections, unmatched_trackers = self.matcher.associate_detections_to_trackers_iou( detections, track_list, iou_threshold=self.iou_match_thresh) #update the matched trackers with detection bbox for match in matches: det_index, track_index = match det_bbox = detections[det_index] update_id = track_list[track_index][5] #print('id {} has been updated'.format(update_id)) self.track_id_dict[update_id]['bbox_and_score'] = det_bbox[0:5] self.update_tracker(rgb_frame, det_bbox, update_id) #create new index for unmatched_detections for new_index in unmatched_detections: det_bbox = detections[new_index] det_score = det_bbox[4] pose_position, pose_value, pose_heatmap = self.pose_detect( frame, det_bbox) if det_score >= self.effective_detection_thresh and np.sum( pose_value >= self.effective_keypoints_thresh ) >= self.effective_keypoints_number: #print('add id{} now'.format(self.new_id_flag)) self.create_id(frame, rgb_frame, det_bbox) #delete unuseful index for unmatched_trackers for delete_index in unmatched_trackers: track_bbox = track_list[delete_index] track_score, delete_id = track_bbox[4], track_bbox[5] #pose_position, pose_value, pose_heatmap = self.pose_detect(frame, track_bbox) # if track_score >= self.effective_track_thresh and np.sum(pose_value >= self.effective_keypoints_thresh) >= self.effective_keypoints_number: # self.track_id_dict[delete_id]['bbox_and_score']=track_bbox[0:5] # else: # #print('delete id{} now'.format(delete_id)) del self.track_id_dict[delete_id] self.tracker.delete_id(delete_id) bbox_list = [] for id, item in self.track_id_dict.items(): bbox_list.append(item['bbox_and_score'] + [id]) #print(bbox_list) return bbox_list #bbox must be format of x1y1x2y2 def update_tracker(self, rgb_frame, bbox, track_id): self.tracker.update_data_dict(rgb_frame, bbox, track_id) def embedding(self, frame, bbox): start = time.time() #print(bbox) feature = self.embedder.get_feature(frame, bbox) #print('Embedding takes {}s'.format(time.time()-start)) return feature def detect(self, im): return self.detector.detect(im, self.nms_thresh, 0.5) def pose_detect(self, im, bbox): return self.posenet.detect_pose(im, bbox)