def regress_tracks(self, blob): pos = self.get_pos() # regress _, scores, bbox_pred, rois = self.obj_detect.test_rois(pos) boxes = bbox_transform_inv(rois, bbox_pred) boxes = clip_boxes(Variable(boxes), blob['im_info'][0][:2]).data pos = boxes[:, self.cl*4:(self.cl+1)*4] scores = scores[:, self.cl] s = [] for i in range(len(self.tracks)-1,-1,-1): t = self.tracks[i] t.score = scores[i] if scores[i] <= self.regression_person_thresh and not self.kill_oracle: self.tracks_to_inactive([t]) else: s.append(scores[i]) if self.regress: t.pos = pos[i].view(1,-1) return torch.Tensor(s[::-1]).cuda()
def step(self, blob): for t in self.tracks: t.last_pos = t.pos.clone() ########################### # Look for new detections # ########################### self.obj_detect.load_image(blob['data'][0], blob['im_info'][0]) if self.public_detections: dets = blob['dets'] if len(dets) > 0: dets = torch.cat(dets, 0)[:,:4] _, scores, bbox_pred, rois = self.obj_detect.test_rois(dets) else: rois = torch.zeros(0).cuda() else: _, scores, bbox_pred, rois = self.obj_detect.detect() if rois.nelement() > 0: boxes = bbox_transform_inv(rois, bbox_pred) boxes = clip_boxes(Variable(boxes), blob['im_info'][0][:2]).data # Filter out tracks that have too low person score scores = scores[:, self.cl] inds = torch.gt(scores, self.detection_person_thresh).nonzero().view(-1) else: inds = torch.zeros(0).cuda() if inds.nelement() > 0: boxes = boxes[inds] det_pos = boxes[:, self.cl*4:(self.cl+1)*4] det_scores = scores[inds] else: det_pos = torch.zeros(0).cuda() det_scores = torch.zeros(0).cuda() ################## # Predict tracks # ################## num_tracks = 0 nms_inp_reg = torch.zeros(0).cuda() if len(self.tracks): # align if self.do_align: self.align(blob) if self.pos_oracle or self.kill_oracle: self.oracle(blob) #regress if len(self.tracks): person_scores = self.regress_tracks(blob) # now NMS step if self.kill_oracle: person_scores = self.nms_oracle(blob, person_scores) if len(self.tracks): # create nms input new_features = self.get_appearances(blob) # nms here if tracks overlap nms_inp_reg = torch.cat((self.get_pos(), person_scores.add_(3).view(-1,1)),1) if self.kill_oracle: keep = torch.arange(nms_inp_reg.size(0)).long().cuda() else: keep = nms(nms_inp_reg, self.regression_nms_thresh) # # Plot the killed tracks for debugging self.tracks_to_inactive([self.tracks[i] for i in list(range(len(self.tracks))) if i not in keep]) if keep.nelement() > 0: nms_inp_reg = nms_inp_reg[keep] new_features[keep] self.add_features(new_features) num_tracks = nms_inp_reg.size(0) else: nms_inp_reg = torch.zeros(0).cuda() num_tracks = 0 ##################### # Create new tracks # ##################### # create nms input and nms new detections if det_pos.nelement() > 0: nms_inp_det = torch.cat((det_pos, det_scores.view(-1,1)), 1) else: nms_inp_det = torch.zeros(0).cuda() if nms_inp_det.nelement() > 0: keep = nms(nms_inp_det, self.detection_nms_thresh) nms_inp_det = nms_inp_det[keep] # check with every track in a single run (problem if tracks delete each other) for i in range(num_tracks): nms_inp = torch.cat((nms_inp_reg[i].view(1,-1), nms_inp_det), 0) keep = nms(nms_inp, self.detection_nms_thresh) keep = keep[torch.ge(keep,1)] if keep.nelement() == 0: nms_inp_det = nms_inp_det.new(0) break nms_inp_det = nms_inp[keep] if nms_inp_det.nelement() > 0: new_det_pos = nms_inp_det[:,:4] new_det_scores = nms_inp_det[:,4] # try to redientify tracks new_det_pos, new_det_scores, new_det_features = self.reid(blob, new_det_pos, new_det_scores) # add new if new_det_pos.nelement() > 0: self.add(new_det_pos, new_det_scores, new_det_features, blob) #################### # Generate Results # #################### for t in self.tracks: track_ind = int(t.id) if track_ind not in self.results.keys(): self.results[track_ind] = {} pos = t.pos[0] / blob['im_info'][0][2] sc = t.score self.results[track_ind][self.im_index] = np.concatenate([pos.cpu().numpy(), np.array([sc])]) self.im_index += 1 self.last_image = blob['data'][0][0] if not self.reid_oracle: self.clear_inactive()
def step(self, blob): for t in self.tracks: # add current position to last_pos list t.last_pos.append(t.pos.clone()) ########################### # Look for new detections # ########################### self.obj_detect.load_image(blob['data'][0], blob['im_info'][0]) if self.public_detections: dets = blob['dets'] if len(dets) > 0: dets = torch.cat(dets, 0)[:, :4] _, scores, bbox_pred, rois = self.obj_detect.test_rois(dets) else: rois = torch.zeros(0).cuda() else: _, scores, bbox_pred, rois = self.obj_detect.detect() if rois.nelement() > 0: boxes = bbox_transform_inv(rois, bbox_pred) boxes = clip_boxes(Variable(boxes), blob['im_info'][0][:2]).data scores = scores[:, self.cl] if self.kill_oracle: gt = blob['gt'] gt_boxes = torch.cat(list(gt.values()), 0).cuda() # calculate IoU distances iou_neg = 1 - bbox_overlaps(boxes, gt_boxes) dist_mat = iou_neg.cpu().numpy() row_ind, col_ind = linear_sum_assignment(dist_mat) matched = [] # normal matching for r, c in zip(row_ind, col_ind): if dist_mat[r, c] <= 0.5: matched.append(r.item()) inds = torch.LongTensor(matched).cuda() else: # Filter out tracks that have too low person score inds = torch.gt(scores, self.detection_person_thresh).nonzero().view(-1) else: inds = torch.zeros(0).cuda() if inds.nelement() > 0: boxes = boxes[inds] det_pos = boxes[:, self.cl * 4:(self.cl + 1) * 4] det_scores = scores[inds] else: det_pos = torch.zeros(0).cuda() det_scores = torch.zeros(0).cuda() ################## # Predict tracks # ################## num_tracks = 0 nms_inp_reg = torch.zeros(0).cuda() if len(self.tracks): # align if self.do_align: self.align(blob) if self.pos_oracle or self.kill_oracle: self.oracle(blob) elif self.motion_model_cfg['enabled']: self.motion() if self.reid_oracle: self.tracks_to_inactive([t for t in self.tracks if not t.has_positive_area()]) else: self.tracks = [t for t in self.tracks if t.has_positive_area()] # regress if len(self.tracks): person_scores = self.regress_tracks(blob) # now NMS step if self.kill_oracle: person_scores = self.nms_oracle(blob, person_scores) if len(self.tracks): # create nms input new_features = self.get_appearances(blob) # nms here if tracks overlap nms_inp_reg = torch.cat((self.get_pos(), person_scores.add_(3).view(-1, 1)), 1) if self.kill_oracle: # keep all keep = torch.arange(nms_inp_reg.size(0)).long().cuda() else: keep = nms(nms_inp_reg, self.regression_nms_thresh) # Plot the killed tracks for debugging self.tracks_to_inactive([self.tracks[i] for i in list(range(len(self.tracks))) if i not in keep]) if keep.nelement() > 0: nms_inp_reg = nms_inp_reg[keep] self.add_features(new_features) num_tracks = nms_inp_reg.size(0) else: nms_inp_reg = torch.zeros(0).cuda() num_tracks = 0 ##################### # Create new tracks # ##################### # create nms input and nms new detections if det_pos.nelement() > 0: nms_inp_det = torch.cat((det_pos, det_scores.view(-1, 1)), 1) else: nms_inp_det = torch.zeros(0).cuda() if nms_inp_det.nelement() > 0: keep = nms(nms_inp_det, self.detection_nms_thresh) nms_inp_det = nms_inp_det[keep] # check with every track in a single run (problem if tracks delete each other) for i in range(num_tracks): nms_inp = torch.cat((nms_inp_reg[i].view(1, -1), nms_inp_det), 0) keep = nms(nms_inp, self.detection_nms_thresh) keep = keep[torch.ge(keep, 1)] if keep.nelement() == 0: nms_inp_det = nms_inp_det.new(0) break nms_inp_det = nms_inp[keep] if nms_inp_det.nelement() > 0: new_det_pos = nms_inp_det[:, :4] new_det_scores = nms_inp_det[:, 4] # try to reidentify tracks new_det_pos, new_det_scores, new_det_features = self.reid(blob, new_det_pos, new_det_scores) # add new if new_det_pos.nelement() > 0: self.add(new_det_pos, new_det_scores, new_det_features, blob) #################### # Generate Results # #################### for t in self.tracks: track_ind = int(t.id) if track_ind not in self.results.keys(): self.results[track_ind] = {} pos = t.pos[0] / blob['im_info'][0][2] sc = t.score self.results[track_ind][self.im_index] = np.concatenate([pos.cpu().numpy(), np.array([sc])]) for t in self.inactive_tracks: t.count_inactive += 1 if not self.reid_oracle: self.inactive_tracks = [ t for t in self.inactive_tracks if t.has_positive_area() and t.count_inactive <= self.inactive_patience ] self.im_index += 1 self.last_image = blob['data'][0][0]