def regress_tracks(self, blob): pos = self.get_pos() # regress _, scores, bbox_pred, rois = self.obj_detect.test_rois(pos) boxes = bbox_transform_inv(rois, bbox_pred) boxes = clip_boxes(Variable(boxes), blob['im_info'][0][:2]).data pos = boxes[:, self.cl*4:(self.cl+1)*4] scores = scores[:, self.cl] s = [] for i in range(len(self.tracks)-1,-1,-1): t = self.tracks[i] t.score = scores[i] if scores[i] <= self.regression_person_thresh and not self.kill_oracle: self.tracks_to_inactive([t]) else: s.append(scores[i]) if self.regress: t.pos = pos[i].view(1,-1) return torch.Tensor(s[::-1]).cuda()
def step(self, blob):
for t in self.tracks:
t.last_pos = t.pos.clone()
###########################
# Look for new detections #
###########################
self.obj_detect.load_image(blob['data'][0], blob['im_info'][0])
if self.public_detections:
dets = blob['dets']
if len(dets) > 0:
dets = torch.cat(dets, 0)[:,:4]
_, scores, bbox_pred, rois = self.obj_detect.test_rois(dets)
else:
rois = torch.zeros(0).cuda()
else:
_, scores, bbox_pred, rois = self.obj_detect.detect()
if rois.nelement() > 0:
boxes = bbox_transform_inv(rois, bbox_pred)
boxes = clip_boxes(Variable(boxes), blob['im_info'][0][:2]).data
# Filter out tracks that have too low person score
scores = scores[:, self.cl]
inds = torch.gt(scores, self.detection_person_thresh).nonzero().view(-1)
else:
inds = torch.zeros(0).cuda()
if inds.nelement() > 0:
boxes = boxes[inds]
det_pos = boxes[:, self.cl*4:(self.cl+1)*4]
det_scores = scores[inds]
else:
det_pos = torch.zeros(0).cuda()
det_scores = torch.zeros(0).cuda()
##################
# Predict tracks #
##################
num_tracks = 0
nms_inp_reg = torch.zeros(0).cuda()
if len(self.tracks):
# align
if self.do_align:
self.align(blob)
if self.pos_oracle or self.kill_oracle:
self.oracle(blob)
#regress
if len(self.tracks):
person_scores = self.regress_tracks(blob)
# now NMS step
if self.kill_oracle:
person_scores = self.nms_oracle(blob, person_scores)
if len(self.tracks):
# create nms input
new_features = self.get_appearances(blob)
# nms here if tracks overlap
nms_inp_reg = torch.cat((self.get_pos(), person_scores.add_(3).view(-1,1)),1)
if self.kill_oracle:
keep = torch.arange(nms_inp_reg.size(0)).long().cuda()
else:
keep = nms(nms_inp_reg, self.regression_nms_thresh)
# # Plot the killed tracks for debugging
self.tracks_to_inactive([self.tracks[i]
for i in list(range(len(self.tracks)))
if i not in keep])
if keep.nelement() > 0:
nms_inp_reg = nms_inp_reg[keep]
new_features[keep]
self.add_features(new_features)
num_tracks = nms_inp_reg.size(0)
else:
nms_inp_reg = torch.zeros(0).cuda()
num_tracks = 0
#####################
# Create new tracks #
#####################
# create nms input and nms new detections
if det_pos.nelement() > 0:
nms_inp_det = torch.cat((det_pos, det_scores.view(-1,1)), 1)
else:
nms_inp_det = torch.zeros(0).cuda()
if nms_inp_det.nelement() > 0:
keep = nms(nms_inp_det, self.detection_nms_thresh)
nms_inp_det = nms_inp_det[keep]
# check with every track in a single run (problem if tracks delete each other)
for i in range(num_tracks):
nms_inp = torch.cat((nms_inp_reg[i].view(1,-1), nms_inp_det), 0)
keep = nms(nms_inp, self.detection_nms_thresh)
keep = keep[torch.ge(keep,1)]
if keep.nelement() == 0:
nms_inp_det = nms_inp_det.new(0)
break
nms_inp_det = nms_inp[keep]
if nms_inp_det.nelement() > 0:
new_det_pos = nms_inp_det[:,:4]
new_det_scores = nms_inp_det[:,4]
# try to redientify tracks
new_det_pos, new_det_scores, new_det_features = self.reid(blob, new_det_pos, new_det_scores)
# add new
if new_det_pos.nelement() > 0:
self.add(new_det_pos, new_det_scores, new_det_features, blob)
####################
# Generate Results #
####################
for t in self.tracks:
track_ind = int(t.id)
if track_ind not in self.results.keys():
self.results[track_ind] = {}
pos = t.pos[0] / blob['im_info'][0][2]
sc = t.score
self.results[track_ind][self.im_index] = np.concatenate([pos.cpu().numpy(), np.array([sc])])
self.im_index += 1
self.last_image = blob['data'][0][0]
if not self.reid_oracle:
self.clear_inactive()
def step(self, blob):
for t in self.tracks:
# add current position to last_pos list
t.last_pos.append(t.pos.clone())
###########################
# Look for new detections #
###########################
self.obj_detect.load_image(blob['data'][0], blob['im_info'][0])
if self.public_detections:
dets = blob['dets']
if len(dets) > 0:
dets = torch.cat(dets, 0)[:, :4]
_, scores, bbox_pred, rois = self.obj_detect.test_rois(dets)
else:
rois = torch.zeros(0).cuda()
else:
_, scores, bbox_pred, rois = self.obj_detect.detect()
if rois.nelement() > 0:
boxes = bbox_transform_inv(rois, bbox_pred)
boxes = clip_boxes(Variable(boxes), blob['im_info'][0][:2]).data
scores = scores[:, self.cl]
if self.kill_oracle:
gt = blob['gt']
gt_boxes = torch.cat(list(gt.values()), 0).cuda()
# calculate IoU distances
iou_neg = 1 - bbox_overlaps(boxes, gt_boxes)
dist_mat = iou_neg.cpu().numpy()
row_ind, col_ind = linear_sum_assignment(dist_mat)
matched = []
# normal matching
for r, c in zip(row_ind, col_ind):
if dist_mat[r, c] <= 0.5:
matched.append(r.item())
inds = torch.LongTensor(matched).cuda()
else:
# Filter out tracks that have too low person score
inds = torch.gt(scores, self.detection_person_thresh).nonzero().view(-1)
else:
inds = torch.zeros(0).cuda()
if inds.nelement() > 0:
boxes = boxes[inds]
det_pos = boxes[:, self.cl * 4:(self.cl + 1) * 4]
det_scores = scores[inds]
else:
det_pos = torch.zeros(0).cuda()
det_scores = torch.zeros(0).cuda()
##################
# Predict tracks #
##################
num_tracks = 0
nms_inp_reg = torch.zeros(0).cuda()
if len(self.tracks):
# align
if self.do_align:
self.align(blob)
if self.pos_oracle or self.kill_oracle:
self.oracle(blob)
elif self.motion_model_cfg['enabled']:
self.motion()
if self.reid_oracle:
self.tracks_to_inactive([t for t in self.tracks if not t.has_positive_area()])
else:
self.tracks = [t for t in self.tracks if t.has_positive_area()]
# regress
if len(self.tracks):
person_scores = self.regress_tracks(blob)
# now NMS step
if self.kill_oracle:
person_scores = self.nms_oracle(blob, person_scores)
if len(self.tracks):
# create nms input
new_features = self.get_appearances(blob)
# nms here if tracks overlap
nms_inp_reg = torch.cat((self.get_pos(), person_scores.add_(3).view(-1, 1)), 1)
if self.kill_oracle:
# keep all
keep = torch.arange(nms_inp_reg.size(0)).long().cuda()
else:
keep = nms(nms_inp_reg, self.regression_nms_thresh)
# Plot the killed tracks for debugging
self.tracks_to_inactive([self.tracks[i] for i in list(range(len(self.tracks))) if i not in keep])
if keep.nelement() > 0:
nms_inp_reg = nms_inp_reg[keep]
self.add_features(new_features)
num_tracks = nms_inp_reg.size(0)
else:
nms_inp_reg = torch.zeros(0).cuda()
num_tracks = 0
#####################
# Create new tracks #
#####################
# create nms input and nms new detections
if det_pos.nelement() > 0:
nms_inp_det = torch.cat((det_pos, det_scores.view(-1, 1)), 1)
else:
nms_inp_det = torch.zeros(0).cuda()
if nms_inp_det.nelement() > 0:
keep = nms(nms_inp_det, self.detection_nms_thresh)
nms_inp_det = nms_inp_det[keep]
# check with every track in a single run (problem if tracks delete each other)
for i in range(num_tracks):
nms_inp = torch.cat((nms_inp_reg[i].view(1, -1), nms_inp_det), 0)
keep = nms(nms_inp, self.detection_nms_thresh)
keep = keep[torch.ge(keep, 1)]
if keep.nelement() == 0:
nms_inp_det = nms_inp_det.new(0)
break
nms_inp_det = nms_inp[keep]
if nms_inp_det.nelement() > 0:
new_det_pos = nms_inp_det[:, :4]
new_det_scores = nms_inp_det[:, 4]
# try to reidentify tracks
new_det_pos, new_det_scores, new_det_features = self.reid(blob, new_det_pos, new_det_scores)
# add new
if new_det_pos.nelement() > 0:
self.add(new_det_pos, new_det_scores, new_det_features, blob)
####################
# Generate Results #
####################
for t in self.tracks:
track_ind = int(t.id)
if track_ind not in self.results.keys():
self.results[track_ind] = {}
pos = t.pos[0] / blob['im_info'][0][2]
sc = t.score
self.results[track_ind][self.im_index] = np.concatenate([pos.cpu().numpy(), np.array([sc])])
for t in self.inactive_tracks:
t.count_inactive += 1
if not self.reid_oracle:
self.inactive_tracks = [
t for t in self.inactive_tracks if t.has_positive_area() and t.count_inactive <= self.inactive_patience
]
self.im_index += 1
self.last_image = blob['data'][0][0]