def update(self, instance_rgb_img, instance_ir_img):
"""track object based on the previous frame
Args:
frame: an RGB image
Returns:
bbox: tuple of 1-based bounding box(xmin, ymin, xmax, ymax)
"""
instance_rgb_img = np.asarray(instance_rgb_img)
frame = instance_rgb_img
#cv2.imshow('instance_img', instance_ir_img)
self.img_mean = np.mean(instance_rgb_img, axis=(0, 1))
instance_rgb_img, _, _, scale_x = self.old_loader.get_instance_image( instance_rgb_img,
self.bbox,
config.template_img_size,
config.detection_img_size,
config.context_amount,
self.img_mean)
self.img_mean_ir = np.mean(instance_ir_img, axis=(0, 1))
instance_ir_img, _, _, _ = self.data_loader.get_instance_image( instance_ir_img,
self.bbox,
config.template_img_size,
config.detection_img_size,
config.context_amount,
self.img_mean_ir)
instance_rgb_img = self.transforms(instance_rgb_img)[None, :, :, :]
instance_ir_img = self.transforms(instance_ir_img)[None, :, :, :]
instance_ir_img = torch.from_numpy(np.zeros(instance_ir_img.size())).float()
if self.cuda:
pred_score, pred_regression = self.model.track(instance_rgb_img.cuda(), instance_ir_img.cuda())
else:
pred_score, pred_regression = self.model.track(instance_rgb_img, instance_ir_img)
pred_conf = pred_score.reshape(-1, 2, config.size ).permute(0, 2, 1)
pred_offset = pred_regression.reshape(-1, 4, config.size ).permute(0, 2, 1)
delta = pred_offset[0].cpu().detach().numpy()
#print(delta)
box_pred = util.box_transform_inv(self.anchors, delta)
#print(box_pred)
score_pred = F.softmax(pred_conf, dim=2)[0, :, 1].cpu().detach().numpy()
#print(score_pred)
s_c = util.change(util.sz(box_pred[:, 2], box_pred[:, 3]) / (util.sz_wh(self.target_sz * scale_x))) # scale penalty
r_c = util.change((self.target_sz[0] / self.target_sz[1]) / (box_pred[:, 2] / box_pred[:, 3])) # ratio penalty
penalty = np.exp(-(r_c * s_c - 1.) * config.penalty_k)
#print('penalty', penalty)
pscore = penalty * score_pred
pscore = pscore * (1 - config.window_influence) + self.window * config.window_influence
#print('window', self.window)
best_pscore_id = np.argmax(pscore)
#print('id', np.argmax(pscore))
target = box_pred[best_pscore_id, :] / scale_x
#print(target)
lr = penalty[best_pscore_id] * score_pred[best_pscore_id] * config.lr_box
res_x = np.clip(target[0] + self.pos[0], 0, frame.shape[1])
#print('resx', target[0] + self.pos[0])
res_y = np.clip(target[1] + self.pos[1], 0, frame.shape[0])
res_w = np.clip(self.target_sz[0] * (1 - lr) + target[2] * lr, config.min_scale * self.origin_target_sz[0],
config.max_scale * self.origin_target_sz[0])
res_h = np.clip(self.target_sz[1] * (1 - lr) + target[3] * lr, config.min_scale * self.origin_target_sz[1],
config.max_scale * self.origin_target_sz[1])
#print('res_h', self.target_sz[1] * (1 - lr))
self.pos = np.array([res_x, res_y])
self.target_sz = np.array([res_w, res_h])
bbox = np.array([res_x, res_y, res_w, res_h])
#print(bbox)
self.bbox = (
np.clip(bbox[0], 0, frame.shape[1]).astype(np.float64),
np.clip(bbox[1], 0, frame.shape[0]).astype(np.float64),
np.clip(bbox[2], 10, frame.shape[1]).astype(np.float64),
np.clip(bbox[3], 10, frame.shape[0]).astype(np.float64))
res_x = res_x - res_w/2 # x -> x1
res_y = res_y - res_h/2 # y -> y1
bbox = np.array([res_x, res_y, res_w, res_h])
#print('result', bbox)
return bbox
def update(self, frame_rgb, frame_ir):
"""track object based on the previous frame
Args:
frame: an RGB image
Returns:
bbox: tuple of 1-based bounding box(xmin, ymin, xmax, ymax)
"""
frame_rgb = np.asarray(frame_rgb)
frame_ir = np.asarray(frame_ir)
instance_img_rgb, _, _, scale_x = self.data_loader.get_instance_image( frame_rgb,
self.bbox,
config.template_img_size,
config.detection_img_size,
config.context_amount,
self.img_rgb_mean)
instance_img_ir, _, _, scale_x = self.data_loader.get_instance_image(frame_ir,
self.bbox,
config.template_img_size,
config.detection_img_size,
config.context_amount,
self.img_ir_mean)
instance_img_rgb = self.transforms(instance_img_rgb)[None, :, :, :]
instance_img_ir = self.transforms(instance_img_ir)[None, :, :, :]
if self.cuda:
instance_img_rgb = instance_img_rgb.cuda()
instance_img_ir = instance_img_ir.cuda()
if self.modality == 1:
pred_score, pred_regression = self.model.track(instance_img_rgb)
else:
pred_score, pred_regression = self.model.track(instance_img_rgb, instance_img_ir)
pred_conf = pred_score.reshape(-1, 2, config.size ).permute(0, 2, 1)
pred_offset = pred_regression.reshape(-1, 4, config.size ).permute(0, 2, 1)
delta = pred_offset[0].cpu().detach().numpy()
box_pred = util.box_transform_inv(self.anchors, delta)
score_pred = F.softmax(pred_conf, dim=2)[0, :, 1].cpu().detach().numpy()
s_c = util.change(util.sz(box_pred[:, 2], box_pred[:, 3]) / (util.sz_wh(self.target_sz * scale_x))) # scale penalty
r_c = util.change((self.target_sz[0] / self.target_sz[1]) / (box_pred[:, 2] / box_pred[:, 3])) # ratio penalty
penalty = np.exp(-(r_c * s_c - 1.) * config.penalty_k)
pscore = penalty * score_pred
pscore = pscore * (1 - config.window_influence) + self.window * config.window_influence
best_pscore_id = np.argmax(pscore)
target = box_pred[best_pscore_id, :] / scale_x
lr = penalty[best_pscore_id] * score_pred[best_pscore_id] * config.lr_box
res_x = np.clip(target[0] + self.pos[0], 0, frame_rgb.shape[1])
res_y = np.clip(target[1] + self.pos[1], 0, frame_rgb.shape[0])
res_w = np.clip(self.target_sz[0] * (1 - lr) + target[2] * lr, config.min_scale * self.origin_target_sz[0],
config.max_scale * self.origin_target_sz[0])
res_h = np.clip(self.target_sz[1] * (1 - lr) + target[3] * lr, config.min_scale * self.origin_target_sz[1],
config.max_scale * self.origin_target_sz[1])
self.pos = np.array([res_x, res_y])
self.target_sz = np.array([res_w, res_h])
bbox = np.array([res_x, res_y, res_w, res_h])
self.bbox = (
np.clip(bbox[0], 0, frame_rgb.shape[1]).astype(np.float64),
np.clip(bbox[1], 0, frame_rgb.shape[0]).astype(np.float64),
np.clip(bbox[2], 10, frame_rgb.shape[1]).astype(np.float64),
np.clip(bbox[3], 10, frame_rgb.shape[0]).astype(np.float64))
res_x = res_x - res_w/2 # x -> x1
res_y = res_y - res_h/2 # y -> y1
bbox = np.array([res_x, res_y, res_w, res_h])
return bbox