def init(self, im, state):
r"""Initialize tracker
Internal target state representation: self._state['state'] = (target_pos, target_sz)
Arguments
---------
im : np.array
initial frame image
state
target state on initial frame (bbox in case of SOT), format: xywh
"""
rect = state # bbox in xywh format is given for initialization in case of tracking
box = xywh2cxywh(rect)
target_pos, target_sz = box[:2], box[2:]
self._state['im_h'] = im.shape[0]
self._state['im_w'] = im.shape[1]
# extract template feature
features, im_z_crop, avg_chans = self.feature(im, target_pos, target_sz)
score_size = self._hyper_params['score_size']
window = self.get_window(score_size)
lost_search_window = self.get_window(
self._hyper_params["lost_score_size"])
self._state['z_crop'] = im_z_crop
self._state['avg_chans'] = avg_chans
self._state['features'] = features
self._state['window'] = window
self._state['lost_search_window'] = lost_search_window
self._state['state'] = (target_pos, target_sz)
def init(self, im, state):
r"""
Initialize tracker
Internal target state representation: self._state['state'] = (target_pos, target_sz)
:param im: initial frame image
:param state: bbox, format: xywh
:return: None
"""
rect = state # bbox in xywh format is given for initialization in case of tracking
box = xywh2cxywh(rect)
target_pos, target_sz = box[:2], box[2:]
self._state['im_h'] = im.shape[0]
self._state['im_w'] = im.shape[1]
# extract template feature
features, im_z_crop, avg_chans = self.feature(im, target_pos,
target_sz)
score_size = self._hyper_params['score_size']
if self._hyper_params['windowing'] == 'cosine':
window = np.outer(np.hanning(score_size), np.hanning(score_size))
window = window.reshape(-1)
elif self._hyper_params['windowing'] == 'uniform':
window = np.ones((score_size, score_size))
else:
window = np.ones((score_size, score_size))
self._state['z_crop'] = im_z_crop
self._state['avg_chans'] = avg_chans
self._state['features'] = features
self._state['window'] = window
# self.state['target_pos'] = target_pos
# self.state['target_sz'] = target_sz
self._state['state'] = (target_pos, target_sz)
def _coord_back(self, rect, size=257, region=129):
"""
Warp the predicted coordinates from cropped patch back to original image.
:param rect: rect with coords in cropped patch
:param size: image size of cropped patch
:param region: region size with template = 127
:return: rect(xywh) and cxywh in full image
"""
target_pos, _ = self._state['current_state']
scale_x = self._state['scale_x']
zoom_ratio = size / region
scale = scale_x * zoom_ratio
cx_f, cy_f = target_pos[0], target_pos[1]
cx_c, cy_c = (size - 1) / 2, (size - 1) / 2
a, b = 1 / (scale), 1 / (scale)
c = cx_f - a * cx_c
d = cy_f - b * cy_c
x1, y1, w, h = rect[0], rect[1], rect[2], rect[3]
x1_t = a * x1 + c
y1_t = b * y1 + d
w_t, h_t = w * a, h * b
return [x1_t, y1_t, w_t, h_t], xywh2cxywh([x1_t, y1_t, w_t, h_t])
def init(self, im, state):
r"""Initialize tracker
Internal target state representation: self._state['state'] = (target_pos, target_sz)
Arguments
---------
im : np.array
initial frame image
state
target state on initial frame (bbox in case of SOT), format: xywh
"""
self.frame_num = 1
self.temp_max = 0
rect = state # bbox in xywh format is given for initialization in case of tracking
box = xywh2cxywh(rect)
target_pos, target_sz = box[:2], box[2:]
self._state['im_h'] = im.shape[0]
self._state['im_w'] = im.shape[1]
# extract template feature
features, im_z_crop, avg_chans, im_z_crop_t = self.feature(
im, target_pos, target_sz)
score_size = self._hyper_params['score_size']
if self._hyper_params['windowing'] == 'cosine':
window = np.outer(np.hanning(score_size), np.hanning(score_size))
window = window.reshape(-1)
elif self._hyper_params['windowing'] == 'uniform':
window = np.ones((score_size, score_size))
else:
window = np.ones((score_size, score_size))
self._state['z_crop'] = im_z_crop
self._state['z0_crop'] = im_z_crop_t
with torch.no_grad():
self._model.instance(im_z_crop_t)
self._state['avg_chans'] = avg_chans
self._state['features'] = features
self._state['window'] = window
self._state['state'] = (target_pos, target_sz)
# init online classifier
z_size = self._hyper_params['z_size']
x_size = self._hyper_params['x_size']
context_amount = self._hyper_params['context_amount']
init_im_crop, scale_x = get_crop(
im,
target_pos,
target_sz,
z_size,
x_size=x_size * 2,
avg_chans=avg_chans,
context_amount=context_amount,
func_get_subwindow=get_subwindow_tracking,
)
init_x_crop_t = imarray_to_tensor(init_im_crop)
self.online_classifier.initialize(init_x_crop_t, state)
def initial_manuel(frame):
initBB = []
while True:
initBB.append(
cv2.selectROI("Frame", frame, fromCenter=False,
showCrosshair=True))
cv2.rectangle(
frame, (initBB[-1][0], initBB[-1][1]),
(initBB[-1][0] + initBB[-1][2], initBB[-1][1] + initBB[-1][3]),
(0, 255, 0), 2)
initBB[-1] = xywh2cxywh(initBB[-1])
key = cv2.waitKey(0) & 0xFF
if key == ord("n"):
initBB[-1] = np.append(initBB[-1], 0)
elif key == ord("q"):
initBB = initBB[:-1]
break
else:
initBB[-1] = np.append(initBB[-1], 1)
return initBB
def update(self, im, state=None):
""" Perform tracking on current frame
Accept provided target state prior on current frame
e.g. search the target in another video sequence simutanously
Arguments
---------
im : np.array
current frame image
state
provided target state prior (bbox in case of SOT), format: xywh
"""
# use prediction on the last frame as target state prior
if state is None:
target_pos_prior, target_sz_prior = self._state['state']
# use provided bbox as target state prior
else:
rect = state # bbox in xywh format is given for initialization in case of tracking
box = xywh2cxywh(rect).reshape(4)
target_pos_prior, target_sz_prior = box[:2], box[2:]
features = self._state['features']
# forward inference to estimate new state
target_pos, target_sz = self.track(im,
target_pos_prior,
target_sz_prior,
features,
update_state=True)
# save underlying state
# self.state['target_pos'], self.state['target_sz'] = target_pos, target_sz
self._state['state'] = target_pos, target_sz
# return rect format
track_rect = cxywh2xywh(np.concatenate([target_pos, target_sz],
axis=-1))
if self._hyper_params["corr_fea_output"]:
return target_pos, target_sz, self._state["corr_fea"]
return track_rect
def init(self, im, state, init_mask):
"""
initialize the whole pipeline :
tracker init => global modeling loop init
:param im: init frame
:param state: bbox in xywh format
:param init_mask: binary mask of target object in shape (h,w)
"""
#========== SiamFC++ init ==============
self._tracker.init(im, state)
avg_chans = self._tracker.get_avg_chans()
self._state['avg_chans'] = avg_chans
rect = state # bbox in xywh format is given for initialization in case of tracking
box = xywh2cxywh(rect)
target_pos, target_sz = box[:2], box[2:]
self._state['state'] = (target_pos, target_sz)
self._state['im_h'] = im.shape[0]
self._state['im_w'] = im.shape[1]
# ========== Global Modeling Loop init ==============
init_image, _ = get_crop(
im,
target_pos,
target_sz,
z_size=self._hyper_params["z_size"],
x_size=self._hyper_params["GMP_image_size"],
avg_chans=avg_chans,
context_amount=self._hyper_params["context_amount"],
func_get_subwindow=get_subwindow_tracking,
)
init_mask_c3 = np.stack([init_mask, init_mask, init_mask],
-1).astype(np.uint8)
init_mask_crop_c3, _ = get_crop(
init_mask_c3,
target_pos,
target_sz,
z_size=self._hyper_params["z_size"],
x_size=self._hyper_params["GMP_image_size"],
avg_chans=avg_chans * 0,
context_amount=self._hyper_params["context_amount"],
func_get_subwindow=get_subwindow_tracking,
)
init_mask_crop = init_mask_crop_c3[:, :, 0]
init_mask_crop = (init_mask_crop >
self._hyper_params['mask_filter_thresh']).astype(
np.uint8)
init_mask_crop = np.expand_dims(init_mask_crop,
axis=-1) #shape: (129,129,1)
filtered_image = init_mask_crop * init_image
self._state['filtered_image'] = filtered_image #shape: (129,129,3)
with torch.no_grad():
deep_feature = self._segmenter(imarray_to_tensor(filtered_image).to(
self.device),
phase='global_feature')[0]
self._state['seg_init_feature'] = deep_feature #shape : (1,256,5,5)
self._state['seg_global_feature'] = deep_feature
self._state['gml_feature'] = deep_feature
self._state['conf_score'] = 1
