def update(self, im):
# get track
target_pos_prior, target_sz_prior = self._state['state']
self._state['current_state'] = deepcopy(self._state['state'])
# forward inference to estimate new state
# tracking for VOS returns regressed box and correlation feature
self._tracker.set_state(self._state["state"])
target_pos_track, target_sz_track, corr_feature = self._tracker.update(
im)
# segmentation returnd predicted masks
gml_feature = self._state['gml_feature']
pred_mask, pred_mask_b = self.joint_segmentation(
im, target_pos_prior, target_sz_prior, corr_feature, gml_feature)
# cropping strategy loop swtiches the coordinate prediction method
if self._hyper_params['cropping_strategy']:
target_pos, target_sz = self.cropping_strategy(
pred_mask_b, target_pos_track, target_sz_track)
else:
target_pos, target_sz = target_pos_track, target_sz_track
# global modeling loop updates global feature for next frame's segmentation
if self._hyper_params['global_modeling']:
if self._state["state_score"] > self._hyper_params[
"update_global_fea_th"]:
self.global_modeling()
# save underlying state
self._state['state'] = target_pos, target_sz
track_rect = cxywh2xywh(
np.concatenate([target_pos_track, target_sz_track], axis=-1))
self._state['track_box'] = track_rect
return self._state['mask_in_full_image']
def check(frame, result, yolo_objects, dataqueues):
global process_to_insert, process_tracking_num
temp = []
for label, rect_box in result.items():
if label in temp:
continue
for other_lable, other_box in result.items():
if int(label) < int(other_lable) and rect_box.inbox_box(
other_box.get_boxes()) > 0.9:
if other_lable not in temp:
temp.append(other_lable)
print("there is a error")
for label in temp:
del result[label]
if len(yolo_objects) == 0:
return result, temp
else:
print(yolo_objects)
add_object = [[] for i in range(process_length)]
for objects in yolo_objects:
if len(result) == tracking_num:
break
objects = np.append(cxywh2xywh(objects[0:4]), objects[-1])
need = True
if (objects[-1] < 1):
for rect_box in result.values():
IOU = rect_box.inbox_box(objects[0:4])
if IOU > 0.9:
need = False
break
if need:
# print("update",process_to_insert*100+process_tracking_num[process_to_insert], process_to_insert)
result[
str(process_to_insert * 100 +
process_tracking_num[process_to_insert])] = trackthing(
objects[0:4], objects[-1])
add_object[process_to_insert].append(objects)
process_tracking_num[process_to_insert] += 1
process_to_insert = (process_to_insert + 1) % process_length
for i in range(process_length):
dataqueues[i].put((frame, add_object[i], []))
return result, temp
def update(self, im):
# get track
target_pos_prior, target_sz_prior = self._state['state']
features = self._state['features']
# forward inference to estimate new state
target_pos, target_sz = self.track(im,
target_pos_prior,
target_sz_prior,
features,
update_state=True)
# save underlying state
self._state['state'] = target_pos, target_sz
# return rect format
track_rect = cxywh2xywh(
np.concatenate([target_pos, target_sz], axis=-1))
return track_rect
def update(self, im, state=None):
""" Perform tracking on current frame
Accept provided target state prior on current frame
e.g. search the target in another video sequence simutanously
Arguments
---------
im : np.array
current frame image
state
provided target state prior (bbox in case of SOT), format: xywh
"""
# use prediction on the last frame as target state prior
if state is None:
target_pos_prior, target_sz_prior = self._state['state']
# use provided bbox as target state prior
else:
rect = state # bbox in xywh format is given for initialization in case of tracking
box = xywh2cxywh(rect).reshape(4)
target_pos_prior, target_sz_prior = box[:2], box[2:]
features = self._state['features']
# forward inference to estimate new state
target_pos, target_sz = self.track(im,
target_pos_prior,
target_sz_prior,
features,
update_state=True)
# save underlying state
# self.state['target_pos'], self.state['target_sz'] = target_pos, target_sz
self._state['state'] = target_pos, target_sz
# return rect format
track_rect = cxywh2xywh(np.concatenate([target_pos, target_sz],
axis=-1))
if self._hyper_params["corr_fea_output"]:
return target_pos, target_sz, self._state["corr_fea"]
return track_rect
def update(self, im):
# get track
# target_pos_prior, target_sz_prior = self.state['target_pos'], self.state['target_sz']
target_pos_prior, target_sz_prior = self._state['state']
f_z = self._state['f_z']
# forward inference to estimate new state
# tracking for VOS returns regressed box and correlation feature
target_pos_track, target_sz_track, corr_feature = self.track4vos(
im, target_pos_prior, target_sz_prior, f_z, update_state=True)
# segmentation returnd predicted masks
gml_feature = self._state['gml_feature']
pred_mask, pred_mask_b = self.joint_segmentation(
im, target_pos_prior, target_sz_prior, corr_feature, gml_feature)
# template updation for tracker
if self._hyper_params['track_ema']:
self.track_template_updation(im, target_pos_track, target_sz_track)
# global modeling loop updates global feature for next frame's segmentation
if self._hyper_params['global_modeling']:
self.global_modeling()
# cropping strategy loop swtiches the coordinate prediction method
if self._hyper_params['cropping_strategy']:
target_pos, target_sz = self.cropping_strategy(
pred_mask_b, target_pos_track, target_sz_track)
else:
target_pos, target_sz = target_pos_track, target_sz_track
# save underlying state
self._state['state'] = target_pos, target_sz
# return rect format
track_rect = cxywh2xywh(
np.concatenate([target_pos_track, target_sz_track], axis=-1))
return track_rect
def update(self, im):
# get track
# target_pos_prior, target_sz_prior = self.state['target_pos'], self.state['target_sz']
target_pos_prior, target_sz_prior = self._state['state']
features = self._state['features']
# forward inference to estimate new state
target_pos, target_sz = self.track(im,
target_pos_prior,
target_sz_prior,
features,
update_state=True)
# save underlying state
# self.state['target_pos'], self.state['target_sz'] = target_pos, target_sz
self._state['state'] = target_pos, target_sz
# return rect format
track_rect = cxywh2xywh(np.concatenate([target_pos, target_sz],
axis=-1))
if self._hyper_params["corr_fea_output"]:
return target_pos, target_sz, self._state["corr_fea"]
return track_rect
def multiprocessing_update(task, task_cfg, index, im, dataqueue, resultqueue):
# build model
Model = model_builder.build_model(task,
task_cfg.model).to(torch.device("cuda"))
Model.eval()
target_pos = []
target_sz = []
im_z_crops = []
lost = []
features = []
tracking_index = []
total_num = 0
avg_chans = np.mean(im, axis=(0, 1))
im_h, im_w = im.shape[0], im.shape[1]
z_size = hyper_params['z_size']
x_size = hyper_params['x_size']
context_amount = hyper_params['context_amount']
phase = hyper_params['phase_init']
phase_track = hyper_params['phase_track']
score_size = (
hyper_params['x_size'] - hyper_params['z_size']
) // hyper_params['total_stride'] + 1 - hyper_params['num_conv3x3'] * 2
if hyper_params['windowing'] == 'cosine':
window = np.outer(np.hanning(score_size), np.hanning(score_size))
window = window.reshape(-1)
elif hyper_params['windowing'] == 'uniform':
window = np.ones((score_size, score_size))
else:
window = np.ones((score_size, score_size))
def init(state, im_x, total_num):
for i in range(len(state)):
target_pos.append(state[i][:2])
target_sz.append(state[i][2:4])
tracking_index.append(index * 100 + total_num + i)
im_z_crop, _ = get_crop(im_x,
target_pos[i],
target_sz[i],
z_size,
avg_chans=avg_chans,
context_amount=context_amount,
func_get_subwindow=get_subwindow_tracking)
im_z_crops.append(im_z_crop)
array = torch.from_numpy(
np.ascontiguousarray(
im_z_crops[i].transpose(2, 0, 1)[np.newaxis, ...],
np.float32)).to(torch.device("cuda"))
lost.append(0)
with torch.no_grad():
features.append(Model(array, phase=phase))
def delete_node(j):
try:
del target_pos[j]
del target_sz[j]
del features[j]
del tracking_index[j]
del lost[j]
except Exception as error:
print("delete error", error)
while True:
try:
im_x, state, delete = dataqueue.get()
except Exception as error:
print(error)
continue
else:
if len(state) > 0:
init(state, im_x, total_num)
total_num += len(state)
continue
if len(delete) > 0:
delete_list = []
for i in delete:
if i in tracking_index:
# print("delete",i)
node = tracking_index.index(i)
delete_node(node)
result = []
im = im_x.copy()
del im_x, state, delete
for i in range(len(features)):
im_x_crop, scale_x = get_crop(
im,
target_pos[i],
target_sz[i],
z_size,
x_size=x_size,
avg_chans=avg_chans,
context_amount=context_amount,
func_get_subwindow=get_subwindow_tracking)
array = torch.from_numpy(
np.ascontiguousarray(
im_x_crop.transpose(2, 0, 1)[np.newaxis, ...],
np.float32)).to(torch.device("cuda"))
with torch.no_grad():
score, box, cls, ctr, *args = Model(array,
*features[i],
phase=phase_track)
box = tensor_to_numpy(box[0])
score = tensor_to_numpy(score[0])[:, 0]
cls = tensor_to_numpy(cls[0])
ctr = tensor_to_numpy(ctr[0])
box_wh = xyxy2cxywh(box)
# #lost goal
if score.max() < 0.2:
lost[i] += 1
continue
elif lost[i] > 0:
lost[i] -= 1
best_pscore_id, pscore, penalty = postprocess_score(
score, box_wh, target_sz[i], scale_x, window)
# box post-processing
new_target_pos, new_target_sz = postprocess_box(
best_pscore_id, score, box_wh, target_pos[i], target_sz[i],
scale_x, x_size, penalty)
new_target_pos, new_target_sz = restrict_box(
im_h, im_w, new_target_pos, new_target_sz)
# save underlying state
target_pos[i], target_sz[i] = new_target_pos, new_target_sz
# return rect format
track_rect = cxywh2xywh(
np.concatenate([target_pos[i], target_sz[i]], axis=-1))
result.append(track_rect)
delete_list = []
for i in range(len(features)):
if lost[i] > 10:
delete_list = []
delete_list.append(i)
for i in delete_list:
delete_node(i)
# print(index, len(features))
resultqueue.put([result, tracking_index])
process[-1].start()
if args["detect"] == "manual":
initBB = initial_manuel(frame)
else:
from yolo import YOLO
yolo1 = YOLO()
initBB = update_yolo(frame, yolo1)
initBB = initBB[:tracking_num]
for i in range(len(dataqueues)):
temp = initBB[int(len(initBB) / len(dataqueues) *
i):int(len(initBB) / len(dataqueues) * (i + 1))]
for j in range(len(temp)):
tracking_object[str(i * 100 + j)] = trackthing(
cxywh2xywh(temp[j][0:4]), temp[j][-1])
process_tracking_num.append(len(temp))
dataqueues[i].put((frame, temp, []))
cv2.setMouseCallback('Frame', get_point)
fps = FPS().start()
while True:
# VideoStream or VideoCapture object
frame = vs.read()
frame = frame[1] if args.get("video", False) else frame
frame = frame[:, 240:-240]
# check to see if we have reached the end of the stream
if frame is None:
break
# resize the frame (so we can process it faster) and grab the
frame = imutils.resize(frame, height=720, width=720)
