def __init__(self,
hog=True,
fixed_window=True,
scale=False,
occlusion_threshold=0.3):
self.tracker = KCFTracker(hog=hog,
fixed_window=fixed_window,
multiscale=scale,
occlusion_threshold=occlusion_threshold)
self.kalman = cv2.KalmanFilter(4, 2)
self.kalman.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]],
np.float32)
self.kalman.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
self.kalman.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * 0.001
self.kalman.measurementNoiseCov = np.array([[1, 0], [0, 1]],
np.float32) * 1
self.trace_array = []
self.predict = [0, 0]
# whether kalman filter can be used
self.iskalman_work = False
# whether the object is occluded
self.isocclution = False
self.occlution_index = 0
self.tem_trace_array = []
self.frame_index = 0
self.last_bbox = []
self.kalman_work_num = 0
self.occlusion_threshold = occlusion_threshold
def reset(self, name):
self.seq_loader.get_sequence(name)
self.seq_name = name
ret, img0 = self.seq_loader.get_cap().read()
ret, img1 = self.seq_loader.get_cap().read()
self.action = None
self.action0_count = 0
self.action1_count = 0
self.reward = None
self.action_list = []
# to draw line
self.frame_idx_list = []
self.reward_list = []
self.hist_tracker = HistTracker()
self.kcf_tracker = KCFTracker(hog=True)
self.hist_tracker.init(self.gt_rects[0], img0)
self.kcf_tracker.init(self.gt_rects[0], img0)
bg_img0 = self.get_sub_window(img0)
bg_img1 = self.get_sub_window(img1)
self.last_frame = img1
self.frame_idx = 1
self.init_figure()
return bg_img0, bg_img1
class TrackerMP(Process):
def __init__(self,
hog=True,
fixed_window=False,
multi_scale=True,
input_queue=None,
output_queue=None):
Process.__init__(self)
self.hog = hog
self.fixed_window = fixed_window
self.multi_scale = multi_scale
self.tracker = KCFTracker(hog, fixed_window, multi_scale)
self.input_queue = input_queue
self.output_queue = output_queue
self.image = None
self.label = None
self.is_valid = False
def get_in_queue(self):
return self.input_queue
def get_out_queue(self):
return self.output_queue
def init(self, roi, image):
return self.tracker.init(roi, image)
# print('tracker init with image shape =', image.shape)
def update(self, image):
self.image = image
# print('tracker update with image shape =', image.shape)
def run(self):
while True:
# print('tracker ', os.getpid(), ' run with args', self.hog, self.fixed_window, self.multi_scale)
input_dict = self.input_queue.get()
cmd = input_dict.get('cmd')
if cmd == 'init':
roi = input_dict.get('roi')
image = input_dict.get('image')
self.label = input_dict.get('label')
self.is_valid = self.tracker.init(roi, image)
elif cmd == 'update':
if self.is_valid:
image = input_dict.get('image')
roi, pv = self.tracker.update(image)
self.output_queue.put({
'roi': roi,
'pv': pv,
'label': self.label
})
else:
self.output_queue.put(None)
elif cmd == 'terminate':
self.input_queue.close()
self.output_queue.close()
break
def __init__(self,
hog=True,
fixed_window=False,
multi_scale=True,
input_queue=None,
output_queue=None):
Process.__init__(self)
self.hog = hog
self.fixed_window = fixed_window
self.multi_scale = multi_scale
self.tracker = KCFTracker(hog, fixed_window, multi_scale)
self.input_queue = input_queue
self.output_queue = output_queue
self.image = None
self.label = None
self.is_valid = False
def reset(self, name):
self.seq_loader.get_sequence(name)
self.seq_name = name
self.save_response = os.path.join(self.save_path,self.seq_name, 'response')
self.save_img = os.path.join(self.save_path, self.seq_name, 'imgs')
self.save_hist = os.path.join(self.save_path, self.seq_name, 'color_map')
if not os.path.exists(self.save_response):
os.makedirs(self.save_response)
os.makedirs(self.save_img)
os.makedirs(self.save_hist)
ret, img0 = self.seq_loader.get_cap().read()
ret, img1 = self.seq_loader.get_cap().read()
self.hist_tracker = HistTracker()
self.kcf_tracker = KCFTracker(hog=True)
self.hist_tracker.init(self.gt_rects[0], img0)
self.kcf_tracker.init(self.gt_rects[0], img0)
bg_img0 = self.get_sub_window(img0)
bg_img1 = self.get_sub_window(img1)
self.last_frame = img1
self.frame_idx = 1
cv2.destroyAllWindows()
cv2.namedWindow(self.seq_name)
cv2.namedWindow('%s color map' % self.seq_name)
cv2.namedWindow('%s response map' % self.seq_name)
cv2.moveWindow(self.seq_name, 20, 20)
cv2.moveWindow('%s color map' % self.seq_name, 20 + self.hist_tracker.w + 100, 20)
cv2.moveWindow('%s response map' % self.seq_name, 20 + self.hist_tracker.w + 100 + self.hist_tracker.bg_box[2] + 100, 20)
os.system('''/usr/bin/osascript -e 'tell app "Finder" to set frontmost of process "python" to true' ''')
return bg_img0, bg_img1
class mecfTracker():
def __init__(self,
hog=True,
fixed_window=True,
scale=False,
occlusion_threshold=0.3):
self.tracker = KCFTracker(hog=hog,
fixed_window=fixed_window,
multiscale=scale,
occlusion_threshold=occlusion_threshold)
self.kalman = cv2.KalmanFilter(4, 2)
self.kalman.measurementMatrix = np.array([[1, 0, 0, 0], [0, 1, 0, 0]],
np.float32)
self.kalman.transitionMatrix = np.array(
[[1, 0, 1, 0], [0, 1, 0, 1], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32)
self.kalman.processNoiseCov = np.array(
[[1, 0, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]],
np.float32) * 0.001
self.kalman.measurementNoiseCov = np.array([[1, 0], [0, 1]],
np.float32) * 1
self.trace_array = []
self.predict = [0, 0]
# whether kalman filter can be used
self.iskalman_work = False
# whether the object is occluded
self.isocclution = False
self.occlution_index = 0
self.tem_trace_array = []
self.frame_index = 0
self.last_bbox = []
self.kalman_work_num = 0
self.occlusion_threshold = occlusion_threshold
def init(self, frame, bbox):
self.trace_array.append(bbox)
self.kalman.correct(
np.array([[np.float32(bbox[0])], [np.float32(bbox[1])]]))
self.tracker.init([bbox[0], bbox[1], bbox[2], bbox[3]], frame)
self.frame_index += 1
def update(self, frame):
if self.iskalman_work:
next_bbox = [
self.predict[0], self.predict[1], self.last_bbox[2],
self.last_bbox[3]
]
self.last_bbox, peak_value = self.tracker.update(frame,
next_bbox,
isUse=True)
# long-term
if peak_value > self.occlusion_threshold:
self.trace_array.append(self.last_bbox.copy())
self.kalman.correct(
np.array([[np.float32(self.last_bbox[0])],
[np.float32(self.last_bbox[1])]]))
self.predict = self.kalman.predict()
else:
self.last_bbox = [
next_bbox[0], next_bbox[1], self.last_bbox[2],
self.last_bbox[3]
]
self.predict = self.kalman.predict()
else:
if len(self.trace_array) > 4:
dx = 0
dy = 0
for i in range(-5, -1):
dx += self.trace_array[i + 1][0] - self.trace_array[i][0]
dy += self.trace_array[i + 1][1] - self.trace_array[i][1]
next_bbox = [
self.last_bbox[0] + dx / 4, self.last_bbox[1] + dy / 4,
self.last_bbox[2], self.last_bbox[3]
]
self.last_bbox, peak_value = self.tracker.update(frame,
next_bbox,
isUse=True)
# long-term
if peak_value < 0.3:
self.last_bbox = [
next_bbox[0], next_bbox[1], self.last_bbox[2],
self.last_bbox[3]
]
self.isocclution = True
else:
if self.isocclution == True:
if self.occlution_index != 0:
self.tem_trace_array.append(self.last_bbox.copy())
self.occlution_index += 1
if self.occlution_index == 6:
self.trace_array.extend(self.tem_trace_array)
self.isocclution = False
else:
self.trace_array.append(self.last_bbox.copy())
else:
self.last_bbox, peak_value = self.tracker.update(frame)
self.trace_array.append(self.last_bbox.copy())
if (abs(self.predict[0] - self.last_bbox[0]) <
2) and (abs(self.predict[1] - self.last_bbox[1]) < 2):
self.kalman_work_num += 1
if self.kalman_work_num == 3:
self.iskalman_work = True
self.kalman_work_num = 0
else:
self.kalman_work_num = 0
self.kalman.correct(
np.array([[np.float32(self.last_bbox[0])],
[np.float32(self.last_bbox[1])]]))
self.predict = self.kalman.predict()
self.frame_index += 1
return True, self.last_bbox
class Env:
def __init__(self, seqs_path, data_set_type, save_path=None):
self.seq_loader = SequenceLoader(seqs_path, data_set_type)
self.save_path = save_path
if self.save_path and not os.path.exists(self.save_path):
os.system('mkdir -p %s' % self.save_path)
self.hist_tracker = None
self.kcf_tracker = None
self.last_frame = None
self.seq_name = None
self.action = None
# hyper parameter
self.precision_threshold = 40.0
self.inteval = 1
@property
def gt_rects(self):
return self.seq_loader.gt_rects
# return frame0, frame1
def reset(self, name):
self.seq_loader.get_sequence(name)
self.seq_name = name
self.save_response = os.path.join(self.save_path,self.seq_name, 'response')
self.save_img = os.path.join(self.save_path, self.seq_name, 'imgs')
self.save_hist = os.path.join(self.save_path, self.seq_name, 'color_map')
if not os.path.exists(self.save_response):
os.makedirs(self.save_response)
os.makedirs(self.save_img)
os.makedirs(self.save_hist)
ret, img0 = self.seq_loader.get_cap().read()
ret, img1 = self.seq_loader.get_cap().read()
self.hist_tracker = HistTracker()
self.kcf_tracker = KCFTracker(hog=True)
self.hist_tracker.init(self.gt_rects[0], img0)
self.kcf_tracker.init(self.gt_rects[0], img0)
bg_img0 = self.get_sub_window(img0)
bg_img1 = self.get_sub_window(img1)
self.last_frame = img1
self.frame_idx = 1
cv2.destroyAllWindows()
cv2.namedWindow(self.seq_name)
cv2.namedWindow('%s color map' % self.seq_name)
cv2.namedWindow('%s response map' % self.seq_name)
cv2.moveWindow(self.seq_name, 20, 20)
cv2.moveWindow('%s color map' % self.seq_name, 20 + self.hist_tracker.w + 100, 20)
cv2.moveWindow('%s response map' % self.seq_name, 20 + self.hist_tracker.w + 100 + self.hist_tracker.bg_box[2] + 100, 20)
os.system('''/usr/bin/osascript -e 'tell app "Finder" to set frontmost of process "python" to true' ''')
return bg_img0, bg_img1
def get_sub_window(self, img):
bg_image = self.hist_tracker.sub_window(img, self.hist_tracker.bg_box)
return bg_image
def show_tracking_result(self):
predict_img = self.hist_tracker.get_predict_img(self.gt_rects[self.frame_idx - 1])
cv2.putText(predict_img, 'S: %s' % self.action, (8, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2)
cv2.imshow(self.seq_name, predict_img)
if self.save_path:
cv2.imwrite('%s/%d_predict.jpg'%(self.save_img, self.frame_idx -1), predict_img)
def show_hist(self):
color_map, response_map = self.hist_tracker.get_hist_map()
cv2.imshow('%s color map' % self.seq_name, color_map)
cv2.imshow('%s response map' % self.seq_name, response_map)
if self.save_path:
cv2.imwrite('%s/%d_color_map.jpg'%(self.save_hist, self.frame_idx -1), color_map)
cv2.imwrite('%s/%d_response.jpg'%(self.save_response, self.frame_idx -1), response_map)
def show_all(self):
self.show_tracking_result()
self.show_hist()
cv2.waitKey(self.inteval)
# return reward, next_frame, done
def step(self, action):
tg_box = (None, None, None, None)
if action == 0:
tg_box = self.hist_tracker.run(self.last_frame, None)
self.kcf_tracker.run(self.last_frame, tg_box[:2])
elif action == 1:
tg_box = self.kcf_tracker.run(self.last_frame, None)
tg_box = map(int, tg_box)
self.hist_tracker.run(self.last_frame, tg_box[:2])
else:
print 'action is neither 0 nor 1!!!'
exit()
self.action = action
gt_box = self.gt_rects[self.frame_idx]
gt_cx, gt_cy = gt_box[0] + gt_box[2] / 2.0, gt_box[1] + gt_box[3] / 2.0
tg_cx, tg_cy = tg_box[0] + tg_box[2] / 2.0, tg_box[1] + tg_box[3] / 2.0
# compute reward
dis = math.sqrt((gt_cx - tg_cx) ** 2 + (gt_cy - tg_cy) ** 2)
reward = 0.0 if dis > self.precision_threshold else 1 - (dis / self.precision_threshold)
# crop img
ret, self.last_frame = self.seq_loader.get_cap().read()
self.frame_idx += 1
bg_img = self.get_sub_window(self.last_frame) if ret else None
return reward, bg_img, not ret
class Env:
def __init__(self, seqs_path, data_set_type, save_path=None):
self.seq_loader = SequenceLoader(seqs_path, data_set_type)
self.save_path = save_path
if self.save_path and not os.path.exists(self.save_path):
os.system('mkdir -p %s' % self.save_path)
self.hist_tracker = None
self.kcf_tracker = None
self.last_frame = None
self.seq_name = None
self.action = None
self.action0_count = 0
self.action1_count = 0
self.action_list = []
self.reward = None
# visualization
self.vis = visdom.Visdom()
# to draw line
self.frame_idx_list = []
self.reward_list = []
# hyper parameter
self.precision_threshold = 40.0
self.inteval = 1
@property
def gt_rects(self):
return self.seq_loader.gt_rects
# return frame0, frame1
def reset(self, name):
self.seq_loader.get_sequence(name)
self.seq_name = name
ret, img0 = self.seq_loader.get_cap().read()
ret, img1 = self.seq_loader.get_cap().read()
self.action = None
self.action0_count = 0
self.action1_count = 0
self.reward = None
self.action_list = []
# to draw line
self.frame_idx_list = []
self.reward_list = []
self.hist_tracker = HistTracker()
self.kcf_tracker = KCFTracker(hog=True)
self.hist_tracker.init(self.gt_rects[0], img0)
self.kcf_tracker.init(self.gt_rects[0], img0)
bg_img0 = self.get_sub_window(img0)
bg_img1 = self.get_sub_window(img1)
self.last_frame = img1
self.frame_idx = 1
self.init_figure()
return bg_img0, bg_img1
def init_figure(self):
self.vis.close()
def get_sub_window(self, img):
bg_image = self.hist_tracker.sub_window(img, self.hist_tracker.bg_box)
return bg_image
def show_tracking_result(self):
predict_img = self.hist_tracker.get_predict_img(self.gt_rects[self.frame_idx - 1])
# w, h = self.hist_tracker.w, self.hist_tracker.h
# scale_rate = 500.0 / w
self.vis.image(predict_img.transpose(2, 0, 1)[::-1, :, :], win=self.seq_name,
opts={'title': self.seq_name})
if self.save_path:
cv2.imwrite('%s/%d_predict.jpg' % (self.save_path, self.frame_idx - 1), predict_img)
def show_hist(self):
color_map, response_map = self.hist_tracker.get_hist_map()
self.vis.image(color_map.transpose(2, 0, 1), win='color_map', opts={'title': 'color_map'})
self.vis.image(response_map.transpose(2, 0, 1), win='response_map', opts={'title': 'response_map'})
if self.save_path:
cv2.imwrite('%s/%d_color_map.jpg' % (self.save_path, self.frame_idx - 1), color_map)
cv2.imwrite('%s/%d_response.jpg' % (self.save_path, self.frame_idx - 1), response_map)
def show_all(self):
self.show_tracking_result()
self.show_hist()
title = '0: %d, 1: %d' % (self.action0_count, self.action1_count)
self.vis.line(Y=np.array(self.reward_list), X=np.array(self.frame_idx_list), win='reward', opts={'title': 'reward', 'width': 600})
self.vis.line(Y=np.array(self.action_list), X=np.array(self.frame_idx_list), win='action', opts={'title': title, 'width': 600})
# self.ax_handle_list[3].set_data(self.frame_idx_list, self.reward_list)
pass
# return reward, next_frame, done
def step(self, action):
tg_box = (None, None, None, None)
action = int(action)
if action == 0:
tg_box = self.hist_tracker.run(self.last_frame, None)
self.kcf_tracker.run(self.last_frame, tg_box[:2])
self.action0_count += 1
elif action == 1:
tg_box = self.kcf_tracker.run(self.last_frame, None)
tg_box = map(int, tg_box)
self.hist_tracker.run(self.last_frame, tg_box[:2])
self.action1_count += 1
else:
print 'action is neither 0 nor 1!!!'
exit()
self.action_list.append(action)
self.action = action
gt_box = self.gt_rects[self.frame_idx]
gt_cx, gt_cy = gt_box[0] + gt_box[2] / 2.0, gt_box[1] + gt_box[3] / 2.0
tg_cx, tg_cy = tg_box[0] + tg_box[2] / 2.0, tg_box[1] + tg_box[3] / 2.0
# compute reward
dis = math.sqrt((gt_cx - tg_cx) ** 2 + (gt_cy - tg_cy) ** 2)
reward = 0.0 if dis > self.precision_threshold else 1 - (dis / self.precision_threshold)
self.reward = reward
self.frame_idx_list.append(self.frame_idx)
self.reward_list.append(self.reward)
# crop img
ret, self.last_frame = self.seq_loader.get_cap().read()
self.frame_idx += 1
bg_img = self.get_sub_window(self.last_frame) if ret else None
return reward, bg_img, not ret