def init(self,first_frame,bbox):
assert len(first_frame.shape)==3 and first_frame.shape[2]==3
if self.features=='gray':
first_frame=cv2.cvtColor(first_frame,cv2.COLOR_BGR2GRAY)
bbox = np.array(bbox).astype(np.int64)
x0, y0, w, h = tuple(bbox)
self.crop_size = (int(np.floor(w * (1 + self.padding))), int(np.floor(h * (1 + self.padding))))# for vis
self._center = (np.floor(x0 + w / 2),np.floor(y0 + h / 2))
self.w, self.h = w, h
self.window_size=(int(np.floor(w*(1+self.padding)))//self.cell_size,int(np.floor(h*(1+self.padding)))//self.cell_size)
self._window = cos_window(self.window_size)
s=np.sqrt(w*h)*self.output_sigma_factor/self.cell_size
self.yf = fft2(gaussian2d_rolled_labels(self.window_size, s))
if self.features=='gray' or self.features=='color':
first_frame = first_frame.astype(np.float32) / 255
x=self._crop(first_frame,self._center,(w,h))
x=x-np.mean(x)
elif self.features=='hog':
x=self._crop(first_frame,self._center,(w,h))
x=cv2.resize(x,(self.window_size[0]*self.cell_size,self.window_size[1]*self.cell_size))
x=extract_hog_feature(x, cell_size=self.cell_size)
elif self.features=='cn':
x = cv2.resize(first_frame, (self.window_size[0] * self.cell_size, self.window_size[1] * self.cell_size))
x=extract_cn_feature(x,self.cell_size)
else:
raise NotImplementedError
self.xf = fft2(self._get_windowed(x, self._window))
self.init_response_center = (0,0)
self.alphaf = self._training(self.xf,self.yf)
def init(self, first_frame, bbox):
assert len(first_frame.shape) == 3 and first_frame.shape[2] == 3
bbox = np.array(bbox).astype(np.int64)
x0, y0, w, h = tuple(bbox)
if w * h >= 100**2:
self.resize = True
x0, y0, w, h = x0 / 2, y0 / 2, w / 2, h / 2
first_frame = cv2.resize(first_frame, dsize=None, fx=0.5,
fy=0.5).astype(np.uint8)
self.crop_size = (int(np.floor(w * (1 + self.padding))),
int(np.floor(h * (1 + self.padding)))) # for vis
self._center = (x0 + w / 2, y0 + h / 2)
self.w, self.h = w, h
self.window_size = (int(np.floor(w * (1 + self.padding))) //
self.cell_size,
int(np.floor(h * (1 + self.padding))) //
self.cell_size)
self._window = cos_window(self.window_size)
s = np.sqrt(w * h) * self.output_sigma_factor / self.cell_size
self.yf = fft2(gaussian2d_rolled_labels(self.window_size, s))
self.search_size = np.linspace(0.985, 1.015, 7)
self.target_sz = (w, h)
#param0=[self._center[0],self._center[1],1,
# 0,1/(self.crop_size[1]/self.crop_size[0]),
# 0]
#param0=self.affparam2mat(param0)
#patch=self.warpimg(first_frame.astype(np.float32),param0,self.crop_size).astype(np.uint8)
patch = cv2.getRectSubPix(first_frame, self.crop_size, self._center)
patch = cv2.resize(patch, dsize=self.crop_size)
hc_features = self.get_features(patch, self.cell_size)
hc_features = hc_features * self._window[:, :, None]
xf = fft2(hc_features)
kf = self._kernel_correlation(xf, xf, kernel=self.kernel)
self.model_alphaf = self.yf / (kf + self.lambda_)
self.model_xf = xf
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x, y, w, h = tuple(bbox)
self.init_mask = np.ones((h, w), dtype=np.uint8)
self._center = (x + w / 2, y + h / 2)
self.w, self.h = w, h
if np.all(first_frame[:, :, 0] == first_frame[:, :, 1]):
self.use_segmentation = False
# change 400 to 300
# for larger cell_size
self.cell_size = int(min(4, max(1, w * h / 300)))
self.base_target_sz = (w, h)
template_size = (int(w + self.padding * np.sqrt(w * h)),
int(h + self.padding * np.sqrt(w * h)))
template_size = (template_size[0] + template_size[1]) // 2
self.template_size = (template_size, template_size)
self.rescale_ratio = np.sqrt(
(200**2) / (self.template_size[0] * self.template_size[1]))
self.rescale_ratio = np.clip(self.rescale_ratio, a_min=None, a_max=1)
self.rescale_template_size = (int(self.rescale_ratio *
self.template_size[0]),
int(self.rescale_ratio *
self.template_size[1]))
self.yf = fft2(
gaussian2d_rolled_labels(
(int(self.rescale_template_size[0] / self.cell_size),
int(self.rescale_template_size[1] / self.cell_size)),
self.y_sigma))
self._window = cos_window((self.yf.shape[1], self.yf.shape[0]))
self.crop_size = self.rescale_template_size
# create dummy mask (approximation for segmentation)
# size of the object in feature space
obj_sz = (int(self.rescale_ratio *
(self.base_target_sz[0] / self.cell_size)),
int(self.rescale_ratio *
(self.base_target_sz[1] / self.cell_size)))
x0 = int((self.yf.shape[1] - obj_sz[0]) / 2)
y0 = int((self.yf.shape[0] - obj_sz[1]) / 2)
x1 = x0 + obj_sz[0]
y1 = y0 + obj_sz[1]
self.target_dummy_mask = np.zeros_like(self.yf, dtype=np.uint8)
self.target_dummy_mask[y0:y1, x0:x1] = 1
self.target_dummy_area = np.sum(self.target_dummy_mask)
if self.use_segmentation:
if self.segcolor_space == 'bgr':
seg_img = first_frame
elif self.segcolor_space == 'hsv':
seg_img = cv2.cvtColor(first_frame, cv2.COLOR_BGR2HSV)
seg_img[:, :,
0] = (seg_img[:, :, 0].astype(np.float32) / 180 * 255)
seg_img = seg_img.astype(np.uint8)
else:
raise ValueError
hist_fg = Histogram(3, self.nbins)
hist_bg = Histogram(3, self.nbins)
self.extract_histograms(seg_img, bbox, hist_fg, hist_bg)
mask = self.segment_region(seg_img, self._center,
self.template_size, self.base_target_sz,
self.sc, hist_fg, hist_bg)
self.hist_bg_p_bins = hist_bg.p_bins
self.hist_fg_p_bins = hist_fg.p_bins
init_mask_padded = np.zeros_like(mask)
pm_x0 = int(np.floor(mask.shape[1] / 2 - bbox[2] / 2))
pm_y0 = int(np.floor(mask.shape[0] / 2 - bbox[3] / 2))
init_mask_padded[pm_y0:pm_y0 + bbox[3], pm_x0:pm_x0 + bbox[2]] = 1
mask = mask * init_mask_padded
mask = cv2.resize(mask, (self.yf.shape[1], self.yf.shape[0]))
if self.mask_normal(mask, self.target_dummy_area) is True:
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3),
anchor=(1, 1))
mask = cv2.dilate(mask, kernel)
else:
mask = self.target_dummy_mask
else:
mask = self.target_dummy_mask
# extract features
f = self.get_csr_features(first_frame, self._center, self.sc,
self.template_size,
self.rescale_template_size, self.cell_size)
f = f * self._window[:, :, None]
# create filters using segmentation mask
self.H = self.create_csr_filter(f, self.yf, mask)
response = np.real(ifft2(fft2(f) * np.conj(self.H)))
chann_w = np.max(response.reshape(
response.shape[0] * response.shape[1], -1),
axis=0)
self.chann_w = chann_w / np.sum(chann_w)
avg_dim = (w + h) / 2.5
self.scale_sz = ((w + avg_dim) / self.sc, (h + avg_dim) / self.sc)
self.scale_sz0 = self.scale_sz
self.cos_window_scale = cos_window(
(self.scale_sz_window[0], self.scale_sz_window[1]))
self.mag = self.cos_window_scale.shape[0] / np.log(
np.sqrt((self.cos_window_scale.shape[0]**2 +
self.cos_window_scale.shape[1]**2) / 4))
# scale lp
patchL = cv2.getRectSubPix(first_frame,
(int(np.floor(self.sc * self.scale_sz[0])),
int(np.floor(self.sc * self.scale_sz[1]))),
self._center)
patchL = cv2.resize(patchL, self.scale_sz_window)
patchLp = cv2.logPolar(patchL.astype(np.float32),
((patchL.shape[1] - 1) / 2,
(patchL.shape[0] - 1) / 2),
self.mag,
flags=cv2.INTER_LINEAR + cv2.WARP_FILL_OUTLIERS)
self.model_patchLp = extract_hog_feature(patchLp, cell_size=4)
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x, y, w, h = tuple(bbox)
self._center = (x + w / 2, y + h / 2)
self.w, self.h = w, h
self.feature_ratio = self.cell_size
self.search_area=(self.w/self.feature_ratio*self.search_area_scale)*\
(self.h/self.feature_ratio*self.search_area_scale)
if self.search_area < self.cell_selection_thresh * self.filter_max_area:
self.cell_size=int(min(self.feature_ratio,max(1,int(np.ceil(np.sqrt(
self.w*self.search_area_scale/(self.cell_selection_thresh*self.filter_max_area)*\
self.h*self.search_area_scale/(self.cell_selection_thresh*self.filter_max_area)
))))))
self.feature_ratio = self.cell_size
self.search_area = (self.w / self.feature_ratio * self.search_area_scale) * \
(self.h / self.feature_ratio * self.search_area_scale)
if self.search_area > self.filter_max_area:
self.current_scale_factor = np.sqrt(self.search_area /
self.filter_max_area)
else:
self.current_scale_factor = 1.
self.base_target_sz = (self.w / self.current_scale_factor,
self.h / self.current_scale_factor)
self.target_sz = self.base_target_sz
if self.search_area_shape == 'proportional':
self.crop_size = (int(self.base_target_sz[0] *
self.search_area_scale),
int(self.base_target_sz[1] *
self.search_area_scale))
elif self.search_area_shape == 'square':
w = int(
np.sqrt(self.base_target_sz[0] * self.base_target_sz[1]) *
self.search_area_scale)
self.crop_size = (w, w)
elif self.search_area_shape == 'fix_padding':
tmp=int(np.sqrt(self.base_target_sz[0]*self.search_area_scale+(self.base_target_sz[1]-self.base_target_sz[0])/4))+\
(self.base_target_sz[0]+self.base_target_sz[1])/2
self.crop_size = (self.base_target_sz[0] + tmp,
self.base_target_sz[1] + tmp)
else:
raise ValueError
self.crop_size = (int(
round(self.crop_size[0] / self.feature_ratio) *
self.feature_ratio),
int(
round(self.crop_size[1] / self.feature_ratio) *
self.feature_ratio))
self.feature_map_sz = (self.crop_size[0] // self.feature_ratio,
self.crop_size[1] // self.feature_ratio)
output_sigma = np.sqrt(
np.floor(self.base_target_sz[0] / self.feature_ratio) *
np.floor(self.base_target_sz[1] /
self.feature_ratio)) * self.output_sigma_factor
y = gaussian2d_rolled_labels(self.feature_map_sz, output_sigma)
self.yf = fft2(y)
if self.interpolate_response == 1:
self.interp_sz = (self.feature_map_sz[0] * self.feature_ratio,
self.feature_map_sz[1] * self.feature_ratio)
else:
self.interp_sz = (self.feature_map_sz[0], self.feature_map_sz[1])
self._window = cos_window(self.feature_map_sz)
if self.number_of_scales > 0:
scale_exp = np.arange(
-int(np.floor((self.number_of_scales - 1) / 2)),
int(np.ceil((self.number_of_scales - 1) / 2)) + 1)
self.scale_factors = self.scale_step**scale_exp
self.min_scale_factor = self.scale_step**(np.ceil(
np.log(max(5 / self.crop_size[0], 5 / self.crop_size[1])) /
np.log(self.scale_step)))
self.max_scale_factor = self.scale_step**(np.floor(
np.log(
min(first_frame.shape[0] / self.base_target_sz[1],
first_frame.shape[1] / self.base_target_sz[0])) /
np.log(self.scale_step)))
if self.interpolate_response >= 3:
self.ky = np.roll(
np.arange(-int(np.floor((self.feature_map_sz[1] - 1) / 2)),
int(np.ceil((self.feature_map_sz[1] - 1) / 2 + 1))),
-int(np.floor((self.feature_map_sz[1] - 1) / 2)))
self.kx = np.roll(
np.arange(-int(np.floor((self.feature_map_sz[0] - 1) / 2)),
int(np.ceil((self.feature_map_sz[0] - 1) / 2 + 1))),
-int(np.floor((self.feature_map_sz[0] - 1) / 2))).T
self.small_filter_sz = (int(
np.floor(self.base_target_sz[0] / self.feature_ratio)),
int(
np.floor(self.base_target_sz[1] /
self.feature_ratio)))
self.scale_estimator = LPScaleEstimator(self.target_sz,
config=self.scale_config)
self.scale_estimator.init(first_frame, self._center,
self.base_target_sz,
self.current_scale_factor)
pixels = self.get_sub_window(
first_frame,
self._center,
model_sz=self.crop_size,
scaled_sz=(int(
np.round(self.crop_size[0] * self.current_scale_factor)),
int(
np.round(self.crop_size[1] *
self.current_scale_factor))))
feature = self.extract_hc_feture(pixels, cell_size=self.feature_ratio)
self.model_xf = fft2(self._window[:, :, None] * feature)
self.g_f = self.ADMM(self.model_xf)
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x0, y0, w, h = tuple(bbox)
self.target_sz = (w, h)
self._center = (int(x0 + w / 2), int(y0 + h / 2))
self.base_target_sz = (w / self.sc, h / self.sc)
self.win_sz = (int(
np.floor(self.base_target_sz[0] * (1 + self.padding))),
int(
np.floor(self.base_target_sz[1] *
(1 + self.padding))))
output_sigma = np.sqrt(
self.base_target_sz[0] *
self.base_target_sz[1]) * self.output_sigma_factor / self.cell_size
use_sz = (int(np.floor(self.win_sz[0] / self.cell_size)),
int(np.floor(self.win_sz[1] / self.cell_size)))
self.yf = fft2(0.5 *
gaussian2d_rolled_labels(use_sz, sigma=output_sigma))
self.interp_sz = (use_sz[0] * self.cell_size,
use_sz[1] * self.cell_size)
self._window = cos_window(use_sz)
avg_dim = (w + h) / 2.5
self.scale_sz = ((w + avg_dim) / self.sc, (h + avg_dim) / self.sc)
self.scale_sz0 = self.scale_sz
self.cos_window_scale = cos_window(
(self.scale_sz_window[0], self.scale_sz_window[1]))
self.mag = self.cos_window_scale.shape[0] / np.log(
np.sqrt((self.cos_window_scale.shape[0]**2 +
self.cos_window_scale.shape[1]**2) / 4))
# scale lp
patchL = cv2.getRectSubPix(first_frame,
(int(np.floor(self.sc * self.scale_sz[0])),
int(np.floor(self.sc * self.scale_sz[1]))),
self._center)
patchL = cv2.resize(patchL, self.scale_sz_window)
patchLp = cv2.logPolar(patchL.astype(np.float32),
((patchL.shape[1] - 1) / 2,
(patchL.shape[0] - 1) / 2),
self.mag,
flags=cv2.INTER_LINEAR + cv2.WARP_FILL_OUTLIERS)
self.model_patchLp = extract_hog_feature(patchLp, cell_size=4)
self.cn_sigma = self.cn_sigma_color
self.hog_sigma = self.hog_sigma_color
self.lr_hog = self.lr_hog_color
self.lr_cn = self.lr_cn_color
self.modnum = self.gap
self.is_gray = False
patch = cv2.getRectSubPix(first_frame, self.win_sz,
self._center).astype(np.uint8)
self.z_hog, self.z_cn = self.get_features(patch,
cell_size=self.cell_size)
data_matrix_cn = self.z_cn.reshape((-1, self.z_cn.shape[2]))
pca_basis_cn, _, _ = np.linalg.svd(
data_matrix_cn.T.dot(data_matrix_cn))
self.projection_matrix_cn = pca_basis_cn[:, :self.
num_compressed_dim_cn]
data_matrix_hog = self.z_hog.reshape((-1, self.z_hog.shape[2]))
pca_basis_hog, _, _ = np.linalg.svd(
data_matrix_hog.T.dot(data_matrix_hog))
self.projection_matrix_hog = pca_basis_hog[:, :self.
num_compressed_dim_hog]
self.z_cn2, self.z_hog2 = self.feature_projection(
self.z_cn, self.z_hog, self.projection_matrix_cn,
self.projection_matrix_hog, self._window)
self.frame_index = 1
self.d = self.train_model()
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x0, y0, w, h = tuple(bbox)
self.target_sz = (w, h)
self._center = (int(x0 + w / 2), int(y0 + h / 2))
if w * h > self.translation_model_max_area:
self.sc = np.sqrt(w * h / self.translation_model_max_area)
else:
self.sc = 1.
self.base_target_sz = (w / self.sc, h / self.sc)
self.win_sz = (int(
np.floor(self.base_target_sz[0] * (1 + self.padding))),
int(
np.floor(self.base_target_sz[1] *
(1 + self.padding))))
output_sigma = np.sqrt(
self.base_target_sz[0] *
self.base_target_sz[1]) * self.output_sigma_factor / self.cell_size
use_sz = (int(np.floor(self.win_sz[0] / self.cell_size)),
int(np.floor(self.win_sz[1] / self.cell_size)))
self.yf = fft2(0.5 *
gaussian2d_rolled_labels(use_sz, sigma=output_sigma))
self.interp_sz = (use_sz[0] * self.cell_size,
use_sz[1] * self.cell_size)
self._window = cos_window(use_sz)
if self.num_of_scales > 0:
self.scale_sigma = self.num_of_interp_scales * self.scale_sigma_factor
scale_exp = np.arange(
-int(np.floor((self.num_of_scales - 1) / 2)),
int(np.ceil((self.num_of_scales - 1) / 2)) +
1) * self.num_of_interp_scales / self.num_of_scales
scale_exp_shift = np.roll(
scale_exp, -int(np.floor((self.num_of_scales - 1) / 2)))
interp_scale_exp = np.arange(
-int(np.floor((self.num_of_interp_scales - 1) / 2)),
int(np.ceil((self.num_of_interp_scales - 1) / 2)) + 1)
interp_scale_exp_shift = np.roll(
interp_scale_exp,
-int(np.floor((self.num_of_interp_scales - 1) / 2)))
self.scale_size_factors = self.scale_step**scale_exp
self.interp_scale_factors = self.scale_step**interp_scale_exp_shift
ys = np.exp(-0.5 * (scale_exp_shift**2) / (self.scale_sigma**2))
self.ysf = np.fft.fft(ys)
self.scale_window = np.hanning(len(self.ysf)).T
if self.scale_model_factor**2 * (
self.base_target_sz[0] *
self.base_target_sz[1]) > self.scale_model_max_area:
self.scale_model_factor = np.sqrt(
self.scale_model_max_area /
(self.base_target_sz[0] * self.base_target_sz[1]))
self.scale_model_sz = (int(
np.floor(self.base_target_sz[0] * self.scale_model_factor)),
int(
np.floor(self.base_target_sz[1] *
self.scale_model_factor)))
self.min_scale_factor = self.scale_step**(int(
np.ceil(
np.log(max(5 / self.win_sz[0], 5 / self.win_sz[1])) /
np.log(self.scale_step))))
self.max_scale_factor = self.scale_step**(int(
np.floor((np.log(
min(first_frame.shape[1] / self.base_target_sz[0],
first_frame.shape[0] / self.base_target_sz[1])) /
np.log(self.scale_step)))))
self.s_num_compressed_dim = len(self.scale_size_factors)
xs = self.get_scale_subwindow(first_frame, self._center)
bigY = xs
bigY_den = xs
self.s_num = xs
scale_basis, _ = np.linalg.qr(bigY)
scale_basis_den, _ = np.linalg.qr(bigY_den)
self.scale_basis = scale_basis.T
self.scale_basis_den = scale_basis_den.T
sf_proj = np.fft.fft(self.scale_window *
self.scale_basis.dot(self.s_num),
axis=1)
self.sf_num = self.ysf * np.conj(sf_proj)
xs = self.scale_basis_den.dot(xs)
xsf = np.fft.fft(xs, axis=1)
new_sf_den = np.sum(xsf * np.conj(xsf), axis=0)
self.sf_den = new_sf_den
self.cn_sigma = self.cn_sigma_color
self.hog_sigma = self.hog_sigma_color
self.lr_hog = self.lr_hog_color
self.lr_cn = self.lr_cn_color
self.modnum = self.gap
self.is_gray = False
patch = cv2.getRectSubPix(first_frame, self.win_sz,
self._center).astype(np.uint8)
self.z_hog, self.z_cn = self.get_features(patch,
cell_size=self.cell_size)
data_matrix_cn = self.z_cn.reshape((-1, self.z_cn.shape[2]))
pca_basis_cn, _, _ = np.linalg.svd(
data_matrix_cn.T.dot(data_matrix_cn))
self.projection_matrix_cn = pca_basis_cn[:, :self.
num_compressed_dim_cn]
data_matrix_hog = self.z_hog.reshape((-1, self.z_hog.shape[2]))
pca_basis_hog, _, _ = np.linalg.svd(
data_matrix_hog.T.dot(data_matrix_hog))
self.projection_matrix_hog = pca_basis_hog[:, :self.
num_compressed_dim_hog]
self.z_cn2, self.z_hog2 = self.feature_projection(
self.z_cn, self.z_hog, self.projection_matrix_cn,
self.projection_matrix_hog, self._window)
self.frame_index = 1
self.d = self.train_model()
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x, y, w, h = tuple(bbox)
self.init_mask = np.ones((h, w), dtype=np.uint8)
self._center = (x + w / 2, y + h / 2)
self.w, self.h = w, h
if np.all(first_frame[:, :, 0] == first_frame[:, :, 1]):
self.use_segmentation = False
# change 400 to 300
# for larger cell_size
self.cell_size = int(min(4, max(1, w * h / 300)))
self.base_target_sz = (w, h)
self.target_sz = self.base_target_sz
template_size = (int(w + self.padding * np.sqrt(w * h)),
int(h + self.padding * np.sqrt(w * h)))
template_size = (template_size[0] + template_size[1]) // 2
self.template_size = (template_size, template_size)
self.rescale_ratio = np.sqrt(
(self.params['template_size']**2) /
(self.template_size[0] * self.template_size[1]))
self.rescale_ratio = np.clip(self.rescale_ratio, a_min=None, a_max=1)
self.rescale_template_size = (int(self.rescale_ratio *
self.template_size[0]),
int(self.rescale_ratio *
self.template_size[1]))
self.yf = fft2(
gaussian2d_rolled_labels(
(int(self.rescale_template_size[0] / self.cell_size),
int(self.rescale_template_size[1] / self.cell_size)),
self.y_sigma))
self._window = cos_window((self.yf.shape[1], self.yf.shape[0]))
self.crop_size = self.rescale_template_size
self.current_scale_factor = 1.
if self.scale_type == 'normal':
self.scale_estimator = DSSTScaleEstimator(self.target_sz,
config=self.scale_config)
self.scale_estimator.init(first_frame, self._center,
self.base_target_sz,
self.current_scale_factor)
self._num_scales = self.scale_estimator.num_scales
self._scale_step = self.scale_estimator.scale_step
self._min_scale_factor = self._scale_step**np.ceil(
np.log(
np.max(5 / np.array(
([self.crop_size[0], self.crop_size[1]])))) /
np.log(self._scale_step))
self._max_scale_factor = self._scale_step**np.floor(
np.log(
np.min(first_frame.shape[:2] / np.array(
[self.base_target_sz[1], self.base_target_sz[0]]))) /
np.log(self._scale_step))
elif self.scale_type == 'LP':
self.scale_estimator = LPScaleEstimator(self.target_sz,
config=self.scale_config)
self.scale_estimator.init(first_frame, self._center,
self.base_target_sz,
self.current_scale_factor)
# create dummy mask (approximation for segmentation)
# size of the object in feature space
obj_sz = (int(self.rescale_ratio *
(self.base_target_sz[0] / self.cell_size)),
int(self.rescale_ratio *
(self.base_target_sz[1] / self.cell_size)))
x0 = int((self.yf.shape[1] - obj_sz[0]) / 2)
y0 = int((self.yf.shape[0] - obj_sz[1]) / 2)
x1 = x0 + obj_sz[0]
y1 = y0 + obj_sz[1]
self.target_dummy_mask = np.zeros_like(self.yf, dtype=np.uint8)
self.target_dummy_mask[y0:y1, x0:x1] = 1
self.target_dummy_area = np.sum(self.target_dummy_mask)
if self.use_segmentation:
if self.segcolor_space == 'bgr':
seg_img = first_frame
elif self.segcolor_space == 'hsv':
seg_img = cv2.cvtColor(first_frame, cv2.COLOR_BGR2HSV)
seg_img[:, :,
0] = (seg_img[:, :, 0].astype(np.float32) / 180 * 255)
seg_img = seg_img.astype(np.uint8)
else:
raise ValueError
hist_fg = Histogram(3, self.nbins)
hist_bg = Histogram(3, self.nbins)
self.extract_histograms(seg_img, bbox, hist_fg, hist_bg)
mask = self.segment_region(seg_img, self._center,
self.template_size, self.base_target_sz,
self.current_scale_factor, hist_fg,
hist_bg)
self.hist_bg_p_bins = hist_bg.p_bins
self.hist_fg_p_bins = hist_fg.p_bins
init_mask_padded = np.zeros_like(mask)
pm_x0 = int(np.floor(mask.shape[1] / 2 - bbox[2] / 2))
pm_y0 = int(np.floor(mask.shape[0] / 2 - bbox[3] / 2))
init_mask_padded[pm_y0:pm_y0 + bbox[3], pm_x0:pm_x0 + bbox[2]] = 1
mask = mask * init_mask_padded
mask = cv2.resize(mask, (self.yf.shape[1], self.yf.shape[0]))
if self.mask_normal(mask, self.target_dummy_area) is True:
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3),
anchor=(1, 1))
mask = cv2.dilate(mask, kernel)
else:
mask = self.target_dummy_mask
else:
mask = self.target_dummy_mask
# extract features
f = get_csr_features(first_frame, self._center,
self.current_scale_factor, self.template_size,
self.rescale_template_size, self.cell_size)
f = f * self._window[:, :, None]
# create filters using segmentation mask
self.H = self.create_csr_filter(f, self.yf, mask)
response = np.real(ifft2(fft2(f) * np.conj(self.H)))
chann_w = np.max(response.reshape(
response.shape[0] * response.shape[1], -1),
axis=0)
self.chann_w = chann_w / np.sum(chann_w)
# New: irrelevant channels!
self.irrelevant_channels = []
top_channels = self.params['top_channels']
if top_channels:
for chan_i, _ in sorted(enumerate(chann_w),
reverse=True,
key=lambda x: x[1])[top_channels:]:
self.irrelevant_channels.append(chan_i)
f = np.delete(f, self.irrelevant_channels, 2)
# create filters using segmentation mask
self.H = self.create_csr_filter(f, self.yf, mask)
response = np.real(ifft2(fft2(f) * np.conj(self.H)))
chann_w = np.max(response.reshape(
response.shape[0] * response.shape[1], -1),
axis=0)
self.chann_w = chann_w / np.sum(chann_w)
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x0, y0, w, h = tuple(bbox)
self._center = (int(x0 + w / 2), int(y0 + h / 2))
self.target_sz = (w, h)
search_area = self.target_sz[
0] * self.search_area_scale * self.target_sz[
1] * self.search_area_scale
self.sc = np.clip(
1,
a_min=np.sqrt(search_area / self.max_image_sample_size),
a_max=np.sqrt(search_area / self.min_image_sample_size))
self.base_target_sz = (self.target_sz[0] / self.sc,
self.target_sz[1] / self.sc)
if self.search_area_shape == 'proportional':
self.crop_size = (int(self.base_target_sz[0] *
self.search_area_scale),
int(self.base_target_sz[1] *
self.search_area_scale))
elif self.search_area_shape == 'square':
w = int(
np.sqrt(self.base_target_sz[0] * self.base_target_sz[1]) *
self.search_area_scale)
self.crop_size = (w, w)
elif self.search_area_shape == 'fix_padding':
tmp=int(np.sqrt(self.base_target_sz[0]*self.search_area_scale+(self.base_target_sz[1]-self.base_target_sz[0])/4))+\
(self.base_target_sz[0]+self.base_target_sz[1])/2
self.crop_size = (self.base_target_sz[0] + tmp,
self.base_target_sz[1] + tmp)
else:
raise ValueError
output_sigma = np.sqrt(np.floor(self.base_target_sz[0]/self.cell_size)*np.floor(self.base_target_sz[1]*self.cell_size))*\
self.output_sigma_factor
self.crop_size = (int(
round(self.crop_size[0] / self.cell_size) * self.cell_size),
int(
round(self.crop_size[1] / self.cell_size) *
self.cell_size))
self.feature_map_sz = (self.crop_size[0] // self.cell_size,
self.crop_size[1] // self.cell_size)
y = gaussian2d_rolled_labels(self.feature_map_sz, output_sigma)
self.cosine_window = (cos_window((y.shape[1], y.shape[0])))
self.yf = fft2(y)
reg_scale = (int(
np.floor(self.base_target_sz[0] / self.feature_downsample_ratio)),
int(
np.floor(self.base_target_sz[1] /
self.feature_downsample_ratio)))
use_sz = self.feature_map_sz
#self.reg_window=self.create_reg_window(reg_scale,use_sz,self.p,self.reg_window_max,
# self.reg_window_min,self.alpha,self.beta)
self.reg_window = self.create_reg_window_const(reg_scale, use_sz,
self.reg_window_max,
self.reg_window_min)
self.ky = np.roll(
np.arange(-int(np.floor((self.feature_map_sz[1] - 1) / 2)),
int(np.ceil((self.feature_map_sz[1] - 1) / 2 + 1))),
-int(np.floor((self.feature_map_sz[1] - 1) / 2)))
self.kx = np.roll(
np.arange(-int(np.floor((self.feature_map_sz[0] - 1) / 2)),
int(np.ceil((self.feature_map_sz[0] - 1) / 2 + 1))),
-int(np.floor((self.feature_map_sz[0] - 1) / 2)))
# scale
scale_exp = np.arange(
-int(np.floor((self.number_of_scales - 1) / 2)),
int(np.ceil((self.number_of_scales - 1) / 2) + 1))
self.scale_factors = self.scale_step**scale_exp
if self.number_of_scales > 0:
self._min_scale_factor = self.scale_step**np.ceil(
np.log(
np.max(5 / np.array(
([self.crop_size[0], self.crop_size[1]])))) /
np.log(self.scale_step))
self._max_scale_factor = self.scale_step**np.floor(
np.log(
np.min(first_frame.shape[:2] / np.array(
[self.base_target_sz[1], self.base_target_sz[0]]))) /
np.log(self.scale_step))
#print(self._min_scale_factor)
#print(self._max_scale_factor)
if self.scale_type == 'normal':
self.scale_estimator = DSSTScaleEstimator(self.target_sz,
config=self.scale_config)
self.scale_estimator.init(first_frame, self._center,
self.base_target_sz, self.sc)
self._num_scales = self.scale_estimator.num_scales
self._scale_step = self.scale_estimator.scale_step
self._min_scale_factor = self._scale_step**np.ceil(
np.log(
np.max(5 / np.array(
([self.crop_size[0], self.crop_size[1]])))) /
np.log(self._scale_step))
self._max_scale_factor = self._scale_step**np.floor(
np.log(
np.min(first_frame.shape[:2] / np.array(
[self.base_target_sz[1], self.base_target_sz[0]]))) /
np.log(self._scale_step))
elif self.scale_type == 'LP':
self.scale_estimator = LPScaleEstimator(self.target_sz,
config=self.scale_config)
self.scale_estimator.init(first_frame, self._center,
self.base_target_sz, self.sc)
patch = self.get_sub_window(
first_frame,
self._center,
model_sz=self.crop_size,
scaled_sz=(int(np.round(self.crop_size[0] * self.sc)),
int(np.round(self.crop_size[1] * self.sc))))
xl_hc = self.extrac_hc_feature(patch, self.cell_size)
xlf_hc = fft2(xl_hc * self.cosine_window[:, :, None])
f_pre_f_hc = np.zeros_like(xlf_hc)
mu_hc = 0
self.f_pre_f_hc = self.ADMM(xlf_hc, f_pre_f_hc, mu_hc)
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x, y, w, h = tuple(bbox)
self._center = (x + w / 2, y + h / 2)
self.w, self.h = w, h
self.feature_ratio = self.cell_size
# 初始边框做为一个统计块，每个统计块包含4个细胞单元；查找区域在每个细胞单元*5 的范围内。 (w/4*5)*(h/4*5)
self.search_area=(self.w/self.feature_ratio*self.search_area_scale)*\
(self.h/self.feature_ratio*self.search_area_scale)
if self.search_area < self.cell_selection_thresh * self.filter_max_area: # 搜索区域< 0.75*50的正方形
# 新的统计块包含细胞单元个数 min(4, max(1, w*5/(0.75**2*50**2)*h*5/(0.75**2*50**2)))
self.cell_size = int(
min(
self.feature_ratio,
max(
1,
int(
np.ceil(
np.sqrt(self.w * self.search_area_scale /
(self.cell_selection_thresh *
self.filter_max_area) * self.h *
self.search_area_scale /
(self.cell_selection_thresh *
self.filter_max_area)))))))
self.feature_ratio = self.cell_size
self.search_area = (self.w / self.feature_ratio * self.search_area_scale) * \
(self.h / self.feature_ratio * self.search_area_scale)
if self.search_area > self.filter_max_area:
self.current_scale_factor = np.sqrt(self.search_area /
self.filter_max_area)
else:
self.current_scale_factor = 1.
# 当搜索区域> 最大区域时，cop 区域会变化，如下
self.base_target_sz = (self.w / self.current_scale_factor,
self.h / self.current_scale_factor)
self.target_sz = self.base_target_sz
if self.search_area_shape == 'proportional':
self.crop_size = (
int(self.base_target_sz[0] * self.search_area_scale),
int(self.base_target_sz[1] * self.search_area_scale)
) # crop_size:
elif self.search_area_shape == 'square': # crop 使用正方形时
w = int(
np.sqrt(self.base_target_sz[0] * self.base_target_sz[1]) *
self.search_area_scale)
self.crop_size = (w, w)
elif self.search_area_shape == 'fix_padding':
# tmp = sqrt(w*5+(h-w)/4)+(w+h)/2
tmp=int(np.sqrt(self.base_target_sz[0]*self.search_area_scale+(self.base_target_sz[1]-self.base_target_sz[0])/4))+\
(self.base_target_sz[0]+self.base_target_sz[1])/2
self.crop_size = (self.base_target_sz[0] + tmp,
self.base_target_sz[1] + tmp)
else:
raise ValueError
self.crop_size = (int(
round(self.crop_size[0] / self.feature_ratio) *
self.feature_ratio),
int(
round(self.crop_size[1] / self.feature_ratio) *
self.feature_ratio))
# frature_map = (crop_w//cell_num, crop_h//cell_num)
self.feature_map_sz = (self.crop_size[0] // self.feature_ratio,
self.crop_size[1] // self.feature_ratio)
# output_sigma = crop 中细胞元的大小/16
output_sigma = np.sqrt(
np.floor(self.base_target_sz[0] / self.feature_ratio) *
np.floor(self.base_target_sz[1] /
self.feature_ratio)) * self.output_sigma_factor # ????`
# 高斯窗，避免图像转化为傅里叶信号时造成频谱泄露现象；也会改变频谱幅值
y = gaussian2d_rolled_labels(self.feature_map_sz, output_sigma)
# 傅里叶变换
self.yf = fft2(y)
if self.interpolate_response == 1:
self.interp_sz = (self.feature_map_sz[0] * self.feature_ratio,
self.feature_map_sz[1] * self.feature_ratio)
else:
self.interp_sz = (self.feature_map_sz[0], self.feature_map_sz[1])
self._window = cos_window(self.feature_map_sz)
if self.number_of_scales > 0:
scale_exp = np.arange(
-int(np.floor((self.number_of_scales - 1) / 2)),
int(np.ceil((self.number_of_scales - 1) / 2)) + 1)
self.scale_factors = self.scale_step**scale_exp
self.min_scale_factor = self.scale_step**(np.ceil(
np.log(max(5 / self.crop_size[0], 5 / self.crop_size[1])) /
np.log(self.scale_step)))
self.max_scale_factor = self.scale_step**(np.floor(
np.log(
min(first_frame.shape[0] / self.base_target_sz[1],
first_frame.shape[1] / self.base_target_sz[0])) /
np.log(self.scale_step)))
if self.interpolate_response >= 3:
self.ky = np.roll(
np.arange(-int(np.floor((self.feature_map_sz[1] - 1) / 2)),
int(np.ceil((self.feature_map_sz[1] - 1) / 2 + 1))),
-int(np.floor((self.feature_map_sz[1] - 1) / 2)))
self.kx = np.roll(
np.arange(-int(np.floor((self.feature_map_sz[0] - 1) / 2)),
int(np.ceil((self.feature_map_sz[0] - 1) / 2 + 1))),
-int(np.floor((self.feature_map_sz[0] - 1) / 2))).T
self.small_filter_sz = (int(
np.floor(self.base_target_sz[0] / self.feature_ratio)),
int(
np.floor(self.base_target_sz[1] /
self.feature_ratio)))
self.scale_estimator = LPScaleEstimator(self.target_sz,
config=self.scale_config)
self.scale_estimator.init(first_frame, self._center,
self.base_target_sz,
self.current_scale_factor)
pixels = self.get_sub_window(
first_frame,
self._center,
model_sz=self.crop_size,
scaled_sz=(int(
np.round(self.crop_size[0] * self.current_scale_factor)),
int(
np.round(self.crop_size[1] *
self.current_scale_factor))))
feature = self.extract_hc_feture(pixels, cell_size=self.feature_ratio)
self.model_xf = fft2(self._window[:, :, None] * feature)
self.g_f = self.ADMM(self.model_xf)
def init(self, first_frame, bbox):
bbox = np.array(bbox).astype(np.int64)
x, y, w, h = tuple(bbox)
self.init_mask = np.ones((h, w), dtype=np.uint8)
self._center = (x + w / 2, y + h / 2)
self.w, self.h = w, h
if np.all(first_frame[:, :, 0] == first_frame[:, :, 1]):
self.use_segmentation = False
# change 400 to 300
# for larger cell_size
self.cell_size = int(min(4, max(1, w * h / 300)))
self.base_target_sz = (w, h)
template_size = (int(w + self.padding * np.sqrt(w * h)),
int(h + self.padding * np.sqrt(w * h)))
template_size = (template_size[0] + template_size[1]) // 2
self.template_size = (template_size, template_size)
self.rescale_ratio = np.sqrt(
(200**2) / (self.template_size[0] * self.template_size[1]))
self.rescale_ratio = np.clip(self.rescale_ratio, a_min=None, a_max=1)
self.rescale_template_size = (int(self.rescale_ratio *
self.template_size[0]),
int(self.rescale_ratio *
self.template_size[1]))
self.yf = fft2(
gaussian2d_rolled_labels(
(int(self.rescale_template_size[0] / self.cell_size),
int(self.rescale_template_size[1] / self.cell_size)),
self.y_sigma))
self._window = cos_window((self.yf.shape[1], self.yf.shape[0]))
self.crop_size = self.rescale_template_size
# the same as DSST
self.scale_sigma = np.sqrt(self.n_scales) * self.scale_sigma_factor
ss = np.arange(1, self.n_scales + 1) - np.ceil(self.n_scales / 2)
ys = np.exp(-0.5 * (ss**2) / (self.scale_sigma**2))
self.ysf = np.fft.fft(ys)
if self.n_scales % 2 == 0:
scale_window = np.hanning(self.n_scales + 1)
self.scale_window = scale_window[1:]
else:
self.scale_window = np.hanning(self.n_scales)
ss = np.arange(1, self.n_scales + 1)
self.scale_factors = self.scale_step**(np.ceil(self.n_scales / 2) - ss)
self.scale_model_factor = 1.
if (self.scale_model_factor**2*self.template_size[0]*self.template_size[1]) >\
self.scale_model_max_area:
self.scale_model_factor = np.sqrt(
self.scale_model_max_area /
(self.template_size[0] * self.template_size[1]))
self.scale_model_sz = (int(
np.floor(self.template_size[0] * self.scale_model_factor)),
int(
np.floor(self.template_size[1] *
self.scale_model_factor)))
self.current_scale_factor = 1.
self.min_scale_factor = self.scale_step**(int(
np.ceil(
np.log(
max(5 / self.template_size[0], 5 / self.template_size[1]))
/ np.log(self.scale_step))))
self.max_scale_factor = self.scale_step**(int(
np.floor((np.log(
min(first_frame.shape[1] / self.base_target_sz[0],
first_frame.shape[0] / self.base_target_sz[1])) /
np.log(self.scale_step)))))
# create dummy mask (approximation for segmentation)
# size of the object in feature space
obj_sz = (int(self.rescale_ratio *
(self.base_target_sz[0] / self.cell_size)),
int(self.rescale_ratio *
(self.base_target_sz[1] / self.cell_size)))
x0 = int((self.yf.shape[1] - obj_sz[0]) / 2)
y0 = int((self.yf.shape[0] - obj_sz[1]) / 2)
x1 = x0 + obj_sz[0]
y1 = y0 + obj_sz[1]
self.target_dummy_mask = np.zeros_like(self.yf, dtype=np.uint8)
self.target_dummy_mask[y0:y1, x0:x1] = 1
self.target_dummy_area = np.sum(self.target_dummy_mask)
if self.use_segmentation:
if self.segcolor_space == 'bgr':
seg_img = first_frame
elif self.segcolor_space == 'hsv':
seg_img = cv2.cvtColor(first_frame, cv2.COLOR_BGR2HSV)
seg_img[:, :,
0] = (seg_img[:, :, 0].astype(np.float32) / 180 * 255)
seg_img = seg_img.astype(np.uint8)
else:
raise ValueError
hist_fg = Histogram(3, self.nbins)
hist_bg = Histogram(3, self.nbins)
self.extract_histograms(seg_img, bbox, hist_fg, hist_bg)
mask = self.segment_region(seg_img, self._center,
self.template_size, self.base_target_sz,
self.current_scale_factor, hist_fg,
hist_bg)
self.hist_bg_p_bins = hist_bg.p_bins
self.hist_fg_p_bins = hist_fg.p_bins
init_mask_padded = np.zeros_like(mask)
pm_x0 = int(np.floor(mask.shape[1] / 2 - bbox[2] / 2))
pm_y0 = int(np.floor(mask.shape[0] / 2 - bbox[3] / 2))
init_mask_padded[pm_y0:pm_y0 + bbox[3], pm_x0:pm_x0 + bbox[2]] = 1
mask = mask * init_mask_padded
mask = cv2.resize(mask, (self.yf.shape[1], self.yf.shape[0]))
if self.mask_normal(mask, self.target_dummy_area) is True:
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3),
anchor=(1, 1))
mask = cv2.dilate(mask, kernel)
else:
mask = self.target_dummy_mask
else:
mask = self.target_dummy_mask
# extract features
f = self.get_csr_features(first_frame, self._center,
self.current_scale_factor,
self.template_size,
self.rescale_template_size, self.cell_size)
f = f * self._window[:, :, None]
# create filters using segmentation mask
self.H = self.create_csr_filter(f, self.yf, mask)
response = np.real(ifft2(fft2(f) * np.conj(self.H)))
chann_w = np.max(response.reshape(
response.shape[0] * response.shape[1], -1),
axis=0)
self.chann_w = chann_w / np.sum(chann_w)
# mask a scale search model as well
xs = self.get_scale_sample(
first_frame, self._center, self.base_target_sz,
self.current_scale_factor * self.scale_factors, self.scale_window,
self.scale_model_sz)
xsf = np.fft.fft(xs, axis=1)
self.sf_num = self.ysf * np.conj(xsf)
self.sf_den = np.sum(xsf * np.conj(xsf), axis=0)