def track(self, image):
left = max(round(self.position[0] - float(self.window) / 2), 0)
top = max(round(self.position[1] - float(self.window) / 2), 0)
right = min(round(self.position[0] + float(self.window) / 2),
image.shape[1] - 1)
bottom = min(round(self.position[1] + float(self.window) / 2),
image.shape[0] - 1)
if right - left < self.template.shape[
1] or bottom - top < self.template.shape[0]:
return [
self.position[0] + self.size[0] / 2,
self.position[1] + self.size[1] / 2, self.size[0], self.size[1]
], 0
new_x, new_y, iters = mean_shift(image, self.position, self.size,
self.q, self.kernel,
self.parameters.histogram_bins,
self.parameters.epsilon)
# MODEL UPDATE
self.template, _ = get_patch(image, (new_x, new_y), self.size)
self.q = (
1 - self.parameters.update_alpha
) * self.q + self.parameters.update_alpha * normalize_histogram(
extract_histogram(self.template,
self.parameters.histogram_bins,
weights=self.kernel))
self.position = (new_x, new_y)
return [new_x, new_y, self.size[0], self.size[1]], iters
def mean_shift(responses, position, kernel_size, epsilon):
kernel_x, kernel_y = create_kernel(kernel_size)
patch_size = (kernel_size[0], kernel_size[1])
converged = False
positions = list()
iters = 0
while not converged:
w, inliers = get_patch(responses, position, patch_size)
x_change = np.divide(np.sum(np.multiply(kernel_x, w)), np.sum(w))
y_change = np.divide(np.sum(np.multiply(kernel_y, w)), np.sum(w))
# print([floor(n) for n in position], x_change, y_change)
if abs(x_change) < epsilon and abs(y_change) < epsilon:
converged = True
position = position[0] + x_change, position[1] + y_change
if (floor(position[0]), floor(position[1])) not in positions:
positions.append((floor(position[0]), floor(position[1])))
iters += 1
if iters % 100 == 0:
print(iters)
return int(floor(position[0])), int(floor(position[1])), iters, positions
def mean_shift(image, position, size, q, kernel, nbins, epsilon):
k_size = [size[1], size[0]]
if k_size[0] % 2 == 0:
k_size[0] = k_size[0] - 1
if k_size[1] % 2 == 0:
k_size[1] = k_size[1] - 1
kernel_x, kernel_y = create_kernel(k_size)
patch_size = (k_size[1], k_size[0])
converged = False
iters = 0
while not converged:
patch, mask = get_patch(image, position, patch_size)
p = normalize_histogram(extract_histogram(patch, nbins,
weights=kernel))
v = np.sqrt(np.divide(q, p + epsilon))
w = backproject_histogram(patch, v, nbins, kernel)
x_change = np.divide(np.sum(np.multiply(kernel_x, w)), np.sum(w))
y_change = np.divide(np.sum(np.multiply(kernel_y, w)), np.sum(w))
# print([floor(n) for n in position], x_change, y_change)
if abs(x_change) < epsilon and abs(y_change) < epsilon:
converged = True
else:
position = position[0] + x_change, position[1] + y_change
iters += 1
return int(floor(position[0])), int(floor(position[1])), iters
def test_meanshift_convergence(shape=(5, 5)):
global image, vis_image
# image = ex2_utils.generate_responses_1()
image = ex2_utils.genereate_response_2()
start_converge_map = {}
for x in range(image.shape[1]):
for y in range(image.shape[0]):
x_start = x
y_start = y
window_shape = shape
finish = False
i = 0
while i < 30 and not finish:
patch = ex2_utils.get_patch(image, (x_start, y_start),
window_shape)[0]
x_start, y_start, finish = mean_shift(patch,
x_start,
y_start,
kernel="epanechnikov")
i += 1
if x_start >= image.shape[0]:
x_start = image.shape[0] - 1
if y_start >= image.shape[1]:
y_start = image.shape[1] - 1
if (x_start, y_start) not in start_converge_map:
start_converge_map[(x_start, y_start)] = [(x, y)]
else:
start_converge_map[(x_start, y_start)].append((x, y))
print(start_converge_map)
return start_converge_map
def test_meanshift():
global image, vis_image
image = ex2_utils.generate_responses_1()
# image = ex2_utils.genereate_response_2()
vis_image = image.copy()
# cv2.imshow("test_map", vis_image * 255)
# cv2.setMouseCallback("test_map", on_click_rectangle)
# cv2.waitKey(0)
# cv2.imshow("test_map", vis_image * 255)
# cv2.waitKey(0)
#
# x_iter, y_iter = click["x"], click["y"]
x_iter = 34
y_iter = 61
window_shape = (51, 51)
finish = False
i = 0
visited = set()
while i < 100 and not finish:
patch = ex2_utils.get_patch(image, (x_iter, y_iter), window_shape)[0]
x_iter, y_iter, finish = mean_shift(patch,
x_iter,
y_iter,
kernel="epanechnikov")
if (x_iter, y_iter) in visited:
break
visited.add((x_iter, y_iter))
i += 1
print(f"Finished in {i} steps ")
print(f"Converged at x: {x_iter}, y: {y_iter}")
print(f"Start at x: {click['x']}, y: {click['y']}")
def track(self, image):
patch = ex2_utils.get_patch(image, self.position, self.size)
cv2.imshow("Debug", image)
cv2.imshow("Patch", patch[0])
cv2.waitKey(0)
print(f"Position:{self.position}, Size:{self.size}")
return [0, 10, 2, 5]
def track(self, image):
left = max(round(self.position[0] - float(self.window) / 2), 0)
top = max(round(self.position[1] - float(self.window) / 2), 0)
right = min(round(self.position[0] + float(self.window) / 2),
image.shape[1] - 1)
bottom = min(round(self.position[1] + float(self.window) / 2),
image.shape[0] - 1)
i = 0
# print(f"Position at start: {self.position}")
should_finish = False
while i < 20 and not should_finish:
current_frame = ex2_utils.get_patch(image, self.position,
self.size)[0]
# print(f"TRACK template shape: {current_frame.shape}")
# print(f"TRACK Kernel shape: {self.kernel.shape}")
# print(f"TRACK Size : {self.size}")
# print(f"TRACK position: {self.position}")
h2 = ex2_utils.extract_histogram(current_frame,
16,
weights=self.kernel)
h2 = h2 / np.sum(h2)
vu = np.sqrt(self.current_model / (h2 + self.parameters.epsilon))
back_proj = ex2_utils.backproject_histogram(current_frame, vu, 16)
# cv2.imshow("Backpproj",back_proj)
# cv2.imshow("Kernel",self.kernel)
# cv2.waitKey(0)
next_x, next_y, should_finish = mean_shift(
back_proj,
int(self.position[0]),
int(self.position[1]),
kernel="uniform",
)
if self.position == (next_x, next_y):
keep_tracking = False
self.position = (next_x, next_y)
# print(f"Position at iteration; {i} --> {self.position}")
i += 1
self.current_model = (
1 - self.parameters.alpha
) * self.current_model + self.parameters.alpha * h2
# self.template = ex2_utils.get_patch(image, self.position, self.size)[0]
# cv2.imshow("back proj", back_proj)
# cv2.waitKey(0)
bounding_box = [
self.position[0] - self.size[0] / 2,
self.position[1] - self.size[1] / 2, self.size[0], self.size[1]
]
# bounding_box = [left + max_loc[0], top + max_loc[1], self.size[0], self.size[1]]
return bounding_box
def draw_particles(image, particles, weights, position, size, color):
image2 = image.copy()
for (x, y, _, _), weight in zip(particles, weights):
r = np.random.randint(0, 255)
g = np.random.randint(0, 255)
b = np.random.randint(0, 255)
thickness = int(weight / 0.002) - 1
image2 = cv2.circle(image2, (int(x), int(y)),
radius=0,
color=(b, g, r),
thickness=thickness)
image2, _ = get_patch(image2, position, (size[0] * 2.5, size[1] * 2.5))
image2 = cv2.resize(image2,
dsize=(int(image2.shape[1] * 3),
int(image2.shape[0] * 3)))
show_image(image2, 0, "-")
def initialize(self, img, region):
"""
Initialize the mean-shift tracker.
Args:
img (numpy.ndarray): First image.
region (list): bounding box specification for the
object on the first image. First and second values
represent the position of the left-upper corner. The
third and fourth values represent the width and the
height of the bounding box respectively.
"""
# Set tracker name.
self.name = "mean-shift-tracker"
# Number of iterations performed to reposition bounding box,
# maximum number of iterations performed and umber of trackings performed.
self.num_it = []
self.max_it = 0
self.num_tracking_runs = 0
# Get initialized position.
self.pos = [region[0] + region[2] / 2, region[1] + region[3] / 2]
# Set tracking patch size. Increment size by one if even.
self.size = (int(region[2]) + abs(int(region[2]) % 2 - 1),
int(region[3]) + abs(int(region[3]) % 2 - 1))
# Initialize tracking window indices grid.
self.mesh_x, self.mesh_y = np.meshgrid(
np.arange(-self.size[0] // 2 + 1, self.size[0] // 2 + 1),
np.arange(-self.size[1] // 2 + 1, self.size[1] // 2 + 1))
# Initialize kernels.
self.kern1 = create_epanechnik_kernel(self.size[0], self.size[1], 2)
self.kern2 = np.ones((self.size[1], self.size[0]))
self.kern_bandwidth = 4
# Get initial patch.
patch = get_patch(img, self.pos, self.size)
# Extract hitrogram from template using the specified kernel.
hist_ = extract_histogram(patch[0],
self.parameters.n_bins,
weights=self.kern1)
self.hist = hist_ / np.sum(hist_)
def initialize(self, image, region):
if len(region) == 8:
x_ = np.array(region[::2])
y_ = np.array(region[1::2])
region = [
np.min(x_),
np.min(y_),
np.max(x_) - np.min(x_) + 1,
np.max(y_) - np.min(y_) + 1
]
self.window = max(region[2],
region[3]) * self.parameters.enlarge_factor
left = max(region[0], 0)
top = max(region[1], 0)
right = min(region[0] + region[2], image.shape[1] - 1)
bottom = min(region[1] + region[3], image.shape[0] - 1)
self.position = (region[0] + region[2] / 2, region[1] + region[3] / 2)
self.size = (region[2], region[3])
self.template = ex2_utils.get_patch(image, self.position, self.size)[0]
# print(self.size)
# print(self.position)
# print(self.window)
# cv2.imshow("Init", self.template)
# cv2.waitKey(0)
ep_kernel = ex2_utils.create_epanechnik_kernel(
self.size[0], self.size[1], self.parameters.kernel_sigma)
self.kernel = ep_kernel[:self.template.shape[0], :self.template.
shape[1]]
# print(f"template shape: {self.template.shape}")
# print(f"Kernel shape: {self.kernel.shape}")
# print(f"Size 0: {self.size[0]}")
# print(f"Size 1: {self.size[1]}")
h1 = ex2_utils.extract_histogram(self.template,
16,
weights=self.kernel)
self.current_model = h1 / np.sum(h1)
def initialize(self, image, region):
region = [int(el) for el in region]
if (region[2] % 2 == 0):
region[2] += 1
if (region[3] % 2 == 0):
region[3] += 1
self.kernel_sigma = 0.5
self.histogram_bins = 8
self.n_of_particles = 50
self.enlarge_factor = 2
self.distance_sigma = 0.11
self.update_alpha = 0.01
self.color_change = (True, "YCRCB")
self.draw_particles = False
self.dynamic_model = "NCV"
if self.color_change[0]:
image = change_colorspace(image, self.color_change[1])
if len(region) == 8:
x_ = np.array(region[::2])
y_ = np.array(region[1::2])
region = [
np.min(x_),
np.min(y_),
np.max(x_) - np.min(x_) + 1,
np.max(y_) - np.min(y_) + 1
]
self.window = max(region[2], region[3]) * self.enlarge_factor
self.position = (region[0] + region[2] / 2, region[1] + region[3] / 2)
self.size = (region[2], region[3])
self.template, _ = get_patch(image, self.position, self.size)
image_pl = image.shape[0] * image.shape[1]
patch_pl = self.size[0] * self.size[1]
q = int(patch_pl / image_pl * 200)
self.q = q if q > 0 else 1
#self.q = 100
# CREATING VISUAL MODEL
self.kernel = create_epanechnik_kernel(self.size[0], self.size[1],
self.kernel_sigma)
self.patch_size = self.kernel.shape
self.template_histogram = normalize_histogram(
extract_histogram(self.template,
self.histogram_bins,
weights=self.kernel))
# GENERATING DYNAMIC MODEL MATRICES
self.system_matrix, self.system_covariance = get_dynamic_model_matrices(
self.q, self.dynamic_model)
self.particle_state = [self.position[0], self.position[1]]
if self.dynamic_model == "NCV":
self.particle_state.extend([0, 0])
if self.dynamic_model == "NCA":
self.particle_state.extend([0, 0, 0, 0])
# GENERATING N PARTICLES AROUND POSITION
self.particles = sample_gauss(self.particle_state,
self.system_covariance,
self.n_of_particles)
self.weights = np.array(
[1 / self.n_of_particles for _ in range(self.n_of_particles)])
def track(self, image):
left = max(round(self.position[0] - float(self.window) / 2), 0)
top = max(round(self.position[1] - float(self.window) / 2), 0)
right = min(round(self.position[0] + float(self.window) / 2),
image.shape[1] - 1)
bottom = min(round(self.position[1] + float(self.window) / 2),
image.shape[0] - 1)
if right - left < self.template.shape[
1] or bottom - top < self.template.shape[0]:
return [
self.position[0] + self.size[0] / 2,
self.position[1] + self.size[1] / 2, self.size[0], self.size[1]
]
if self.color_change[0]:
image = change_colorspace(image, self.color_change[1])
# PARTICLE SAMPLING
weights_cumsumed = np.cumsum(self.weights)
rand_samples = np.random.rand(self.n_of_particles, 1)
sampled_idxs = np.digitize(rand_samples, weights_cumsumed)
particles_new = self.particles[sampled_idxs.flatten(), :]
noises = sample_gauss([0 for _ in range(self.system_matrix.shape[0])],
self.system_covariance, self.n_of_particles)
self.particles = np.transpose(
np.matmul(self.system_matrix,
np.transpose(particles_new))) + noises
for index, p in enumerate(particles_new):
p_x = self.particles[index][0]
p_y = self.particles[index][1]
try:
patch, _ = get_patch(image, (p_x, p_y), self.patch_size)
histogram = normalize_histogram(
extract_histogram(patch,
self.histogram_bins,
weights=self.kernel))
hell_dist = hellinger(histogram, self.template_histogram)
prob = dist_to_prob(hell_dist, self.distance_sigma)
except Exception as e:
prob = 0
self.weights[index] = prob
# NORMALIZE WEIGHTS
self.weights = self.weights / np.sum(self.weights)
# DRAWING PARTICLES
if self.draw_particles:
draw_particles(image, self.particles, self.weights, self.position,
self.size, (255, 0, 0))
# COMPUTE NEW POSITION
new_x = sum([
particle[0] * self.weights[index]
for index, particle in enumerate(self.particles)
])
new_y = sum([
particle[1] * self.weights[index]
for index, particle in enumerate(self.particles)
])
self.position = (new_x, new_y)
# UPDATE VISUAL MODEL
if self.update_alpha > 0:
self.template, _ = get_patch(image, (new_x, new_y),
self.patch_size)
self.template_histogram = (
1 - self.update_alpha
) * self.template_histogram + self.update_alpha * normalize_histogram(
extract_histogram(
self.template, self.histogram_bins, weights=self.kernel))
return [
new_x - self.size[0] / 2, new_y - self.size[1] / 2, self.size[0],
self.size[1]
]
def track(self, img):
"""
Perform tracking on next image using reference color histogram model.
Args:
img (numpy.ndarray): Image on which to localize the object
using the reference model.
Returns:
(list): bounding box specification for the
object on the first image. First and second values
represent the position of the left-upper corner. The
third and fourth values represent the width and the
height of the bounding box respectively.
"""
# Initialize convergence flag.
convergence_flg = False
# Initialize iteration counter.
num_it = 0
# Repeat until convergence or until maximum number of iterations
# exceeded.
while not convergence_flg and num_it < self.parameters.max_it:
# Increment iteration counter.
num_it += 1
# Extract histogram and current location.
patch = get_patch(img, self.pos, self.size)
hist_ = extract_histogram(patch[0],
self.parameters.n_bins,
weights=self.kern1)
hist_nxt = hist_ / np.sum(hist_)
# Compute the weights w_{i}.
weights = np.sqrt(self.hist / (hist_nxt + 1.0e-4))
# Backproject within extracted patch using weights v.
bp = backproject_histogram(patch[0], weights,
self.parameters.n_bins)
# Get changes in x and y directions.
delta_x = np.sum(self.mesh_x * bp) / np.sum(bp)
delta_y = np.sum(self.mesh_y * bp) / np.sum(bp)
# Check if division successful.
if np.isnan(delta_x) or np.isnan(delta_y):
break
# If changes sufficiently small or if maximum number of iterations exceeded.
if abs(delta_x) < 1.0 and abs(delta_y) < 1.0:
# Set convergence flag.
convergence_flg = True
# Increment number of total iterations and trackings.
self.num_it.append(num_it)
self.num_tracking_runs += 1
# If new maximum of iteration number observed.
if num_it > self.max_it:
self.max_it = num_it
else:
# Add changes in x and y direction to current position.
self.pos[0] += np.round(delta_x)
self.pos[1] += np.round(delta_y)
# Update reference model with current model.
self.hist = (1 - self.parameters.alpha
) * self.hist + self.parameters.alpha * hist_nxt
# Return found position.
return [
self.pos[0] - self.size[0] / 2, self.pos[1] - self.size[1] / 2,
self.size[0], self.size[1]
]