def process_frame(self, im_gray):
tracked_keypoints, _ = util.track(self.im_prev, im_gray,
self.active_keypoints)
(center, scale_estimate, rotation_estimate,
tracked_keypoints) = self.estimate(tracked_keypoints)
# Detect keypoints, compute descriptors
keypoints_cv = self.detector.detect(im_gray)
keypoints_cv, features = self.descriptor.compute(im_gray, keypoints_cv)
# Create list of active keypoints
active_keypoints = zeros((0, 3))
# Get the best two matches for each feature
matches_all = self.matcher.knnMatch(features, self.features_database,
2)
# Get all matches for selected features
if not any(isnan(center)):
selected_matches_all = self.matcher.knnMatch(
features, self.selected_features, len(self.selected_features))
# For each keypoint and its descriptor
if len(keypoints_cv) > 0:
transformed_springs = scale_estimate * util.rotate(
self.springs, -rotation_estimate)
for i in range(len(keypoints_cv)):
# Retrieve keypoint location
location = np.array(keypoints_cv[i].pt)
# First: Match over whole image
# Compute distances to all descriptors
matches = matches_all[i]
distances = np.array([m.distance for m in matches])
# Convert distances to confidences, do not weight
combined = 1 - distances / self.DESC_LENGTH
classes = self.database_classes
# Get best and second best index
bestInd = matches[0].trainIdx
secondBestInd = matches[1].trainIdx
# Compute distance ratio according to Lowe
ratio = (1 - combined[0]) / (1 - combined[1])
# Extract class of best match
keypoint_class = classes[bestInd]
# If distance ratio is ok and absolute distance is ok and keypoint class is not background
if ratio < self.THR_RATIO and combined[
0] > self.THR_CONF and keypoint_class != 0:
# Add keypoint to active keypoints
new_kpt = append(location, keypoint_class)
active_keypoints = append(active_keypoints,
array([new_kpt]),
axis=0)
# In a second step, try to match difficult keypoints
# If structural constraints are applicable
if not any(isnan(center)):
# Compute distances to initial descriptors
matches = selected_matches_all[i]
distances = np.array([m.distance for m in matches])
# Re-order the distances based on indexing
idxs = np.argsort(np.array([m.trainIdx for m in matches]))
distances = distances[idxs]
# Convert distances to confidences
confidences = 1 - distances / self.DESC_LENGTH
# Compute the keypoint location relative to the object center
relative_location = location - center
# Compute the distances to all springs
displacements = util.L2norm(transformed_springs -
relative_location)
# For each spring, calculate weight
weight = displacements < self.THR_OUTLIER # Could be smooth function
combined = weight * confidences
classes = self.selected_classes
# Sort in descending order
sorted_conf = argsort(combined)[::-1] # reverse
# Get best and second best index
bestInd = sorted_conf[0]
secondBestInd = sorted_conf[1]
# Compute distance ratio according to Lowe
ratio = (1 - combined[bestInd]) / (1 -
combined[secondBestInd])
# Extract class of best match
keypoint_class = classes[bestInd]
# If distance ratio is ok and absolute distance is ok and keypoint class is not background
if ratio < self.THR_RATIO and combined[
bestInd] > self.THR_CONF and keypoint_class != 0:
# Add keypoint to active keypoints
new_kpt = append(location, keypoint_class)
# Check whether same class already exists
if active_keypoints.size > 0:
same_class = np.nonzero(
active_keypoints[:, 2] == keypoint_class)
active_keypoints = np.delete(active_keypoints,
same_class,
axis=0)
active_keypoints = append(active_keypoints,
array([new_kpt]),
axis=0)
# If some keypoints have been tracked
if tracked_keypoints.size > 0:
# Extract the keypoint classes
tracked_classes = tracked_keypoints[:, 2]
# If there already are some active keypoints
if active_keypoints.size > 0:
# Add all tracked keypoints that have not been matched
associated_classes = active_keypoints[:, 2]
missing = ~np.in1d(tracked_classes, associated_classes)
active_keypoints = append(active_keypoints,
tracked_keypoints[missing, :],
axis=0)
# Else use all tracked keypoints
else:
active_keypoints = tracked_keypoints
# Update object state estimate
_ = active_keypoints
self.center = center
self.scale_estimate = scale_estimate
self.rotation_estimate = rotation_estimate
self.tracked_keypoints = tracked_keypoints
self.active_keypoints = active_keypoints
self.im_prev = im_gray
self.keypoints_cv = keypoints_cv
_ = time.time()
self.tl = (nan, nan)
self.tr = (nan, nan)
self.br = (nan, nan)
self.bl = (nan, nan)
self.bb = array([nan, nan, nan, nan])
self.has_result = False
if not any(
isnan(self.center)
) and self.active_keypoints.shape[0] > self.num_initial_keypoints / 10:
self.has_result = True
tl = util.array_to_int_tuple(center + scale_estimate * util.rotate(
self.center_to_tl[None, :], rotation_estimate).squeeze())
tr = util.array_to_int_tuple(center + scale_estimate * util.rotate(
self.center_to_tr[None, :], rotation_estimate).squeeze())
br = util.array_to_int_tuple(center + scale_estimate * util.rotate(
self.center_to_br[None, :], rotation_estimate).squeeze())
bl = util.array_to_int_tuple(center + scale_estimate * util.rotate(
self.center_to_bl[None, :], rotation_estimate).squeeze())
min_x = min((tl[0], tr[0], br[0], bl[0]))
min_y = min((tl[1], tr[1], br[1], bl[1]))
max_x = max((tl[0], tr[0], br[0], bl[0]))
max_y = max((tl[1], tr[1], br[1], bl[1]))
self.tl = tl
self.tr = tr
self.bl = bl
self.br = br
self.bb = np.array([min_x, min_y, max_x - min_x, max_y - min_y])
def process_frame(self, im_gray): tracked_keypoints, status = util.track(self.im_prev, im_gray, self.active_keypoints) (center, scale_estimate, rotation_estimate, tracked_keypoints) = self.estimate(tracked_keypoints) #Detect keypoints, compute descriptors keypoints_cv = self.detector.detect(im_gray) keypoints_cv, features = self.descriptor.compute(im_gray, keypoints_cv) #Create list of active keypoints active_keypoints = zeros((0,3)) #For each keypoint and its descriptor if len(keypoints_cv) > 0: for (keypoint_cv, feature) in zip(keypoints_cv, features): #Retrieve keypoint location location = np.array(keypoint_cv.pt) #First: Match over whole image #Compute distances to all descriptors matches = self.matcher.match(self.features_database, feature[None,:]) distances = np.array([m.distance for m in matches]) #Convert distances to confidences, do not weight combined = 1 - distances / self.DESC_LENGTH classes = self.database_classes #Sort in descending order sorted_conf = argsort(combined)[::-1] #reverse #Get best and second best index bestInd = sorted_conf[0] secondBestInd = sorted_conf[1] #Compute distance ratio according to Lowe ratio = (1-combined[bestInd]) / (1-combined[secondBestInd]) #Extract class of best match keypoint_class = classes[bestInd] #If distance ratio is ok and absolute distance is ok and keypoint class is not background if ratio < self.THR_RATIO and combined[bestInd] > self.THR_CONF and keypoint_class != 0: #Add keypoint to active keypoints new_kpt = append(location, keypoint_class) active_keypoints = append(active_keypoints, array([new_kpt]), axis=0) #In a second step, try to match difficult keypoints #If structural constraints are applicable if not any(isnan(center)): #Compute distances to initial descriptors matches = self.matcher.match(self.selected_features, feature[None,:]) distances = np.array([m.distance for m in matches]) #Convert distances to confidences confidences = 1 - distances / self.DESC_LENGTH #Compute the keypoint location relative to the object center relative_location = location - center #Compute the distances to all springs displacements = util.L2norm(scale_estimate * util.rotate(self.springs, -rotation_estimate) - relative_location) #For each spring, calculate weight weight = displacements < self.THR_OUTLIER #Could be smooth function combined = weight * confidences classes = self.selected_classes #Sort in descending order sorted_conf = argsort(combined)[::-1] #reverse #Get best and second best index bestInd = sorted_conf[0] secondBestInd = sorted_conf[1] #Compute distance ratio according to Lowe ratio = (1-combined[bestInd]) / (1-combined[secondBestInd]) #Extract class of best match keypoint_class = classes[bestInd] #If distance ratio is ok and absolute distance is ok and keypoint class is not background if ratio < self.THR_RATIO and combined[bestInd] > self.THR_CONF and keypoint_class != 0: #Add keypoint to active keypoints new_kpt = append(location, keypoint_class) #Check whether same class already exists if active_keypoints.size > 0: same_class = np.nonzero(active_keypoints[:,2] == keypoint_class) active_keypoints = np.delete(active_keypoints, same_class, axis=0) active_keypoints = append(active_keypoints, array([new_kpt]), axis=0) #If some keypoints have been tracked if tracked_keypoints.size > 0: #Extract the keypoint classes tracked_classes = tracked_keypoints[:,2] #If there already are some active keypoints if active_keypoints.size > 0: #Add all tracked keypoints that have not been matched associated_classes = active_keypoints[:,2] missing = ~np.in1d(tracked_classes, associated_classes) active_keypoints = append(active_keypoints, tracked_keypoints[missing,:], axis=0) #Else use all tracked keypoints else: active_keypoints = tracked_keypoints #Update object state estimate active_keypoints_before = active_keypoints self.center = center self.scale_estimate = scale_estimate self.rotation_estimate = rotation_estimate self.tracked_keypoints = tracked_keypoints self.active_keypoints = active_keypoints self.im_prev = im_gray self.keypoints_cv = keypoints_cv toc = time.time() self.tl = (nan,nan) self.tr = (nan,nan) self.br = (nan,nan) self.bl = (nan,nan) self.bb = array([nan,nan,nan,nan]) self.has_result = False if not any(isnan(self.center)) and self.active_keypoints.shape[0] > self.num_initial_keypoints / 10: self.has_result = True tl = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_tl[None,:], rotation_estimate).squeeze()) tr = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_tr[None,:], rotation_estimate).squeeze()) br = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_br[None,:], rotation_estimate).squeeze()) bl = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_bl[None,:], rotation_estimate).squeeze()) min_x = min((tl[0],tr[0],br[0],bl[0])) min_y = min((tl[1],tr[1],br[1],bl[1])) max_x = max((tl[0],tr[0],br[0],bl[0])) max_y = max((tl[1],tr[1],br[1],bl[1])) self.tl = tl self.tr = tr self.bl = bl self.br = br self.bb = np.array([min_x, min_y, max_x - min_x, max_y - min_y])
init_region = np.genfromtxt('region.txt', delimiter=',')
num_frames = len(images)
results = empty((num_frames, 4))
results[:] = nan
results[0, :] = init_region
frame = 0
im0 = cv2.imread(images[frame])
im_gray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
im_draw = np.copy(im0)
tl, br = (util.array_to_int_tuple(init_region[:2]),
util.array_to_int_tuple(init_region[:2] + init_region[2:4] - 1))
try:
CMT.initialise(im_gray0, tl, br)
while frame < num_frames:
im = cv2.imread(images[frame])
im_gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
CMT.process_frame(im_gray)
results[frame, :] = CMT.bb
# Advance frame number
frame += 1
except:
pass # Swallow errors
init_region = np.genfromtxt('region.txt', delimiter=',')
num_frames = len(images)
results = empty((num_frames, 4))
results[:] = nan
results[0, :] = init_region
frame = 0
im0 = cv2.imread(images[frame])
im_gray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
im_draw = np.copy(im0)
tl, br = (util.array_to_int_tuple(init_region[:2]), util.array_to_int_tuple(init_region[:2] + init_region[2:4] - 1))
try:
CMT.initialise(im_gray0, tl, br)
while frame < num_frames:
im = cv2.imread(images[frame])
im_gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
CMT.process_frame(im_gray)
results[frame, :] = CMT.bb
# Advance frame number
frame += 1
except:
pass # Swallow errors
np.savetxt('output.txt', results, delimiter=',')
