Exemple #1
0
    def process_frame(self, im_gray):

        tracked_keypoints, _ = util.track(self.im_prev, im_gray,
                                          self.active_keypoints)
        (center, scale_estimate, rotation_estimate,
         tracked_keypoints) = self.estimate(tracked_keypoints)

        # Detect keypoints, compute descriptors
        keypoints_cv = self.detector.detect(im_gray)
        keypoints_cv, features = self.descriptor.compute(im_gray, keypoints_cv)

        # Create list of active keypoints
        active_keypoints = zeros((0, 3))

        # Get the best two matches for each feature
        matches_all = self.matcher.knnMatch(features, self.features_database,
                                            2)
        # Get all matches for selected features
        if not any(isnan(center)):
            selected_matches_all = self.matcher.knnMatch(
                features, self.selected_features, len(self.selected_features))

        # For each keypoint and its descriptor
        if len(keypoints_cv) > 0:
            transformed_springs = scale_estimate * util.rotate(
                self.springs, -rotation_estimate)
            for i in range(len(keypoints_cv)):

                # Retrieve keypoint location
                location = np.array(keypoints_cv[i].pt)

                # First: Match over whole image
                # Compute distances to all descriptors
                matches = matches_all[i]
                distances = np.array([m.distance for m in matches])

                # Convert distances to confidences, do not weight
                combined = 1 - distances / self.DESC_LENGTH

                classes = self.database_classes

                # Get best and second best index
                bestInd = matches[0].trainIdx
                secondBestInd = matches[1].trainIdx

                # Compute distance ratio according to Lowe
                ratio = (1 - combined[0]) / (1 - combined[1])

                # Extract class of best match
                keypoint_class = classes[bestInd]

                # If distance ratio is ok and absolute distance is ok and keypoint class is not background
                if ratio < self.THR_RATIO and combined[
                        0] > self.THR_CONF and keypoint_class != 0:
                    # Add keypoint to active keypoints
                    new_kpt = append(location, keypoint_class)
                    active_keypoints = append(active_keypoints,
                                              array([new_kpt]),
                                              axis=0)

                # In a second step, try to match difficult keypoints
                # If structural constraints are applicable
                if not any(isnan(center)):

                    # Compute distances to initial descriptors
                    matches = selected_matches_all[i]
                    distances = np.array([m.distance for m in matches])
                    # Re-order the distances based on indexing
                    idxs = np.argsort(np.array([m.trainIdx for m in matches]))
                    distances = distances[idxs]

                    # Convert distances to confidences
                    confidences = 1 - distances / self.DESC_LENGTH

                    # Compute the keypoint location relative to the object center
                    relative_location = location - center

                    # Compute the distances to all springs
                    displacements = util.L2norm(transformed_springs -
                                                relative_location)

                    # For each spring, calculate weight
                    weight = displacements < self.THR_OUTLIER  # Could be smooth function

                    combined = weight * confidences

                    classes = self.selected_classes

                    # Sort in descending order
                    sorted_conf = argsort(combined)[::-1]  # reverse

                    # Get best and second best index
                    bestInd = sorted_conf[0]
                    secondBestInd = sorted_conf[1]

                    # Compute distance ratio according to Lowe
                    ratio = (1 - combined[bestInd]) / (1 -
                                                       combined[secondBestInd])

                    # Extract class of best match
                    keypoint_class = classes[bestInd]

                    # If distance ratio is ok and absolute distance is ok and keypoint class is not background
                    if ratio < self.THR_RATIO and combined[
                            bestInd] > self.THR_CONF and keypoint_class != 0:

                        # Add keypoint to active keypoints
                        new_kpt = append(location, keypoint_class)

                        # Check whether same class already exists
                        if active_keypoints.size > 0:
                            same_class = np.nonzero(
                                active_keypoints[:, 2] == keypoint_class)
                            active_keypoints = np.delete(active_keypoints,
                                                         same_class,
                                                         axis=0)

                        active_keypoints = append(active_keypoints,
                                                  array([new_kpt]),
                                                  axis=0)

        # If some keypoints have been tracked
        if tracked_keypoints.size > 0:

            # Extract the keypoint classes
            tracked_classes = tracked_keypoints[:, 2]

            # If there already are some active keypoints
            if active_keypoints.size > 0:

                # Add all tracked keypoints that have not been matched
                associated_classes = active_keypoints[:, 2]
                missing = ~np.in1d(tracked_classes, associated_classes)
                active_keypoints = append(active_keypoints,
                                          tracked_keypoints[missing, :],
                                          axis=0)

            # Else use all tracked keypoints
            else:
                active_keypoints = tracked_keypoints

        # Update object state estimate
        _ = active_keypoints
        self.center = center
        self.scale_estimate = scale_estimate
        self.rotation_estimate = rotation_estimate
        self.tracked_keypoints = tracked_keypoints
        self.active_keypoints = active_keypoints
        self.im_prev = im_gray
        self.keypoints_cv = keypoints_cv
        _ = time.time()

        self.tl = (nan, nan)
        self.tr = (nan, nan)
        self.br = (nan, nan)
        self.bl = (nan, nan)

        self.bb = array([nan, nan, nan, nan])

        self.has_result = False
        if not any(
                isnan(self.center)
        ) and self.active_keypoints.shape[0] > self.num_initial_keypoints / 10:
            self.has_result = True

            tl = util.array_to_int_tuple(center + scale_estimate * util.rotate(
                self.center_to_tl[None, :], rotation_estimate).squeeze())
            tr = util.array_to_int_tuple(center + scale_estimate * util.rotate(
                self.center_to_tr[None, :], rotation_estimate).squeeze())
            br = util.array_to_int_tuple(center + scale_estimate * util.rotate(
                self.center_to_br[None, :], rotation_estimate).squeeze())
            bl = util.array_to_int_tuple(center + scale_estimate * util.rotate(
                self.center_to_bl[None, :], rotation_estimate).squeeze())

            min_x = min((tl[0], tr[0], br[0], bl[0]))
            min_y = min((tl[1], tr[1], br[1], bl[1]))
            max_x = max((tl[0], tr[0], br[0], bl[0]))
            max_y = max((tl[1], tr[1], br[1], bl[1]))

            self.tl = tl
            self.tr = tr
            self.bl = bl
            self.br = br

            self.bb = np.array([min_x, min_y, max_x - min_x, max_y - min_y])
Exemple #2
0
	def process_frame(self, im_gray):

		tracked_keypoints, status = util.track(self.im_prev, im_gray, self.active_keypoints)
		(center, scale_estimate, rotation_estimate, tracked_keypoints) = self.estimate(tracked_keypoints)

		#Detect keypoints, compute descriptors
		keypoints_cv = self.detector.detect(im_gray) 
		keypoints_cv, features = self.descriptor.compute(im_gray, keypoints_cv)

		#Create list of active keypoints
		active_keypoints = zeros((0,3)) 

		#For each keypoint and its descriptor
		if len(keypoints_cv) > 0:
			for (keypoint_cv, feature) in zip(keypoints_cv, features):

				#Retrieve keypoint location
				location = np.array(keypoint_cv.pt)

				#First: Match over whole image
				#Compute distances to all descriptors
				matches = self.matcher.match(self.features_database, feature[None,:])
				distances = np.array([m.distance for m in matches])

				#Convert distances to confidences, do not weight
				combined = 1 - distances / self.DESC_LENGTH

				classes = self.database_classes

				#Sort in descending order
				sorted_conf = argsort(combined)[::-1] #reverse

				#Get best and second best index
				bestInd = sorted_conf[0]
				secondBestInd = sorted_conf[1]

				#Compute distance ratio according to Lowe
				ratio = (1-combined[bestInd]) / (1-combined[secondBestInd])

				#Extract class of best match
				keypoint_class = classes[bestInd]

				#If distance ratio is ok and absolute distance is ok and keypoint class is not background
				if ratio < self.THR_RATIO and combined[bestInd] > self.THR_CONF and keypoint_class != 0:

					#Add keypoint to active keypoints
					new_kpt = append(location, keypoint_class)
					active_keypoints = append(active_keypoints, array([new_kpt]), axis=0)

				#In a second step, try to match difficult keypoints
				#If structural constraints are applicable
				if not any(isnan(center)):

					#Compute distances to initial descriptors
					matches = self.matcher.match(self.selected_features, feature[None,:])
					distances = np.array([m.distance for m in matches])

					#Convert distances to confidences
					confidences = 1 - distances / self.DESC_LENGTH

					#Compute the keypoint location relative to the object center
					relative_location = location - center

					#Compute the distances to all springs
					displacements = util.L2norm(scale_estimate * util.rotate(self.springs, -rotation_estimate) - relative_location)

					#For each spring, calculate weight
					weight = displacements < self.THR_OUTLIER #Could be smooth function

					combined = weight * confidences

					classes = self.selected_classes

					#Sort in descending order
					sorted_conf = argsort(combined)[::-1] #reverse

					#Get best and second best index
					bestInd = sorted_conf[0]
					secondBestInd = sorted_conf[1]

					#Compute distance ratio according to Lowe
					ratio = (1-combined[bestInd]) / (1-combined[secondBestInd])

					#Extract class of best match
					keypoint_class = classes[bestInd]

					#If distance ratio is ok and absolute distance is ok and keypoint class is not background
					if ratio < self.THR_RATIO and combined[bestInd] > self.THR_CONF and keypoint_class != 0:

						#Add keypoint to active keypoints
						new_kpt = append(location, keypoint_class)

						#Check whether same class already exists
						if active_keypoints.size > 0:
							same_class = np.nonzero(active_keypoints[:,2] == keypoint_class)
							active_keypoints = np.delete(active_keypoints, same_class, axis=0)

						active_keypoints = append(active_keypoints, array([new_kpt]), axis=0)

		#If some keypoints have been tracked
		if tracked_keypoints.size > 0:

			#Extract the keypoint classes
			tracked_classes = tracked_keypoints[:,2]

			#If there already are some active keypoints
			if active_keypoints.size > 0:

				#Add all tracked keypoints that have not been matched
				associated_classes = active_keypoints[:,2]
				missing = ~np.in1d(tracked_classes, associated_classes)
				active_keypoints = append(active_keypoints, tracked_keypoints[missing,:], axis=0)

			#Else use all tracked keypoints
			else:
				active_keypoints = tracked_keypoints

		#Update object state estimate
		active_keypoints_before = active_keypoints
		self.center = center
		self.scale_estimate = scale_estimate
		self.rotation_estimate = rotation_estimate
		self.tracked_keypoints = tracked_keypoints
		self.active_keypoints = active_keypoints
		self.im_prev = im_gray
		self.keypoints_cv = keypoints_cv
		toc = time.time()

		self.tl = (nan,nan)
		self.tr = (nan,nan)
		self.br = (nan,nan)
		self.bl = (nan,nan)

		self.bb = array([nan,nan,nan,nan])

		self.has_result = False
		if not any(isnan(self.center)) and self.active_keypoints.shape[0] > self.num_initial_keypoints / 10:
			self.has_result = True

			tl = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_tl[None,:], rotation_estimate).squeeze())
			tr = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_tr[None,:], rotation_estimate).squeeze())
			br = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_br[None,:], rotation_estimate).squeeze())
			bl = util.array_to_int_tuple(center + scale_estimate*util.rotate(self.center_to_bl[None,:], rotation_estimate).squeeze())

			min_x = min((tl[0],tr[0],br[0],bl[0]))
			min_y = min((tl[1],tr[1],br[1],bl[1]))
			max_x = max((tl[0],tr[0],br[0],bl[0]))
			max_y = max((tl[1],tr[1],br[1],bl[1]))

			self.tl = tl
			self.tr = tr
			self.bl = bl
			self.br = br

			self.bb = np.array([min_x, min_y, max_x - min_x, max_y - min_y])
Exemple #3
0
    init_region = np.genfromtxt('region.txt', delimiter=',')
    num_frames = len(images)

    results = empty((num_frames, 4))
    results[:] = nan

    results[0, :] = init_region

    frame = 0

    im0 = cv2.imread(images[frame])
    im_gray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
    im_draw = np.copy(im0)

    tl, br = (util.array_to_int_tuple(init_region[:2]),
              util.array_to_int_tuple(init_region[:2] + init_region[2:4] - 1))

    try:
        CMT.initialise(im_gray0, tl, br)
        while frame < num_frames:
            im = cv2.imread(images[frame])
            im_gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
            CMT.process_frame(im_gray)
            results[frame, :] = CMT.bb

            # Advance frame number
            frame += 1
    except:
        pass  # Swallow errors
Exemple #4
0
	init_region = np.genfromtxt('region.txt', delimiter=',')
	num_frames = len(images)

	results = empty((num_frames, 4))
	results[:] = nan

	results[0, :] = init_region

	frame = 0

	im0 = cv2.imread(images[frame])
	im_gray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
	im_draw = np.copy(im0)

	tl, br = (util.array_to_int_tuple(init_region[:2]), util.array_to_int_tuple(init_region[:2] + init_region[2:4] - 1))

	try: 
		CMT.initialise(im_gray0, tl, br)
		while frame < num_frames:
			im = cv2.imread(images[frame])
			im_gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
			CMT.process_frame(im_gray)
			results[frame, :] = CMT.bb

			# Advance frame number
			frame += 1
	except:
		pass  # Swallow errors

	np.savetxt('output.txt', results, delimiter=',')