def test_dataset_load_features_sift(tmpdir):
data = data_generation.create_berlin_test_folder(tmpdir)
assert len(data.images()) == 3
data.config["feature_type"] = "SIFT"
image = data.images()[0]
points = np.random.random((3, 4))
descriptors = np.random.random((128, 4))
colors = np.random.random((3, 4))
segmentations = np.random.random((3, 4))
instances = np.random.random((3, 4))
semantic_data = features.SemanticData(
segmentations, instances, data.segmentation_labels()
)
before = features.FeaturesData(points, descriptors, colors, semantic_data)
data.save_features(image, before)
after = data.load_features(image)
assert np.allclose(points, after.points)
assert np.allclose(descriptors, after.descriptors)
assert np.allclose(colors, after.colors)
assert np.allclose(
segmentations,
after.semantic.segmentation,
)
assert np.allclose(instances, after.semantic.instances)
def load_features(self, image: str) -> Optional[features.FeaturesData]:
return features.FeaturesData(
self.features[image],
self.descriptors[image],
self.colors[image],
None,
)
def load_points_colors_segmentations_instances(
self, data: DataSetBase, image: str) -> Optional[ft.FeaturesData]:
all_features_data = self._load_features_nocache(data, image)
if not all_features_data:
return None
return ft.FeaturesData(
all_features_data.points,
None,
all_features_data.colors,
all_features_data.semantic,
)
def import_features(db, data, image_map, camera_map):
cursor = db.cursor()
cursor.execute("SELECT image_id, rows, cols, data FROM keypoints;")
keypoints = {}
colors = {}
for row in cursor:
image_id, n_rows, n_cols, arr = row
filename, camera_id = image_map[image_id]
cam = camera_map[camera_id]
arr = np.fromstring(arr, dtype=np.float32).reshape((n_rows, n_cols))
rgb = data.load_image(filename).astype(np.float32)
xc = np.clip(arr[:, 1].astype(int), 0, rgb.shape[0] - 1)
yc = np.clip(arr[:, 0].astype(int), 0, rgb.shape[1] - 1)
colors[image_id] = rgb[xc, yc, :]
arr[:, :2] = features.normalized_image_coordinates(
arr[:, :2], cam.width, cam.height)
if n_cols == 4:
x, y, s, o = arr[:, 0], arr[:, 1], arr[:, 2], arr[:, 3]
elif n_cols == 6:
x, y = arr[:, 0], arr[:, 1]
s, o = get_scale_orientation_from_affine(arr)
elif n_cols == 2:
x, y = arr[:, 0], arr[:, 1]
s = np.zeros_like(x)
o = np.zeros_like(x)
else:
raise ValueError
s = s / max(cam.width, cam.height)
keypoints[image_id] = np.vstack((x, y, s, o)).T
cursor.execute("SELECT image_id, rows, cols, data FROM descriptors;")
for row in cursor:
image_id, n_rows, n_cols, arr = row
filename, _ = image_map[image_id]
descriptors = np.fromstring(arr, dtype=np.uint8).reshape(
(n_rows, n_cols))
kp = keypoints[image_id]
features_data = features.FeaturesData(kp, descriptors,
colors[image_id], None)
data.save_features(filename, features_data)
cursor.close()
return keypoints
def generate_track_data(
reconstruction: types.Reconstruction,
maximum_depth: float,
projection_noise: float,
gcp_noise: Tuple[float, float],
gcps_count: Optional[int],
gcp_shift: Optional[np.ndarray],
on_disk_features_filename: Optional[str],
) -> Tuple[
sd.SyntheticFeatures, pymap.TracksManager, Dict[str, pymap.GroundControlPoint]
]:
"""Generate projection data from a reconstruction, considering a maximum
viewing depth and gaussian noise added to the ideal projections.
Returns feature/descriptor/color data per shot and a tracks manager object.
"""
tracks_manager = pymap.TracksManager()
feature_data_type = np.float32
desc_size = 128
non_zeroes = 5
points_ids = list(reconstruction.points)
points_coordinates = [p.coordinates for p in reconstruction.points.values()]
points_colors = [p.color for p in reconstruction.points.values()]
# generate random descriptors per point
track_descriptors = []
for _ in points_coordinates:
descriptor = np.zeros(desc_size)
for _ in range(non_zeroes):
index = np.random.randint(0, desc_size)
descriptor[index] = np.random.random() * 255
track_descriptors.append(descriptor.round().astype(feature_data_type))
# should speed-up projection queries
points_tree = spatial.cKDTree(points_coordinates)
start = time.time()
features = sd.SyntheticFeatures(on_disk_features_filename)
default_scale = 0.004
for index, (shot_index, shot) in enumerate(reconstruction.shots.items()):
# query all closest points
neighbors = list(
sorted(points_tree.query_ball_point(shot.pose.get_origin(), maximum_depth))
)
# project them
projections = shot.project_many(
np.array([points_coordinates[c] for c in neighbors])
)
# shot constants
center = shot.pose.get_origin()
z_axis = shot.pose.get_rotation_matrix()[2]
is_panorama = pygeometry.Camera.is_panorama(shot.camera.projection_type)
perturbation = float(projection_noise) / float(
max(shot.camera.width, shot.camera.height)
)
sigmas = np.array([perturbation, perturbation])
# pre-generate random perturbations
perturbations = np.random.normal(0.0, sigmas, (len(projections), 2))
# run and check valid projections
projections_inside = []
descriptors_inside = []
colors_inside = []
for i, (p_id, projection) in enumerate(zip(neighbors, projections)):
if not _is_inside_camera(projection, shot.camera):
continue
point = points_coordinates[p_id]
if not is_panorama and not _is_in_front(point, center, z_axis):
continue
# add perturbation
projection += perturbations[i]
# push data
color = points_colors[p_id]
original_id = points_ids[p_id]
projections_inside.append([projection[0], projection[1], default_scale])
descriptors_inside.append(track_descriptors[p_id])
colors_inside.append(color)
obs = pymap.Observation(
projection[0],
projection[1],
default_scale,
color[0],
color[1],
color[2],
len(projections_inside) - 1,
)
tracks_manager.add_observation(str(shot_index), str(original_id), obs)
features[shot_index] = oft.FeaturesData(
np.array(projections_inside),
np.array(descriptors_inside),
np.array(colors_inside),
None,
)
if index % 100 == 0:
logger.info(
f"Flushing images # {index} ({(time.time() - start)/(index+1)} sec. per image"
)
features.sync()
gcps = {}
if gcps_count is not None and gcp_shift is not None:
all_track_ids = list(tracks_manager.get_track_ids())
gcps_ids = [
all_track_ids[i]
for i in np.random.randint(len(all_track_ids) - 1, size=gcps_count)
]
sigmas_gcp = np.random.normal(
0.0,
np.array([gcp_noise[0], gcp_noise[0], gcp_noise[1]]),
(len(gcps_ids), 3),
)
for i, gcp_id in enumerate(gcps_ids):
point = reconstruction.points[gcp_id]
gcp = pymap.GroundControlPoint()
gcp.id = f"gcp-{gcp_id}"
enu = point.coordinates + gcp_shift + sigmas_gcp[i]
lat, lon, alt = reconstruction.reference.to_lla(*enu)
gcp.lla = {"latitude": lat, "longitude": lon, "altitude": alt}
gcp.has_altitude = True
for shot_id, obs in tracks_manager.get_track_observations(gcp_id).items():
o = pymap.GroundControlPointObservation()
o.shot_id = shot_id
o.projection = obs.point
gcp.add_observation(o)
gcps[gcp.id] = gcp
return features, tracks_manager, gcps
def detect(
image: str,
image_array: np.ndarray,
segmentation_array: Optional[np.ndarray],
instances_array: Optional[np.ndarray],
data: DataSetBase,
force: bool = False,
) -> None:
log.setup()
need_words = (
data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0
)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if not force and has_features and has_words:
logger.info(
"Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image
)
)
return
logger.info(
"Extracting {} features for image {}".format(data.feature_type().upper(), image)
)
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image, image_array)
)
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
s_unsorted, i_unsorted = bake_segmentation(
image_array, p_unmasked, segmentation_array, instances_array, exif
)
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = masking.load_features_mask(data, image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
if s_unsorted is not None:
semantic_data = features.SemanticData(
s_unsorted[order],
i_unsorted[order] if i_unsorted is not None else None,
data.segmentation_labels(),
)
else:
semantic_data = None
features_data = features.FeaturesData(p_sorted, f_sorted, c_sorted, semantic_data)
data.save_features(image, features_data)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(
f_sorted, n_closest, data.config["bow_matcher_type"]
)
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def generate_track_data(
reconstruction: types.Reconstruction, maximum_depth: float,
noise: float
) -> Tuple[Dict[str, oft.FeaturesData], pysfm.TracksManager, ]:
"""Generate projection data from a reconstruction, considering a maximum
viewing depth and gaussian noise added to the ideal projections.
Returns feature/descriptor/color data per shot and a tracks manager object.
"""
tracks_manager = pysfm.TracksManager()
feature_data_type = np.float32
desc_size = 128
non_zeroes = 5
points_ids = list(reconstruction.points)
points_coordinates = [
p.coordinates for p in reconstruction.points.values()
]
points_colors = [p.color for p in reconstruction.points.values()]
# generate random descriptors per point
track_descriptors = []
for _ in points_coordinates:
descriptor = np.zeros(desc_size)
for _ in range(non_zeroes):
index = np.random.randint(0, desc_size)
descriptor[index] = np.random.random() * 255
track_descriptors.append(descriptor.round().astype(feature_data_type))
# should speed-up projection queries
points_tree = spatial.cKDTree(points_coordinates)
features = {}
default_scale = 0.004
for shot_index, shot in reconstruction.shots.items():
# query all closest points
neighbors = list(
sorted(
points_tree.query_ball_point(shot.pose.get_origin(),
maximum_depth)))
# project them
projections = shot.project_many(
np.array([points_coordinates[c] for c in neighbors]))
# shot constants
center = shot.pose.get_origin()
z_axis = shot.pose.get_rotation_matrix()[2]
is_panorama = pygeometry.Camera.is_panorama(
shot.camera.projection_type)
perturbation = float(noise) / float(
max(shot.camera.width, shot.camera.height))
sigmas = np.array([perturbation, perturbation])
# pre-generate random perturbations
perturbations = np.random.normal(0.0, sigmas, (len(projections), 2))
# run and check valid projections
projections_inside = []
descriptors_inside = []
colors_inside = []
for i, (p_id, projection) in enumerate(zip(neighbors, projections)):
if not _is_inside_camera(projection, shot.camera):
continue
point = points_coordinates[p_id]
if not is_panorama and not _is_in_front(point, center, z_axis):
continue
# add perturbation
projection += perturbations[i]
# push data
color = points_colors[p_id]
original_id = points_ids[p_id]
projections_inside.append(
[projection[0], projection[1], default_scale])
descriptors_inside.append(track_descriptors[p_id])
colors_inside.append(color)
obs = pysfm.Observation(
projection[0],
projection[1],
default_scale,
color[0],
color[1],
color[2],
len(projections_inside) - 1,
)
tracks_manager.add_observation(str(shot_index), str(original_id),
obs)
features[shot_index] = oft.FeaturesData(
np.array(projections_inside),
np.array(descriptors_inside),
np.array(colors_inside),
None,
)
return features, tracks_manager