def match_images(
data: DataSetBase,
config_override: Dict[str, Any],
ref_images: List[str],
cand_images: List[str],
) -> Tuple[Dict[Tuple[str, str], List[Tuple[int, int]]], Dict[str, Any]]:
"""Perform pair matchings between two sets of images.
It will do matching for each pair (i, j), i being in
ref_images and j in cand_images, taking assumption that
matching(i, j) == matching(j ,i). This does not hold for
non-symmetric matching options like WORDS. Data will be
stored in i matching only.
"""
# Get EXIFs data
all_images = list(set(ref_images + cand_images))
exifs = {im: data.load_exif(im) for im in all_images}
# Generate pairs for matching
pairs, preport = pairs_selection.match_candidates_from_metadata(
ref_images,
cand_images,
exifs,
data,
config_override,
)
# Match them !
return (
match_images_with_pairs(data, config_override, exifs, pairs),
preport,
)
def _not_on_blackvue_watermark(p1: np.ndarray, p2: np.ndarray, matches,
im1: str, im2: str,
data: DataSetBase) -> List[Tuple[int, int]]:
"""Filter Blackvue's watermark."""
meta1 = data.load_exif(im1)
meta2 = data.load_exif(im2)
if meta1["make"].lower() == "blackvue":
matches = [m for m in matches if _blackvue_valid_mask(p1[m[0]])]
if meta2["make"].lower() == "blackvue":
matches = [m for m in matches if _blackvue_valid_mask(p2[m[1]])]
return matches
def is_high_res_panorama(
data: DataSetBase, image_key: str, image_array: np.ndarray
) -> bool:
"""Detect if image is a panorama."""
exif = data.load_exif(image_key)
if exif:
camera = data.load_camera_models()[exif["camera"]]
w, h = int(exif["width"]), int(exif["height"])
exif_pano = pygeometry.Camera.is_panorama(camera.projection_type)
elif image_array is not None:
h, w = image_array.shape[:2]
exif_pano = False
else:
return False
return w == 2 * h or exif_pano
def _not_on_vermont_watermark(
p1: np.ndarray,
p2: np.ndarray,
matches: List[Tuple[int, int]],
im1: str,
im2: str,
data: DataSetBase,
) -> List[Tuple[int, int]]:
"""Filter Vermont images watermark."""
meta1 = data.load_exif(im1)
meta2 = data.load_exif(im2)
if meta1["make"] == "VTrans_Camera" and meta1["model"] == "VTrans_Camera":
matches = [m for m in matches if _vermont_valid_mask(p1[m[0]])]
if meta2["make"] == "VTrans_Camera" and meta2["model"] == "VTrans_Camera":
matches = [m for m in matches if _vermont_valid_mask(p2[m[1]])]
return matches
def load_features_mask(
data: DataSetBase,
image: str,
points: np.ndarray,
mask_image: Optional[np.ndarray] = None,
) -> np.ndarray:
"""Load a feature-wise mask.
This is a binary array true for features that lie inside the
combined mask.
The array is all true when there's no mask.
"""
if points is None or len(points) == 0:
return np.array([], dtype=bool)
if mask_image is None:
mask_image = _load_combined_mask(data, image)
if mask_image is None:
logger.debug(
"No segmentation for {}, no features masked.".format(image))
return np.ones((points.shape[0], ), dtype=bool)
exif = data.load_exif(image)
width = exif["width"]
height = exif["height"]
orientation = exif["orientation"]
new_height, new_width = mask_image.shape
ps = upright.opensfm_to_upright(
points[:, :2],
width,
height,
orientation,
new_width=new_width,
new_height=new_height,
).astype(int)
mask = mask_image[ps[:, 1], ps[:, 0]]
n_removed = np.sum(mask == 0)
logger.debug("Masking {} / {} ({:.2f}) features for {}".format(
n_removed, len(mask), n_removed / len(mask), image))
return np.array(mask, dtype=bool)
def invent_reference_from_gps_and_gcp(
data: DataSetBase,
images: Optional[List[str]] = None) -> geo.TopocentricConverter:
lat, lon, alt = 0.0, 0.0, 0.0
wlat, wlon, walt = 0.0, 0.0, 0.0
if images is None:
images = data.images()
for image in images:
d = data.load_exif(image)
if "gps" in d and "latitude" in d["gps"] and "longitude" in d["gps"]:
w = 1.0 / max(0.01, d["gps"].get("dop", 15))
lat += w * d["gps"]["latitude"]
lon += w * d["gps"]["longitude"]
wlat += w
wlon += w
if "altitude" in d["gps"]:
alt += w * d["gps"]["altitude"]
walt += w
if not wlat and not wlon:
for gcp in data.load_ground_control_points():
lat += gcp.lla["latitude"]
lon += gcp.lla["longitude"]
wlat += 1
wlon += 1
if gcp.has_altitude:
alt += gcp.lla["altitude"]
walt += 1
if wlat:
lat /= wlat
if wlon:
lon /= wlon
if walt:
alt /= walt
return geo.TopocentricConverter(lat, lon, 0) # Set altitude manually.
def reconstruction_from_metadata(data: DataSetBase, images: Iterable[str]) -> types.Reconstruction:
"""Initialize a reconstruction by using EXIF data for constructing shot poses and cameras."""
data.init_reference()
rig_assignments = rig.rig_assignments_per_image(data.load_rig_assignments())
reconstruction = types.Reconstruction()
reconstruction.reference = data.load_reference()
reconstruction.cameras = data.load_camera_models()
for image in images:
camera_id = data.load_exif(image)["camera"]
if image in rig_assignments:
rig_instance_id, rig_camera_id, _ = rig_assignments[image]
else:
rig_instance_id = image
rig_camera_id = camera_id
reconstruction.add_rig_camera(pymap.RigCamera(pygeometry.Pose(), rig_camera_id))
reconstruction.add_rig_instance(pymap.RigInstance(rig_instance_id))
shot = reconstruction.create_shot(
shot_id=image,
camera_id=camera_id,
rig_camera_id=rig_camera_id,
rig_instance_id=rig_instance_id,
)
shot.metadata = get_image_metadata(data, image)
if not shot.metadata.gps_position.has_value:
reconstruction.remove_shot(image)
continue
gps_pos = shot.metadata.gps_position.value
shot.pose.set_rotation_matrix(rotation_from_shot_metadata(shot))
shot.pose.set_origin(gps_pos)
shot.scale = 1.0
return reconstruction
def run_dataset(data: DataSetBase):
"""Extract metadata from images' EXIF tag."""
exif_overrides = {}
if data.exif_overrides_exists():
exif_overrides = data.load_exif_overrides()
camera_models = {}
for image in data.images():
if data.exif_exists(image):
logging.info("Loading existing EXIF for {}".format(image))
d = data.load_exif(image)
else:
logging.info("Extracting EXIF for {}".format(image))
d = _extract_exif(image, data)
if image in exif_overrides:
d.update(exif_overrides[image])
data.save_exif(image, d)
if d["camera"] not in camera_models:
camera = exif.camera_from_exif_metadata(d, data)
camera_models[d["camera"]] = camera
# Override any camera specified in the camera models overrides file.
if data.camera_models_overrides_exists():
overrides = data.load_camera_models_overrides()
if "all" in overrides:
for key in camera_models:
camera_models[key] = copy.copy(overrides["all"])
camera_models[key].id = key
else:
for key, value in overrides.items():
camera_models[key] = value
data.save_camera_models(camera_models)
def detect(
image: str,
image_array: np.ndarray,
segmentation_array: Optional[np.ndarray],
instances_array: Optional[np.ndarray],
data: DataSetBase,
force: bool = False,
) -> None:
log.setup()
need_words = (
data.config["matcher_type"] == "WORDS"
or data.config["matching_bow_neighbors"] > 0
)
has_words = not need_words or data.words_exist(image)
has_features = data.features_exist(image)
if not force and has_features and has_words:
logger.info(
"Skip recomputing {} features for image {}".format(
data.feature_type().upper(), image
)
)
return
logger.info(
"Extracting {} features for image {}".format(data.feature_type().upper(), image)
)
start = timer()
p_unmasked, f_unmasked, c_unmasked = features.extract_features(
image_array, data.config, is_high_res_panorama(data, image, image_array)
)
# Load segmentation and bake it in the data
if data.config["features_bake_segmentation"]:
exif = data.load_exif(image)
s_unsorted, i_unsorted = bake_segmentation(
image_array, p_unmasked, segmentation_array, instances_array, exif
)
p_unsorted = p_unmasked
f_unsorted = f_unmasked
c_unsorted = c_unmasked
# Load segmentation, make a mask from it mask and apply it
else:
s_unsorted, i_unsorted = None, None
fmask = masking.load_features_mask(data, image, p_unmasked)
p_unsorted = p_unmasked[fmask]
f_unsorted = f_unmasked[fmask]
c_unsorted = c_unmasked[fmask]
if len(p_unsorted) == 0:
logger.warning("No features found in image {}".format(image))
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
if s_unsorted is not None:
semantic_data = features.SemanticData(
s_unsorted[order],
i_unsorted[order] if i_unsorted is not None else None,
data.segmentation_labels(),
)
else:
semantic_data = None
features_data = features.FeaturesData(p_sorted, f_sorted, c_sorted, semantic_data)
data.save_features(image, features_data)
if need_words:
bows = bow.load_bows(data.config)
n_closest = data.config["bow_words_to_match"]
closest_words = bows.map_to_words(
f_sorted, n_closest, data.config["bow_matcher_type"]
)
data.save_words(image, closest_words)
end = timer()
report = {
"image": image,
"num_features": len(p_sorted),
"wall_time": end - start,
}
data.save_report(io.json_dumps(report), "features/{}.json".format(image))
def get_image_metadata(data: DataSetBase, image: str) -> pymap.ShotMeasurements:
"""Get image metadata as a ShotMetadata object."""
exif = data.load_exif(image)
reference = data.load_reference()
return exif_to_metadata(exif, data.config["use_altitude_tag"], reference)
