def test_simple_trial_run_generated(self):
# TODO: The state of things:
# - With the configured sequence, the ORB-SLAM subprocess silently dies about frame 346-347
# - With my bastardised removal of the subprocess, it seems to work? and not crash?
# - I have no idea what is different? the pipe?
sequence_folder, left_path, right_path = ndds_loader.find_files(NDDS_SEQUENCE)
camera_intrinsics = ndds_loader.read_camera_intrinsics(left_path / '_camera_settings.json')
max_img_id = ndds_loader.find_max_img_id(lambda idx: left_path / ndds_loader.IMG_TEMPLATE.format(idx))
with (NDDS_SEQUENCE / 'timestamps.json').open('r') as fp:
timestamps = json.load(fp)
self.temp_folder.mkdir(parents=True, exist_ok=True)
path_manager = PathManager([VOCAB_PATH.parent], self.temp_folder)
subject = OrbSlam2(
vocabulary_file=str(VOCAB_PATH.name),
mode=SensorMode.STEREO,
vocabulary_branching_factor=5,
vocabulary_depth=6,
vocabulary_seed=0,
depth_threshold=387.0381720715473,
orb_num_features=598,
orb_scale_factor=np.power(480 / 104, 1 / 11), # = (480 / min_height)^(1/num_levels)
orb_num_levels=11,
orb_ini_threshold_fast=86,
orb_min_threshold_fast=48
)
subject.resolve_paths(path_manager)
subject.set_camera_intrinsics(camera_intrinsics, 0.1)
# Read the first frame data to get the baseline
left_frame_data = ndds_loader.read_json(left_path / ndds_loader.DATA_TEMPLATE.format(0))
right_frame_data = ndds_loader.read_json(right_path / ndds_loader.DATA_TEMPLATE.format(0))
left_camera_pose = ndds_loader.read_camera_pose(left_frame_data)
right_camera_pose = ndds_loader.read_camera_pose(right_frame_data)
subject.set_stereo_offset(left_camera_pose.find_relative(right_camera_pose))
subject.start_trial(ImageSequenceType.SEQUENTIAL, seed=0)
image_group = 'test'
with image_manager.get().get_group(image_group, allow_write=True):
for img_idx in range(max_img_id + 1):
left_pixels = image_utils.read_colour(left_path / ndds_loader.IMG_TEMPLATE.format(img_idx))
right_pixels = image_utils.read_colour(right_path / ndds_loader.IMG_TEMPLATE.format(img_idx))
image = StereoImage(
pixels=left_pixels,
right_pixels=right_pixels,
image_group=image_group,
metadata=imeta.ImageMetadata(camera_pose=left_camera_pose),
right_metadata=imeta.ImageMetadata(camera_pose=right_camera_pose)
)
subject.process_image(image, timestamps[img_idx])
result = subject.finish_trial()
self.assertIsInstance(result, SLAMTrialResult)
self.assertEqual(subject, result.system)
self.assertTrue(result.success)
self.assertFalse(result.has_scale)
self.assertIsNotNone(result.run_time)
self.assertEqual(max_img_id + 1, len(result.results))
def test_simple_trial_run_generated(self):
sequence_folder, left_path, right_path = ndds_loader.find_files(
NDDS_SEQUENCE)
camera_intrinsics = ndds_loader.read_camera_intrinsics(
left_path / '_camera_settings.json')
max_img_id = ndds_loader.find_max_img_id(
lambda idx: left_path / ndds_loader.IMG_TEMPLATE.format(idx))
with (NDDS_SEQUENCE / 'timestamps.json').open('r') as fp:
timestamps = json.load(fp)
subject = LibVisOMonoSystem(
matcher_nms_n=10,
matcher_nms_tau=66,
matcher_match_binsize=50,
matcher_match_radius=245,
matcher_match_disp_tolerance=2,
matcher_outlier_disp_tolerance=5,
matcher_outlier_flow_tolerance=2,
matcher_multi_stage=False,
matcher_half_resolution=False,
matcher_refinement=MatcherRefinement.SUBPIXEL,
bucketing_max_features=6,
bucketing_bucket_width=136,
bucketing_bucket_height=102,
height=1.0,
pitch=0.0,
ransac_iters=439,
inlier_threshold=4.921875,
motion_threshold=609.375)
subject.set_camera_intrinsics(camera_intrinsics, 0.1)
subject.start_trial(ImageSequenceType.SEQUENTIAL, seed=0)
image_group = 'test'
with image_manager.get().get_group(image_group, allow_write=True):
for img_idx in range(max_img_id + 1):
pixels = image_utils.read_colour(
left_path / ndds_loader.IMG_TEMPLATE.format(img_idx))
image = Image(
_id=bson.ObjectId(),
pixels=pixels,
image_group=image_group,
metadata=imeta.ImageMetadata(camera_pose=Transform()))
subject.process_image(image, timestamps[img_idx])
result = subject.finish_trial()
self.assertIsInstance(result, SLAMTrialResult)
self.assertEqual(subject, result.system)
self.assertTrue(result.success)
self.assertFalse(result.has_scale)
self.assertIsNotNone(result.run_time)
self.assertEqual(max_img_id + 1, len(result.results))
def import_sequence(root_folder: Path, left_path: Path, right_path: Path,
depth_quality: DepthNoiseQuality = DepthNoiseQuality.KINECT_NOISE) -> ImageCollection:
"""
Import the sequence, as a bunch of stereo images, and then organised into an ImageCollection.
ImageCollection and StereoImage objects are saved.
:param root_folder: The root folder to import from, containing the timestamps and settings files
:param left_path: The path to the left image sequences
:param right_path: The path to the right image sequences
:param depth_quality: Noisy depth is generated as we go, the quality to use when doing so
:return: The imported image collection
"""
# Read the timestamps and the generator settings
# These are saved from python, so need for special loading
with (root_folder / 'settings.json').open('r') as fp:
settings = json_load(fp)
with (root_folder / 'timestamps.json').open('r') as fp:
timestamps = json_load(fp)
# Read the camera settings from file
left_camera_intrinsics = read_camera_intrinsics(left_path / '_camera_settings.json')
right_camera_intrinsics = read_camera_intrinsics(right_path / '_camera_settings.json')
# left_object_labels = read_object_classes(left_path / '_object_settings.json')
# right_object_labels = read_object_classes(right_path / '_object_settings.json')
max_img_id = min(
find_max_img_id(lambda idx: left_path / IMG_TEMPLATE.format(idx)),
find_max_img_id(lambda idx: right_path / IMG_TEMPLATE.format(idx)),
)
if len(timestamps) != max_img_id + 1:
raise RuntimeError(f"Maximum image id {max_img_id} didn't match the number "
f"of available timestamps ({len(timestamps)}), cannot associate.")
# Read meta-information from the generator, including the timestamps
sequence_name = root_folder.name
(
trajectory_id, environment_type, light_level, time_of_day, simulation_world,
lighting_model, texture_mipmap_bias, normal_maps_enabled, roughness_enabled, min_object_size,
geometry_decimation
) = parse_settings(settings)
# Import all the images
images = []
image_group = sequence_name
origin = None
# Open the image manager for writing once, so that we're not constantly opening and closing it with each image
with arvet.database.image_manager.get().get_group(image_group, allow_write=True):
for img_idx in range(max_img_id + 1):
logging.getLogger(__name__).info(f"Loading image {img_idx}...")
# Read the raw data for the left image
left_frame_data = read_json(left_path / DATA_TEMPLATE.format(img_idx))
left_pixels = image_utils.read_colour(left_path / IMG_TEMPLATE.format(img_idx))
# left_label_image = image_utils.read_colour(left_path / INSTANCE_TEMPLATE.format(img_idx))
left_true_depth = load_depth_image(left_path / DEPTH_TEMPLATE.format(img_idx))
# Read the raw data for the right image
right_frame_data = read_json(right_path / DATA_TEMPLATE.format(img_idx))
right_pixels = image_utils.read_colour(right_path / IMG_TEMPLATE.format(img_idx))
# right_label_image = image_utils.read_colour(right_path / INSTANCE_TEMPLATE.format(img_idx))
right_true_depth = load_depth_image(right_path / DEPTH_TEMPLATE.format(img_idx))
# Ensure all images are c_contiguous
if not left_pixels.flags.c_contiguous:
left_pixels = np.ascontiguousarray(left_pixels)
# if not left_label_image.flags.c_contiguous:
# left_label_image = np.ascontiguousarray(left_label_image)
if not left_true_depth.flags.c_contiguous:
left_true_depth = np.ascontiguousarray(left_true_depth)
if not right_pixels.flags.c_contiguous:
right_pixels = np.ascontiguousarray(right_pixels)
# if not right_label_image.flags.c_contiguous:
# right_label_image = np.ascontiguousarray(right_label_image)
if not right_true_depth.flags.c_contiguous:
right_true_depth = np.ascontiguousarray(right_true_depth)
# Extract the poses
left_camera_pose = read_camera_pose(left_frame_data)
right_camera_pose = read_camera_pose(right_frame_data)
# If we have object data, extract labels for them as well
# Not working? removed
# if len(left_object_labels) > 0:
# left_labelled_objects = find_labelled_objects(left_label_image, left_frame_data, left_object_labels)
# else:
# left_labelled_objects = []
# if len(right_object_labels) > 0:
# right_labelled_objects = find_labelled_objects(right_label_image, right_frame_data, right_object_labels)
# else:
# right_labelled_objects = []
left_labelled_objects = []
right_labelled_objects = []
# Compute a noisy depth image
noisy_depth = create_noisy_depth_image(
left_true_depth=left_true_depth,
right_true_depth=right_true_depth,
camera_intrinsics=left_camera_intrinsics,
right_camera_relative_pose=left_camera_pose.find_relative(right_camera_pose),
quality_level=depth_quality
)
# Re-centre the camera poses relative to the first frame
if origin is None:
origin = left_camera_pose
left_camera_pose = origin.find_relative(left_camera_pose)
right_camera_pose = origin.find_relative(right_camera_pose)
left_metadata = imeta.make_metadata(
pixels=left_pixels,
depth=left_true_depth,
camera_pose=left_camera_pose,
intrinsics=left_camera_intrinsics,
source_type=imeta.ImageSourceType.SYNTHETIC,
environment_type=environment_type,
light_level=light_level,
time_of_day=time_of_day,
simulation_world=simulation_world,
lighting_model=lighting_model,
texture_mipmap_bias=texture_mipmap_bias,
normal_maps_enabled=normal_maps_enabled,
roughness_enabled=roughness_enabled,
geometry_decimation=geometry_decimation,
minimum_object_volume=min_object_size,
labelled_objects=left_labelled_objects
)
right_metadata = imeta.make_right_metadata(
pixels=right_pixels,
depth=right_true_depth,
camera_pose=right_camera_pose,
intrinsics=right_camera_intrinsics,
labelled_objects=right_labelled_objects,
left_metadata=left_metadata
)
image = StereoImage(
pixels=left_pixels,
right_pixels=right_pixels,
depth=noisy_depth,
true_depth=left_true_depth,
right_true_depth=right_true_depth,
image_group=image_group,
metadata=left_metadata,
right_metadata=right_metadata,
)
try:
image.save()
except KeyError as exp:
logging.getLogger(__name__).info(f"Key error while saving image {img_idx} in sequence {sequence_name}, "
f"read from {left_path / IMG_TEMPLATE.format(img_idx)}")
raise exp
images.append(image)
# Create and save the image collection
collection = ImageCollection(
images=images,
image_group=sequence_name,
timestamps=timestamps,
sequence_type=ImageSequenceType.SEQUENTIAL,
dataset="generated-SLAM-data",
sequence_name=sequence_name,
trajectory_id=trajectory_id
)
collection.save()
return collection
def verify_sequence(image_collection: ImageCollection,
root_folder: Path) -> bool:
"""
Load a dataset produced by the Nvidia dataset generator
:return:
"""
root_folder = Path(root_folder)
# depth_quality = DepthNoiseQuality(depth_quality)
valid = True
sequence_name = image_collection.sequence_name
# Step 0: Check the root folder to see if it needs to be extracted from a tarfile
sequence_folder = root_folder / sequence_name
delete_when_done = None
if not sequence_folder.is_dir():
sequence_tarfile = root_folder / (sequence_name + '.tar.gz')
if sequence_tarfile.is_file() and tarfile.is_tarfile(sequence_tarfile):
logging.getLogger(__name__).info(
f"Extracting sequence from {sequence_tarfile}")
delete_when_done = sequence_folder
with tarfile.open(sequence_tarfile) as tar_fp:
tar_fp.extractall(sequence_folder)
else:
# Could find neither a dir nor a tarfile to extract from
raise NotADirectoryError(
f"Neither {sequence_folder} nor {sequence_tarfile} exists for us to extract"
)
sequence_folder, left_path, right_path = ndds_loader.find_files(
sequence_folder)
logging.getLogger(__name__).info(
f"{sequence_name}: Selected {sequence_folder}, {left_path}, {right_path} as source folders"
)
# Read the maximum image id, and make sure it matches the collection
max_img_id = min(
ndds_loader.find_max_img_id(
lambda idx: left_path / ndds_loader.IMG_TEMPLATE.format(idx)),
ndds_loader.find_max_img_id(
lambda idx: right_path / ndds_loader.IMG_TEMPLATE.format(idx)),
)
if len(image_collection) != max_img_id + 1:
logging.getLogger(__name__).warning(
f"Maximum image id {max_img_id} did not match the length of the "
f"image collection ({len(image_collection)})")
# make sure we don't try and get images from the collection it doesn't have
max_img_id = min(max_img_id, len(image_collection) - 1)
valid = False
# Verify the images
# Open the image manager for writing once, so that we're not constantly opening and closing it with each image
total_invalid_images = 0
with arvet.database.image_manager.get().get_group(
image_collection.get_image_group()):
for img_idx in range(max_img_id + 1):
img_valid = True
# Expand the file paths for this image
left_img_path = left_path / ndds_loader.IMG_TEMPLATE.format(
img_idx)
left_depth_path = left_path / ndds_loader.DEPTH_TEMPLATE.format(
img_idx)
right_img_path = right_path / ndds_loader.IMG_TEMPLATE.format(
img_idx)
right_depth_path = right_path / ndds_loader.DEPTH_TEMPLATE.format(
img_idx)
# Read the raw data for the left image
left_pixels = image_utils.read_colour(left_img_path)
left_true_depth = ndds_loader.load_depth_image(left_depth_path)
# Read the raw data for the right image
right_pixels = image_utils.read_colour(right_img_path)
right_true_depth = ndds_loader.load_depth_image(right_depth_path)
# Ensure all images are c_contiguous
if not left_pixels.flags.c_contiguous:
left_pixels = np.ascontiguousarray(left_pixels)
if not left_true_depth.flags.c_contiguous:
left_true_depth = np.ascontiguousarray(left_true_depth)
if not right_pixels.flags.c_contiguous:
right_pixels = np.ascontiguousarray(right_pixels)
if not right_true_depth.flags.c_contiguous:
right_true_depth = np.ascontiguousarray(right_true_depth)
# Compute image hashes
left_hash = bytes(xxhash.xxh64(left_pixels).digest())
right_hash = bytes(xxhash.xxh64(right_pixels).digest())
# Compute a noisy depth image
# noisy_depth = create_noisy_depth_image(
# left_ground_truth_depth=left_ground_truth_depth,
# right_ground_truth_depth=right_ground_truth_depth,
# camera_intrinsics=left_camera_intrinsics,
# right_camera_relative_pose=left_camera_pose.find_relative(right_camera_pose),
# quality_level=depth_quality
# )
# Load the image from the database
try:
_, image = image_collection[img_idx]
left_actual_pixels = image.left_pixels
left_actual_ground_truth_depth = image.left_true_depth
right_actual_pixels = image.right_pixels
right_actual_ground_truth_depth = image.right_true_depth
except (KeyError, IOError, RuntimeError):
logging.getLogger(__name__).exception(
f"Error loading image {img_idx}")
valid = False
continue
# Compare the loaded image data to the data read from disk
if not np.array_equal(left_pixels, left_actual_pixels):
logging.getLogger(__name__).error(
f"Image {img_idx}: Left pixels do not match data read from {left_img_path}"
)
total_invalid_images += 1
valid = False
if left_hash != bytes(image.metadata.img_hash):
logging.getLogger(__name__).error(
f"Image {img_idx}: Left hash does not match metadata {image.metadata.img_hash}"
)
valid = False
img_valid = False
if not np.array_equal(left_true_depth,
left_actual_ground_truth_depth):
logging.getLogger(__name__).error(
f"Image {img_idx}: Left depth does not match data read from {left_depth_path}"
)
valid = False
img_valid = False
if not np.array_equal(right_pixels, right_actual_pixels):
logging.getLogger(__name__).error(
f"Image {img_idx}: Right pixels do not match data read from {right_img_path}"
)
valid = False
img_valid = False
if right_hash != bytes(image.right_metadata.img_hash):
logging.getLogger(__name__).error(
f"Image {img_idx}: Right hash does not match metadata {image.right_metadata.img_hash}"
)
valid = False
img_valid = False
if not np.array_equal(right_true_depth,
right_actual_ground_truth_depth):
logging.getLogger(__name__).error(
f"Image {img_idx}: Right depth does not match data read from {right_depth_path}"
)
valid = False
img_valid = False
if not img_valid:
total_invalid_images += 1
if delete_when_done is not None and delete_when_done.exists():
# We're done and need to clean up after ourselves
shutil.rmtree(delete_when_done)
if valid:
logging.getLogger(__name__).info(
f"Verification of {sequence_name} successful.")
else:
logging.getLogger(__name__).info(
f"Verification of {sequence_name} ({image_collection.pk}) "
f"FAILED, ({total_invalid_images} images failed)")
return valid
def import_dataset(root_folder, dataset_name, **_):
"""
Load a TUM RGB-D sequence into the database.
:return:
"""
root_folder = Path(root_folder)
# Step 0: Check the root folder to see if it needs to be extracted from a tarfile
delete_when_done = None
if not root_folder.is_dir():
if (root_folder.parent / dataset_name).is_dir():
# The root was a tarball, but the extracted data already exists, just use that as the root
root_folder = root_folder.parent / dataset_name
else:
candidate_tar_file = root_folder.parent / (dataset_name + '.tgz')
if candidate_tar_file.is_file() and tarfile.is_tarfile(
candidate_tar_file):
# Root is actually a tarfile, extract it. find_roots with handle folder structures
with tarfile.open(candidate_tar_file) as tar_fp:
tar_fp.extractall(root_folder.parent / dataset_name)
root_folder = root_folder.parent / dataset_name
delete_when_done = root_folder
else:
# Could find neither a dir nor a tarfile to extract from
raise NotADirectoryError(
"'{0}' is not a directory".format(root_folder))
# Step 1: Find the relevant metadata files
root_folder, rgb_path, depth_path, trajectory_path = find_files(
root_folder)
# Step 2: Read the metadata from them
image_files = read_image_filenames(rgb_path)
trajectory = read_trajectory(trajectory_path, image_files.keys())
depth_files = read_image_filenames(depth_path)
# Step 3: Associate the different data types by timestamp
all_metadata = associate_data(image_files, trajectory, depth_files)
# Step 3: Load the images from the metadata
first_timestamp = None
image_group = dataset_name
images = []
timestamps = []
with arvet.database.image_manager.get().get_group(image_group,
allow_write=True):
for timestamp, image_file, camera_pose, depth_file in all_metadata:
# Re-zero the timestamps
if first_timestamp is None:
first_timestamp = timestamp
timestamp = (timestamp - first_timestamp)
rgb_data = image_utils.read_colour(
os.path.join(root_folder, image_file))
depth_data = image_utils.read_depth(
os.path.join(root_folder, depth_file))
depth_data = depth_data / 5000 # Re-scale depth to meters
camera_intrinsics = get_camera_intrinsics(root_folder)
metadata = imeta.make_metadata(
pixels=rgb_data,
depth=depth_data,
camera_pose=camera_pose,
intrinsics=camera_intrinsics,
source_type=imeta.ImageSourceType.REAL_WORLD,
environment_type=environment_types.get(
dataset_name, imeta.EnvironmentType.INDOOR_CLOSE),
light_level=imeta.LightingLevel.WELL_LIT,
time_of_day=imeta.TimeOfDay.DAY,
)
image = Image(pixels=rgb_data,
depth=depth_data,
image_group=image_group,
metadata=metadata)
image.save()
images.append(image)
timestamps.append(timestamp)
# Create and save the image collection
collection = ImageCollection(images=images,
timestamps=timestamps,
sequence_type=ImageSequenceType.SEQUENTIAL,
dataset='TUM RGB-D',
sequence_name=dataset_name,
trajectory_id=dataset_name)
collection.save()
if delete_when_done is not None and delete_when_done.exists():
# We're done and need to clean up after ourselves
shutil.rmtree(delete_when_done)
return collection
def verify_dataset(image_collection: ImageCollection,
root_folder: typing.Union[str, Path],
dataset_name: str,
repair: bool = False):
"""
Load a TUM RGB-D sequence into the database.
:return:
"""
root_folder = Path(root_folder)
dataset_name = str(dataset_name)
repair = bool(repair)
valid = True
irreparable = False
image_group = dataset_name
# Check the root folder to see if it needs to be extracted from a tarfile
delete_when_done = None
if not root_folder.is_dir():
if (root_folder.parent / dataset_name).is_dir():
# The root was a tarball, but the extracted data already exists, just use that as the root
root_folder = root_folder.parent / dataset_name
else:
candidate_tar_file = root_folder.parent / (dataset_name + '.tgz')
if candidate_tar_file.is_file() and tarfile.is_tarfile(
candidate_tar_file):
# Root is actually a tarfile, extract it. find_roots with handle folder structures
with tarfile.open(candidate_tar_file) as tar_fp:
tar_fp.extractall(root_folder.parent / dataset_name)
root_folder = root_folder.parent / dataset_name
delete_when_done = root_folder
else:
# Could find neither a dir nor a tarfile to extract from
raise NotADirectoryError(
"'{0}' is not a directory".format(root_folder))
# Check the image group on the image collection
if image_collection.image_group != image_group:
if repair:
image_collection.image_group = image_group
image_collection.save()
logging.getLogger(__name__).info(
f"Fixed incorrect image group for {image_collection.sequence_name}"
)
else:
logging.getLogger(__name__).warning(
f"{image_collection.sequence_name} has incorrect image group {image_collection.image_group}"
)
valid = False
# Find the relevant metadata files
root_folder, rgb_path, depth_path, trajectory_path = tum_loader.find_files(
root_folder)
# Step 2: Read the metadata from them
image_files = tum_loader.read_image_filenames(rgb_path)
trajectory = tum_loader.read_trajectory(trajectory_path,
image_files.keys())
depth_files = tum_loader.read_image_filenames(depth_path)
# Step 3: Associate the different data types by timestamp
all_metadata = tum_loader.associate_data(image_files, trajectory,
depth_files)
# Step 3: Load the images from the metadata
total_invalid_images = 0
total_fixed_images = 0
with arvet.database.image_manager.get().get_group(image_group,
allow_write=repair):
for img_idx, (timestamp, image_file, camera_pose,
depth_file) in enumerate(all_metadata):
changed = False
img_valid = True
img_path = root_folder / image_file
depth_path = root_folder / depth_file
rgb_data = image_utils.read_colour(img_path)
depth_data = image_utils.read_depth(depth_path)
depth_data = depth_data / 5000 # Re-scale depth to meters
img_hash = bytes(xxhash.xxh64(rgb_data).digest())
# Load the image from the database
try:
_, image = image_collection[img_idx]
except (KeyError, IOError, RuntimeError):
logging.getLogger(__name__).exception(
f"Error loading image object {img_idx}")
valid = False
total_invalid_images += 1
continue
# First, check the image group
if image.image_group != image_group:
if repair:
image.image_group = image_group
changed = True
logging.getLogger(__name__).warning(
f"Image {img_idx} has incorrect group {image.image_group}")
valid = False
img_valid = False
# Load the pixels from the image
try:
actual_pixels = image.pixels
except (KeyError, IOError, RuntimeError):
actual_pixels = None
try:
actual_depth = image.depth
except (KeyError, IOError, RuntimeError):
actual_depth = None
# Compare the loaded image data to the data read from disk
if actual_pixels is None or not np.array_equal(
rgb_data, actual_pixels):
if repair:
image.store_pixels(rgb_data)
changed = True
else:
logging.getLogger(__name__).error(
f"Image {img_idx}: Pixels do not match data read from {img_path}"
)
valid = False
img_valid = False
if img_hash != bytes(image.metadata.img_hash):
if repair:
image.metadata.img_hash = img_hash
changed = True
else:
logging.getLogger(__name__).error(
f"Image {img_idx}: Image hash does not match metadata")
valid = False
img_valid = False
if actual_depth is None or not np.array_equal(
depth_data, actual_depth):
if repair:
image.store_depth(depth_data)
changed = True
else:
logging.getLogger(__name__).error(
f"Image {img_idx}: Depth does not match data read from {depth_path}"
)
valid = False
img_valid = False
if changed and repair:
logging.getLogger(__name__).warning(
f"Image {img_idx}: repaired")
image.save()
total_fixed_images += 1
if not img_valid:
total_invalid_images += 1
if irreparable:
# Images are missing entirely, needs re-import
logging.getLogger(__name__).error(
f"Image Collection {image_collection.pk} for sequence {dataset_name} "
"is IRREPARABLE, invalidate and re-import")
elif repair:
# Re-save the modified image collection
logging.getLogger(__name__).info(
f"{image_collection.sequence_name} repaired successfully "
f"({total_fixed_images} image files fixed).")
elif valid:
logging.getLogger(__name__).info(
f"Verification of {image_collection.sequence_name} successful.")
else:
logging.getLogger(__name__).error(
f"Verification of {image_collection.sequence_name} ({image_collection.pk}) "
f"FAILED, ({total_invalid_images} images failed)")
if delete_when_done is not None and delete_when_done.exists():
# We're done and need to clean up after ourselves
shutil.rmtree(delete_when_done)
return valid
def verify_dataset(image_collection: ImageCollection,
root_folder: typing.Union[str, PurePath],
dataset_name: str,
repair: bool = False):
"""
Examine an existing Autonomous Systems Lab dataset in the database, and check if for errors
See http://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets#downloads
Some information drawn from the ethz_asl dataset tools, see: https://github.com/ethz-asl/dataset_tools
:param image_collection: The existing image collection from the loader
:param root_folder: The body folder, containing body.yaml (i.e. the extracted mav0 folder)
:param dataset_name: The name of the dataset, see the manager for the list of valid values.
:param repair: If possible, fix missing images in the dataset
:return:
"""
root_folder = Path(root_folder)
dataset_name = str(dataset_name)
repair = bool(repair)
if not root_folder.is_dir():
raise NotADirectoryError(
"'{0}' is not a directory".format(root_folder))
image_group = dataset_name
valid = True
irreparable = False
# Check the image group on the image collection
if image_collection.image_group != image_group:
if repair:
image_collection.image_group = image_group
image_collection.save()
logging.getLogger(__name__).info(
f"Fixed incorrect image group for {image_collection.sequence_name}"
)
else:
logging.getLogger(__name__).warning(
f"{image_collection.sequence_name} has incorrect image group {image_group}"
)
valid = False
# Find the various files containing the data (that's a 6-element tuple unpack for the return value)
(root_folder, left_rgb_path, left_camera_intrinsics_path, right_rgb_path,
right_camera_intrinsics_path,
trajectory_path) = euroc_loader.find_files(root_folder)
# Read the meta-information from the files (that's a 6-element tuple unpack for the return value)
left_image_files = euroc_loader.read_image_filenames(left_rgb_path)
left_extrinsics, left_intrinsics = euroc_loader.get_camera_calibration(
left_camera_intrinsics_path)
right_image_files = euroc_loader.read_image_filenames(left_rgb_path)
right_extrinsics, right_intrinsics = euroc_loader.get_camera_calibration(
right_camera_intrinsics_path)
# Create stereo rectification matrices from the intrinsics
left_x, left_y, left_intrinsics, right_x, right_y, right_intrinsics = euroc_loader.rectify(
left_extrinsics, left_intrinsics, right_extrinsics, right_intrinsics)
# Associate the different data types by timestamp. Trajectory last because it's bigger than the stereo.
all_metadata = euroc_loader.associate_data(left_image_files,
right_image_files)
# Load the images from the metadata
total_invalid_images = 0
total_fixed_images = 0
image_index = 0
with arvet.database.image_manager.get().get_group(image_group,
allow_write=repair):
for timestamp, left_image_file, right_image_file in all_metadata:
changed = False
img_valid = True
# Skip if we've hit the end of the data
if image_index >= len(image_collection):
logging.getLogger(__name__).error(
f"Image {image_index} is missing from the dataset")
irreparable = True
valid = False
total_invalid_images += 1
continue
left_img_path = root_folder / 'cam0' / 'data' / left_image_file
right_img_path = root_folder / 'cam1' / 'data' / right_image_file
left_pixels = image_utils.read_colour(left_img_path)
right_pixels = image_utils.read_colour(right_img_path)
# Error check the loaded image data
# The EuRoC Sequences MH_04_difficult and V2_03_difficult are missing the first right frame
# So we actually start loading from
# In general, frames that are missing are skipped, and do not increment image index
if left_pixels is None or left_pixels.size is 0:
logging.getLogger(__name__).warning(
f"Could not read left image \"{left_img_path}\", result is empty. Image is skipped."
)
continue
if right_pixels is None or right_pixels.size is 0:
logging.getLogger(__name__).warning(
f"Could not read right image \"{right_img_path}\", result is empty. Image is skipped."
)
continue
left_pixels = cv2.remap(left_pixels, left_x, left_y,
cv2.INTER_LINEAR)
right_pixels = cv2.remap(right_pixels, right_x, right_y,
cv2.INTER_LINEAR)
left_hash = bytes(xxhash.xxh64(left_pixels).digest())
right_hash = bytes(xxhash.xxh64(right_pixels).digest())
# Load the image from the database
try:
_, image = image_collection[image_index]
except (KeyError, IOError, RuntimeError):
logging.getLogger(__name__).exception(
f"Error loading image object {image_index}")
valid = False
image_index += 1 # Index is valid, increment when done
continue
# First, check the image group
if image.image_group != image_group:
if repair:
image.image_group = image_group
changed = True
logging.getLogger(__name__).warning(
f"Image {image_index} has incorrect group {image.image_group}"
)
valid = False
img_valid = False
# Load the pixels from the image
try:
left_actual_pixels = image.left_pixels
except (KeyError, IOError, RuntimeError):
left_actual_pixels = None
try:
right_actual_pixels = image.right_pixels
except (KeyError, IOError, RuntimeError):
right_actual_pixels = None
# Compare the loaded image data to the data read from disk
if left_actual_pixels is None or not np.array_equal(
left_pixels, left_actual_pixels):
if repair:
image.store_pixels(left_pixels)
changed = True
else:
logging.getLogger(__name__).error(
f"Image {image_index}: Left pixels do not match data read from {left_img_path}"
)
img_valid = False
valid = False
if left_hash != bytes(image.metadata.img_hash):
if repair:
image.metadata.img_hash = left_hash
changed = True
else:
logging.getLogger(__name__).error(
f"Image {image_index}: Left hash does not match metadata"
)
valid = False
img_valid = False
if right_actual_pixels is None or not np.array_equal(
right_pixels, right_actual_pixels):
if repair:
image.store_right_pixels(right_pixels)
changed = True
else:
logging.getLogger(__name__).error(
f"Image {image_index}: Right pixels do not match data read from {right_img_path}"
)
valid = False
img_valid = False
if right_hash != bytes(image.right_metadata.img_hash):
if repair:
image.right_metadata.img_hash = right_hash
changed = True
else:
logging.getLogger(__name__).error(
f"Image {image_index}: Right hash does not match metadata"
)
valid = False
img_valid = False
if changed and repair:
logging.getLogger(__name__).warning(
f"Image {image_index}: repaired")
image.save()
total_fixed_images += 1
if not img_valid:
total_invalid_images += 1
image_index += 1
if irreparable:
# Images are missing entirely, needs re-import
logging.getLogger(__name__).error(
f"Image Collection {image_collection.pk} for sequence {dataset_name} "
"is IRREPARABLE, invalidate and re-import")
elif repair:
# Re-save the modified image collection
logging.getLogger(__name__).info(
f"{image_collection.sequence_name} repaired successfully "
f"({total_fixed_images} image files fixed).")
elif valid:
logging.getLogger(__name__).info(
f"Verification of {image_collection.sequence_name} successful.")
else:
logging.getLogger(__name__).error(
f"Verification of {image_collection.sequence_name} ({image_collection.pk}) "
f"FAILED, ({total_invalid_images} images failed)")
return valid
def import_dataset(root_folder, dataset_name, **_):
"""
Load an Autonomous Systems Lab dataset into the database.
See http://projects.asl.ethz.ch/datasets/doku.php?id=kmavvisualinertialdatasets#downloads
Some information drawn from the ethz_asl dataset tools, see: https://github.com/ethz-asl/dataset_tools
:param root_folder: The body folder, containing body.yaml (i.e. the extracted mav0 folder)
:param dataset_name: The name of the dataset, see the manager for the list of valid values.
:return:
"""
if not os.path.isdir(root_folder):
raise NotADirectoryError(
"'{0}' is not a directory".format(root_folder))
# Step 1: Find the various files containing the data (that's a 6-element tuple unpack for the return value)
(root_folder, left_rgb_path, left_camera_intrinsics_path, right_rgb_path,
right_camera_intrinsics_path, trajectory_path) = find_files(root_folder)
# Step 2: Read the meta-information from the files (that's a 6-element tuple unpack for the return value)
left_image_files = read_image_filenames(left_rgb_path)
left_extrinsics, left_intrinsics = get_camera_calibration(
left_camera_intrinsics_path)
right_image_files = read_image_filenames(left_rgb_path)
right_extrinsics, right_intrinsics = get_camera_calibration(
right_camera_intrinsics_path)
trajectory = read_trajectory(trajectory_path, left_image_files.keys())
# Step 3: Create stereo rectification matrices from the intrinsics
left_x, left_y, left_intrinsics, right_x, right_y, right_intrinsics = rectify(
left_extrinsics, left_intrinsics, right_extrinsics, right_intrinsics)
# Change the coordinates correctly on the extrinsics. Has to happen after rectification
left_extrinsics = fix_coordinates(left_extrinsics)
right_extrinsics = fix_coordinates(right_extrinsics)
# Step 4: Associate the different data types by timestamp. Trajectory last because it's bigger than the stereo.
all_metadata = associate_data(left_image_files, right_image_files,
trajectory)
# Step 5: Load the images from the metadata
first_timestamp = None
image_group = dataset_name
images = []
timestamps = []
with arvet.database.image_manager.get().get_group(image_group,
allow_write=True):
for timestamp, left_image_file, right_image_file, robot_pose in all_metadata:
# Timestamps are in POSIX nanoseconds, re-zero them to the start of the dataset, and scale to seconds
if first_timestamp is None:
first_timestamp = timestamp
timestamp = (timestamp - first_timestamp) / 1e9
left_data = image_utils.read_colour(
os.path.join(root_folder, 'cam0', 'data', left_image_file))
right_data = image_utils.read_colour(
os.path.join(root_folder, 'cam1', 'data', right_image_file))
# Error check the loaded image data
if left_data is None or left_data.size is 0:
logging.getLogger(__name__).warning(
"Could not read left image \"{0}\", result is empty. Skipping."
.format(
os.path.join(root_folder, 'cam0', 'data',
left_image_file)))
continue
if right_data is None or right_data.size is 0:
logging.getLogger(__name__).warning(
"Could not read right image \"{0}\", result is empty. Skipping."
.format(
os.path.join(root_folder, 'cam1', 'data',
right_image_file)))
continue
left_data = cv2.remap(left_data, left_x, left_y, cv2.INTER_LINEAR)
right_data = cv2.remap(right_data, right_x, right_y,
cv2.INTER_LINEAR)
left_pose = robot_pose.find_independent(left_extrinsics)
right_pose = robot_pose.find_independent(right_extrinsics)
left_metadata = imeta.make_metadata(
pixels=left_data,
camera_pose=left_pose,
intrinsics=left_intrinsics,
source_type=imeta.ImageSourceType.REAL_WORLD,
environment_type=environment_types.get(
dataset_name, imeta.EnvironmentType.INDOOR_CLOSE),
light_level=imeta.LightingLevel.WELL_LIT,
time_of_day=imeta.TimeOfDay.DAY,
)
right_metadata = imeta.make_right_metadata(
pixels=right_data,
left_metadata=left_metadata,
camera_pose=right_pose,
intrinsics=right_intrinsics)
image = StereoImage(pixels=left_data,
right_pixels=right_data,
image_group=image_group,
metadata=left_metadata,
right_metadata=right_metadata)
image.save()
images.append(image)
timestamps.append(timestamp)
# Create and save the image collection
collection = ImageCollection(images=images,
timestamps=timestamps,
sequence_type=ImageSequenceType.SEQUENTIAL,
dataset='EuRoC MAV',
sequence_name=dataset_name,
trajectory_id=dataset_name)
collection.save()
return collection
