def test_validate_data_files_label():
"""
Test the validate_data_files functions
that looks for inconsistencies in the fixed/moving image and label lists.
If there is any issue it will raise an error, otherwise it returns None.
"""
for key_file_loader, file_loader in FileLoaderDict.items():
for split in ["train", "test"]:
data_dir_path = [join(DataPaths[key_file_loader], split)]
common_args = dict(
file_loader=file_loader,
labeled=True,
sample_label="all",
seed=None if split == "train" else 0,
)
data_loader = PairedDataLoader(
data_dir_paths=data_dir_path,
fixed_image_shape=fixed_image_shape,
moving_image_shape=moving_image_shape,
**common_args,
)
assert data_loader.validate_data_files() is None
data_loader.close()
def test_close():
"""
Test the close function. Only needed for H5 data loaders for now.
Since fixed/moving loaders are the same for
unpaired data loader, only need to test the moving.
"""
for key_file_loader, file_loader in FileLoaderDict.items():
for split in ["train", "test"]:
data_dir_path = [join(DataPaths[key_file_loader], split)]
common_args = dict(
file_loader=file_loader,
labeled=True,
sample_label="all",
seed=None if split == "train" else 0,
)
data_loader = PairedDataLoader(
data_dir_paths=data_dir_path,
fixed_image_shape=fixed_image_shape,
moving_image_shape=moving_image_shape,
**common_args,
)
if key_file_loader == "h5":
data_loader.close()
for f in data_loader.loader_moving_image.h5_files.values():
assert not f.__bool__()
def test_file_loader_init():
"""
check file loader is correctly called in __init__:
"""
loader = PairedDataLoader(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
file_loader = H5FileLoader(dir_path=data_dir_path,
name="moving_images",
grouped=False)
expected = ["case000025.nii.gz"]
loader_got = loader.loader_moving_image.get_data_ids()
file_loader_got = file_loader.get_data_ids()
loader.close()
file_loader.close()
assert loader_got == expected, "paired_loader has loaded incorrect moving image"
assert loader_got == file_loader_got, "paired_loader incorrectly calling h5_loader"
def test_get_dataset_and_preprocess(self, labeled, moving_shape,
fixed_shape, batch_size,
data_augmentation):
"""
Test get_transforms() function. For that, an Abstract Data Loader is created
only to set the moving and fixed shapes that are used in get_transforms().
Here we test that the get_transform() returns a function and the shape of
the output of this function. See test_preprocess.py for more testing regarding
the concrete params.
:param labeled: bool
:param moving_shape: tuple
:param fixed_shape: tuple
:param batch_size: int
:param data_augmentation: dict
:return:
"""
data_dir_path = [
"data/test/nifti/paired/train",
"data/test/nifti/paired/test",
]
common_args = dict(file_loader=NiftiFileLoader,
labeled=True,
sample_label="all",
seed=None)
data_loader = PairedDataLoader(
data_dir_paths=data_dir_path,
fixed_image_shape=fixed_shape,
moving_image_shape=moving_shape,
**common_args,
)
dataset = data_loader.get_dataset_and_preprocess(
training=True,
batch_size=batch_size,
repeat=True,
shuffle_buffer_num_batch=1,
**data_augmentation,
)
for outputs in dataset.take(1):
assert (outputs["moving_image"].shape == (batch_size, ) +
data_loader.moving_image_shape)
assert (outputs["fixed_image"].shape == (batch_size, ) +
data_loader.fixed_image_shape)
assert (outputs["moving_label"].shape == (batch_size, ) +
data_loader.moving_image_shape)
assert (outputs["fixed_label"].shape == (batch_size, ) +
data_loader.fixed_image_shape)
def get_single_data_loader(data_type, data_config, common_args, data_dir_path):
if data_type == "paired":
moving_image_shape = data_config["moving_image_shape"]
fixed_image_shape = data_config["fixed_image_shape"]
return PairedDataLoader(
data_dir_path=data_dir_path,
moving_image_shape=moving_image_shape,
fixed_image_shape=fixed_image_shape,
**common_args,
)
elif data_type == "grouped":
image_shape = data_config["image_shape"]
intra_group_prob = data_config["intra_group_prob"]
intra_group_option = data_config["intra_group_option"]
sample_image_in_group = data_config["sample_image_in_group"]
return GroupedDataLoader(
data_dir_path=data_dir_path,
intra_group_prob=intra_group_prob,
intra_group_option=intra_group_option,
sample_image_in_group=sample_image_in_group,
image_shape=image_shape,
**common_args,
)
elif data_type == "unpaired":
image_shape = data_config["image_shape"]
return UnpairedDataLoader(
data_dir_path=data_dir_path, image_shape=image_shape, **common_args
)
else:
raise ValueError(
"Unknown data format. "
"Supported types are paired, unpaired, and grouped, got {}\n".format(
data_type
)
)
def test_init():
"""
Check that data loader __init__() method is correct:
"""
for key_file_loader, file_loader in FileLoaderDict.items():
data_dir_path = [
join(DataPaths[key_file_loader], "train"),
join(DataPaths[key_file_loader], "test"),
]
common_args = dict(file_loader=file_loader,
labeled=True,
sample_label="all",
seed=None)
data_loader = PairedDataLoader(
data_dir_paths=data_dir_path,
fixed_image_shape=fixed_image_shape,
moving_image_shape=moving_image_shape,
**common_args,
)
# Check that file loaders are initialized correctly
file_loader_method = file_loader(dir_paths=data_dir_path,
name="moving_images",
grouped=False)
assert isinstance(data_loader.loader_moving_image,
type(file_loader_method))
assert isinstance(data_loader.loader_fixed_image,
type(file_loader_method))
assert isinstance(data_loader.loader_moving_label,
type(file_loader_method))
assert isinstance(data_loader.loader_fixed_label,
type(file_loader_method))
data_loader.close()
# Check the data_dir_path variable assertion error.
data_dir_path_int = [0, "1", 2, 3]
with pytest.raises(AssertionError):
PairedDataLoader(
data_dir_paths=data_dir_path_int,
fixed_image_shape=fixed_image_shape,
moving_image_shape=moving_image_shape,
**common_args,
)
def test_init_num_images():
"""
check init reads expected number of image pairs from given data path
"""
loader = PairedDataLoader(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
got = loader.num_images
expected = 1
loader.close()
assert got == expected
def test_sample_index_generator():
"""
Test to check the randomness and deterministic index generator
for train/test respectively.
"""
for key_file_loader, file_loader in FileLoaderDict.items():
for split in ["train", "test"]:
data_dir_path = [join(DataPaths[key_file_loader], split)]
indices_to_compare = []
for seed in [0, 1, 0]:
data_loader = PairedDataLoader(
data_dir_paths=data_dir_path,
fixed_image_shape=fixed_image_shape,
moving_image_shape=moving_image_shape,
file_loader=file_loader,
labeled=True,
sample_label="all",
seed=seed,
)
data_indices = []
for (
moving_index,
fixed_index,
indices,
) in data_loader.sample_index_generator():
assert isinstance(moving_index, int)
assert isinstance(fixed_index, int)
assert isinstance(indices, list)
assert moving_index == fixed_index
data_indices += indices
indices_to_compare.append(data_indices)
data_loader.close()
if data_loader.num_images > 1:
# test different seeds give different indices
assert not (np.allclose(indices_to_compare[0],
indices_to_compare[1]))
# test same seeds give the same indices
assert np.allclose(indices_to_compare[0],
indices_to_compare[2])
def test_sample_index_generator():
"""
check image index is expected value and format
"""
loader = PairedDataLoader(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
expected = (0, 0, [0])
image_index = PairedDataLoader.sample_index_generator(loader)
got = next(image_index)
loader.close()
assert expected == got
def test_init_sufficient_args():
"""
check if init method of loader returns any errors when all required
arguments given
"""
loader = PairedDataLoader(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
loader.__init__(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
loader.close()
def test_validate_data_files_label():
"""
check validate_data_files throws exception when moving and fixed label IDs vary
"""
loader = PairedDataLoader(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
# alter a data ID to cause error
loader.loader_moving_label.data_keys = "foo"
with pytest.raises(Exception) as execinfo:
PairedDataLoader.validate_data_files(loader)
msg = " ".join(execinfo.value.args[0].split())
loader.close()
assert "two lists are not identical" in msg
def get_single_data_loader(
data_type: str, data_config: dict, common_args: dict, data_dir_path: str
) -> DataLoader:
"""
Return one single data loader.
:param data_type: type of the data, paired / unpaired / grouped
:param data_config: dictionary containing the configuration of the data
:param common_args: some shared arguments for all data loaders
:param data_dir_path: path of the directory containing data
:return: a basic data loader
"""
try:
if data_type == "paired":
moving_image_shape = data_config["moving_image_shape"]
fixed_image_shape = data_config["fixed_image_shape"]
return PairedDataLoader(
data_dir_path=data_dir_path,
moving_image_shape=moving_image_shape,
fixed_image_shape=fixed_image_shape,
**common_args,
)
elif data_type == "grouped":
image_shape = data_config["image_shape"]
intra_group_prob = data_config["intra_group_prob"]
intra_group_option = data_config["intra_group_option"]
sample_image_in_group = data_config["sample_image_in_group"]
return GroupedDataLoader(
data_dir_path=data_dir_path,
intra_group_prob=intra_group_prob,
intra_group_option=intra_group_option,
sample_image_in_group=sample_image_in_group,
image_shape=image_shape,
**common_args,
)
elif data_type == "unpaired":
image_shape = data_config["image_shape"]
return UnpairedDataLoader(
data_dir_path=data_dir_path, image_shape=image_shape, **common_args
)
except KeyError as e:
msg = f"{e.args[0]} is not provided in the dataset config for paired data.\n"
if data_type == "paired":
msg += (
"Paired Loader requires 'moving_image_shape' and 'fixed_image_shape'.\n"
)
elif data_type == "grouped":
msg += (
"Grouped Loader requires 'image_shape', "
"as the data are not paired and will be resized to the same shape.\n"
"It also requires 'intra_group_prob', 'intra_group_option', and 'sample_image_in_group'.\n"
)
elif data_type == "unpaired":
msg += (
"Unpaired Loader requires 'image_shape', "
"as the data are not paired and will be resized to the same shape.\n"
)
raise ValueError(f"{msg}" f"The given dataset config is {data_config}\n")
raise ValueError(
f"Unknown data format {data_type}. "
f"Supported types are paired, unpaired, and grouped.\n"
)
