def test_file_loader_init():
"""
check file loader is correctly called in __init__:
"""
loader = PairedDataLoader(
file_loader=H5FileLoader,
data_dir_path=data_dir_path,
labeled=True,
sample_label=sample_label,
moving_image_shape=moving_image_shape_arr,
fixed_image_shape=fixed_image_shape_arr,
seed=seed,
)
file_loader = H5FileLoader(dir_path=data_dir_path,
name="moving_images",
grouped=False)
expected = ["case000025.nii.gz"]
loader_got = loader.loader_moving_image.get_data_ids()
file_loader_got = file_loader.get_data_ids()
loader.close()
file_loader.close()
assert loader_got == expected, "paired_loader has loaded incorrect moving image"
assert loader_got == file_loader_got, "paired_loader incorrectly calling h5_loader"
def test_init_duplicated_dirs(self, name):
# duplicated dir_paths
loader = get_loader(name)
dir_paths = loader.dir_paths * 2
with pytest.raises(ValueError) as err_info:
H5FileLoader(dir_paths=dir_paths, name=loader.name, grouped=loader.grouped)
assert "dir_paths have repeated elements" in str(err_info.value)
def test_set_data_structure():
"""
test set_data_structure in addition to tests above in test_init
"""
# test not existed files
dir_paths = ["./data/test/h5/paired/test"]
name = "images"
with pytest.raises(AssertionError) as err_info:
H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
assert "does not exist" in str(err_info.value)
# test wrong keys for grouped data
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
with pytest.raises(AssertionError) as err_info:
H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
assert "h5_file keys must be of form group-X-Y" in str(err_info.value)
def test_set_data_structure_err2(self):
dir_paths = ["./data/test/h5/paired/test"]
name = "error"
file_path = os.path.join(dir_paths[0], f"{name}.h5")
with h5py.File(file_path, "w"):
pass
with pytest.raises(ValueError) as err_info:
H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
assert "No data collected" in str(err_info.value)
os.remove(file_path)
def test_init_sufficient_args():
"""
check if init method of loader returns any errors when all required
arguments given
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
loader.__init__(dir_path=dir_path, name=name, grouped=False)
loader.close()
def test_set_group_structure():
"""
test set_group_structure in addition to tests above in test_init
"""
# data is not grouped but try using group_struct
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
with pytest.raises(AttributeError) as err_info:
loader.group_struct
assert "object has no attribute" in str(err_info.value)
def test_get_num_images():
"""
check if the get_num_images method works as expected
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
got = int(loader.get_num_images())
expected = int(1)
loader.close()
assert got == expected
def test_close():
"""
check if close method works as intended and closes file
close is the same code for all cases, so no need to test all cases
"""
# paired
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
loader.close()
for f in loader.h5_files.values():
assert not f.__bool__()
def test_close():
"""
check if close method works as intended and closes file
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
loader.close()
got = loader.h5_file.__bool__()
expected = False
assert got is expected
def test_data_keys():
"""
check if the data_keys are the same as expected
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
got = loader.data_keys
expected = ["case000025.nii.gz"]
loader.close()
assert got == expected
def test_get_data_incompatible_args():
"""
check if the get_data method works as expected and raises an error when
data is ungrouped but index is not an int
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
index = (0, 1)
with pytest.raises(AssertionError):
loader.get_data(index)
def test_h5_file():
"""
check if the filename is the same as expected
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
got = loader.h5_file.filename
expected = "./data/test/h5/paired/test/fixed_images.h5"
loader.close()
assert got == expected
def test_get_data_ids():
"""
check if the get_data_ids method works as expected
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
got = loader.get_data_ids()
expected = ["case000025.nii.gz"]
loader.close()
assert got == expected
def test_get_data_negative_index():
"""
check if the get_data method works as expected and raises an error when
the index is out of range
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
index = -1
with pytest.raises(AssertionError):
loader.get_data(index)
def test_init_incompatible_conditions():
"""
check if the initialisation works as expected and raises an error when
directories to ungrouped files is given but grouped variable is set to
True
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
with pytest.raises(IndexError) as execinfo:
H5FileLoader(dir_path=dir_path, name=name, grouped=True)
msg = " ".join(execinfo.value.args[0].split())
assert "index out of range" in msg
def test_get_num_images():
"""
check if the get_num_images method works as expected
"""
# paired
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = loader.get_num_images()
expected = 1
loader.close()
assert got == expected
# unpaired
dir_paths = ["./data/test/h5/unpaired/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = loader.get_num_images()
expected = 2
loader.close()
assert got == expected
# grouped
dir_paths = ["./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
got = loader.get_num_images()
expected = 2
loader.close()
assert got == expected
# multi dirs
dir_paths = ["./data/test/h5/grouped/train", "./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
got = loader.get_num_images()
expected = 9
loader.close()
assert got == expected
def test_set_group_structure():
"""
check if the set_group_structure method works as intended when data is
grouped
"""
dir_path = "./data/test/h5/grouped/test"
name = "images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=True)
loader.set_group_structure()
got = [loader.group_ids, loader.group_sample_dict]
expected = [["1"], {"1": ["1", "2"]}]
loader.close()
assert got == expected
def test_set_group_structure_ungrouped():
"""
check if the set_group_structure method works as intended when data is
not grouped
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
loader.set_group_structure()
with pytest.raises(AttributeError) as execinfo:
loader.group_ids
msg = " ".join(execinfo.value.args[0].split())
assert "object has no attribute" in msg
def test_get_data_incorrect_args():
"""
check if the get_data method works as expected and raises an error when
an incorrect data type is fed in
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
index = "abc"
with pytest.raises(ValueError) as execinfo:
loader.get_data(index)
msg = " ".join(execinfo.value.args[0].split())
assert "must be int, or tuple" in msg
def get_loader(loader_name):
if loader_name in [
"paired",
"unpaired",
"grouped",
]:
dir_paths = [f"./data/test/h5/{loader_name}/test"]
name = "fixed_images" if loader_name == "paired" else "images"
grouped = loader_name == "grouped"
elif loader_name == "multi_dirs_grouped":
dir_paths = ["./data/test/h5/grouped/train", "./data/test/h5/grouped/test"]
name = "images"
grouped = True
else:
raise ValueError
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=grouped)
return loader
def test_get_data():
"""
check if the get_data method works as expected and returns array
as expected
"""
dir_path = "./data/test/h5/paired/test"
name = "fixed_images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=False)
index = 0
array = loader.get_data(index)
got = [
np.shape(array),
[np.amax(array),
np.amin(array),
np.mean(array),
np.std(array)],
]
expected = [(44, 59, 41), [255.0, 0.0, 68.359276, 65.84009]]
loader.close()
if got[0] == expected[0]:
assert check_equal(np.array(got[1]), np.array(expected[1])).all()
else:
raise AssertionError
def test_get_data_grouped():
"""
check if the get_data method works as expected and returns array
as expected
"""
dir_path = "./data/test/h5/grouped/test"
name = "images"
loader = H5FileLoader(dir_path=dir_path, name=name, grouped=True)
index = (0, 1)
array = loader.get_data(index)
got = [
np.shape(array),
[np.amax(array),
np.amin(array),
np.mean(array),
np.std(array)],
]
expected = [(64, 64, 60), [255.0, 0.0, 60.073948, 47.27648]]
loader.close()
if got[0] == expected[0]:
assert check_equal(np.array(got[1]), np.array(expected[1])).all()
else:
raise AssertionError
def test_get_data():
"""
test get_data method by verifying returns' shape and value stats
"""
# paired
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
index = 0
array = loader.get_data(index)
got = [
np.shape(array),
[np.amax(array),
np.amin(array),
np.mean(array),
np.std(array)],
]
expected = [(44, 59, 41), [255.0, 0.0, 68.359276, 65.84009]]
loader.close()
assert got[0] == expected[0]
assert is_equal_np(got[1], expected[1])
# unpaired
dir_paths = ["./data/test/h5/unpaired/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
index = 0
array = loader.get_data(index)
got = [
np.shape(array),
[np.amax(array),
np.amin(array),
np.mean(array),
np.std(array)],
]
expected = [(64, 64, 60), [255.0, 0.0, 60.073948, 47.27648]]
loader.close()
assert got[0] == expected[0]
assert is_equal_np(got[1], expected[1])
# grouped
dir_paths = ["./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
index = (0, 1)
array = loader.get_data(index)
got = [
np.shape(array),
[np.amax(array),
np.amin(array),
np.mean(array),
np.std(array)],
]
expected = [(64, 64, 60), [255.0, 0.0, 60.073948, 47.27648]]
loader.close()
assert got[0] == expected[0]
assert is_equal_np(got[1], expected[1])
# multi dirs
dir_paths = ["./data/test/h5/grouped/train", "./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
index = (0, 1)
array = loader.get_data(index)
got = [
np.shape(array),
[np.amax(array),
np.amin(array),
np.mean(array),
np.std(array)],
]
expected = [(64, 64, 60), [255.0, 0.0, 60.073948, 47.27648]]
loader.close()
assert got[0] == expected[0]
assert is_equal_np(got[1], expected[1])
def test_set_data_structure_err1(self, name, err_msg):
dir_paths = ["./data/test/h5/paired/test"]
with pytest.raises(AssertionError) as err_info:
H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
assert err_msg in str(err_info.value)
def test_get_data_ids():
"""
check if the get_data_ids method works as expected
"""
# paired
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = loader.get_data_ids()
expected = [("./data/test/h5/paired/test", "case000025.nii.gz")]
loader.close()
assert got == expected
# unpaired
dir_paths = ["./data/test/h5/unpaired/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = loader.get_data_ids()
expected = [
("./data/test/h5/unpaired/test", "case000025.nii.gz"),
("./data/test/h5/unpaired/test", "case000026.nii.gz"),
]
loader.close()
assert got == expected
# grouped
dir_paths = ["./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = loader.get_data_ids()
expected = [
("./data/test/h5/grouped/test", "group-1-1"),
("./data/test/h5/grouped/test", "group-1-2"),
]
loader.close()
assert got == expected
# multi dirs
dir_paths = ["./data/test/h5/grouped/train", "./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = loader.get_data_ids()
expected = [
("./data/test/h5/grouped/train", "group-1-1"),
("./data/test/h5/grouped/train", "group-1-2"),
("./data/test/h5/grouped/train", "group-1-3"),
("./data/test/h5/grouped/train", "group-1-4"),
("./data/test/h5/grouped/train", "group-2-1"),
("./data/test/h5/grouped/train", "group-2-2"),
("./data/test/h5/grouped/train", "group-2-3"),
("./data/test/h5/grouped/test", "group-1-1"),
("./data/test/h5/grouped/test", "group-1-2"),
]
loader.close()
assert got == expected
# wrong index for paired
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
with pytest.raises(AssertionError):
# negative data_index
loader.get_data(index=-1)
with pytest.raises(IndexError):
# out of range data_index
loader.get_data(index=64)
with pytest.raises(AssertionError):
# non int data_index
loader.get_data(index=(0, 1))
with pytest.raises(ValueError) as err_info:
loader.get_data(index="wrong")
assert "must be int, or tuple" in str(err_info.value)
loader.close()
# wrong index for grouped
dir_paths = ["./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
with pytest.raises(AssertionError):
# non-tuple data_index
loader.get_data(index=1)
loader.close()
def test_init():
"""
check init of H5FileLoader
this includes some test on set_data_structure/set_group_structure
as these functions are called in init
"""
# paired
# test h5_files and data_path_splits
dir_paths = ["./data/test/h5/paired/test"]
name = "fixed_images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = [get_loader_h5_file_names(loader), loader.data_path_splits]
expected = [
["./data/test/h5/paired/test/fixed_images.h5"],
[("./data/test/h5/paired/test", "case000025.nii.gz")],
]
loader.close()
assert got == expected
# unpaired
# test h5_files and data_path_splits
dir_paths = ["./data/test/h5/unpaired/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=False)
got = [get_loader_h5_file_names(loader), loader.data_path_splits]
expected = [
["./data/test/h5/unpaired/test/images.h5"],
[
("./data/test/h5/unpaired/test", "case000025.nii.gz"),
("./data/test/h5/unpaired/test", "case000026.nii.gz"),
],
]
loader.close()
assert got == expected
# grouped
# test h5_files, data_path_splits and group_struct
dir_paths = ["./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
got = [
get_loader_h5_file_names(loader),
loader.data_path_splits,
loader.group_struct,
]
expected = [
["./data/test/h5/grouped/test/images.h5"],
[
("./data/test/h5/grouped/test", "1", "1"),
("./data/test/h5/grouped/test", "1", "2"),
],
[[0, 1]],
]
loader.close()
assert got == expected
# multi dirs
# test h5_files, data_path_splits and group_struct
dir_paths = ["./data/test/h5/grouped/train", "./data/test/h5/grouped/test"]
name = "images"
loader = H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
got = [
get_loader_h5_file_names(loader),
loader.data_path_splits,
loader.group_struct,
]
expected = [
[
"./data/test/h5/grouped/train/images.h5",
"./data/test/h5/grouped/test/images.h5",
],
[
("./data/test/h5/grouped/train", "1", "1"),
("./data/test/h5/grouped/train", "1", "2"),
("./data/test/h5/grouped/train", "1", "3"),
("./data/test/h5/grouped/train", "1", "4"),
("./data/test/h5/grouped/train", "2", "1"),
("./data/test/h5/grouped/train", "2", "2"),
("./data/test/h5/grouped/train", "2", "3"),
("./data/test/h5/grouped/test", "1", "1"),
("./data/test/h5/grouped/test", "1", "2"),
],
[[7, 8], [0, 1, 2, 3], [4, 5, 6]],
]
loader.close()
assert got == expected
# duplicated dir_paths
dir_paths = ["./data/test/h5/grouped/test", "./data/test/h5/grouped/test"]
name = "images"
with pytest.raises(ValueError) as err_info:
H5FileLoader(dir_paths=dir_paths, name=name, grouped=True)
assert "dir_paths have repeated elements" in str(err_info.value)