def test_shape(self, expected_shape):
test_image = nib.Nifti1Image(
np.random.randint(0, 2, size=[128, 128, 128]), np.eye(4))
tempdir = tempfile.mkdtemp()
nib.save(test_image, os.path.join(tempdir, "test_image1.nii.gz"))
nib.save(test_image, os.path.join(tempdir, "test_label1.nii.gz"))
nib.save(test_image, os.path.join(tempdir, "test_extra1.nii.gz"))
nib.save(test_image, os.path.join(tempdir, "test_image2.nii.gz"))
nib.save(test_image, os.path.join(tempdir, "test_label2.nii.gz"))
nib.save(test_image, os.path.join(tempdir, "test_extra2.nii.gz"))
test_data = [
{
"image": os.path.join(tempdir, "test_image1.nii.gz"),
"label": os.path.join(tempdir, "test_label1.nii.gz"),
"extra": os.path.join(tempdir, "test_extra1.nii.gz"),
},
{
"image": os.path.join(tempdir, "test_image2.nii.gz"),
"label": os.path.join(tempdir, "test_label2.nii.gz"),
"extra": os.path.join(tempdir, "test_extra2.nii.gz"),
},
]
test_transform = Compose([
LoadNiftid(keys=["image", "label", "extra"]),
SimulateDelayd(keys=["image", "label", "extra"],
delay_time=[1e-7, 1e-6, 1e-5]),
])
dataset = Dataset(data=test_data, transform=test_transform)
data1 = dataset[0]
data2 = dataset[1]
self.assertTupleEqual(data1["image"].shape, expected_shape)
self.assertTupleEqual(data1["label"].shape, expected_shape)
self.assertTupleEqual(data1["extra"].shape, expected_shape)
self.assertTupleEqual(data2["image"].shape, expected_shape)
self.assertTupleEqual(data2["label"].shape, expected_shape)
self.assertTupleEqual(data2["extra"].shape, expected_shape)
dataset = Dataset(
data=test_data,
transform=LoadNiftid(keys=["image", "label", "extra"]))
data1_simple = dataset[0]
data2_simple = dataset[1]
self.assertTupleEqual(data1_simple["image"].shape, expected_shape)
self.assertTupleEqual(data1_simple["label"].shape, expected_shape)
self.assertTupleEqual(data1_simple["extra"].shape, expected_shape)
self.assertTupleEqual(data2_simple["image"].shape, expected_shape)
self.assertTupleEqual(data2_simple["label"].shape, expected_shape)
self.assertTupleEqual(data2_simple["extra"].shape, expected_shape)
shutil.rmtree(tempdir)
def run_interaction(self, train, compose):
data = [{
"image": np.ones((1, 2, 2, 2)).astype(np.float32),
"label": np.ones((1, 2, 2, 2))
} for _ in range(5)]
network = torch.nn.Linear(2, 2)
lr = 1e-3
opt = torch.optim.SGD(network.parameters(), lr)
loss = torch.nn.L1Loss()
train_transforms = Compose([
FindAllValidSlicesd(label="label", sids="sids"),
AddInitialSeedPointd(label="label",
guidance="guidance",
sids="sids"),
AddGuidanceSignald(image="image", guidance="guidance"),
ToTensord(keys=("image", "label")),
])
dataset = Dataset(data, transform=train_transforms)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=5)
iteration_transforms = [
Activationsd(keys="pred", sigmoid=True),
ToNumpyd(keys=["image", "label", "pred"]),
FindDiscrepancyRegionsd(label="label",
pred="pred",
discrepancy="discrepancy"),
AddRandomGuidanced(guidance="guidance",
discrepancy="discrepancy",
probability="probability"),
AddGuidanceSignald(image="image", guidance="guidance"),
ToTensord(keys=("image", "label")),
]
iteration_transforms = Compose(
iteration_transforms) if compose else iteration_transforms
i = Interaction(transforms=iteration_transforms,
train=train,
max_interactions=5)
self.assertEqual(len(i.transforms.transforms), 6,
"Mismatch in expected transforms")
# set up engine
engine = SupervisedTrainer(
device=torch.device("cpu"),
max_epochs=1,
train_data_loader=data_loader,
network=network,
optimizer=opt,
loss_function=loss,
iteration_update=i,
)
engine.add_event_handler(IterationEvents.INNER_ITERATION_STARTED,
add_one)
engine.add_event_handler(IterationEvents.INNER_ITERATION_COMPLETED,
add_one)
engine.run()
self.assertIsNotNone(engine.state.batch[0].get("guidance"),
"guidance is missing")
self.assertEqual(engine.state.best_metric, 9)
def test_shape(self):
expected_shape = (128, 128, 128)
test_image = nib.Nifti1Image(
np.random.randint(0, 2, size=[128, 128, 128]), np.eye(4))
test_data = []
with tempfile.TemporaryDirectory() as tempdir:
for i in range(6):
nib.save(test_image,
os.path.join(tempdir, f"test_image{str(i)}.nii.gz"))
test_data.append(
{"image": os.path.join(tempdir, f"test_image{i}.nii.gz")})
test_transform = Compose([
LoadImaged(keys="image"),
SimulateDelayd(keys="image", delay_time=1e-7)
])
data_iterator = _Stream(test_data)
with self.assertRaises(TypeError): # Dataset doesn't work
dataset = Dataset(data=data_iterator, transform=test_transform)
for _ in dataset:
pass
dataset = IterableDataset(data=data_iterator,
transform=test_transform)
for d in dataset:
self.assertTupleEqual(d["image"].shape, expected_shape)
num_workers = 2 if sys.platform == "linux" else 0
dataloader = DataLoader(dataset=dataset,
batch_size=3,
num_workers=num_workers)
for d in dataloader:
self.assertTupleEqual(d["image"].shape[1:], expected_shape)
def query_by_case(self, patient_id: str) -> Dataset:
"""Return nodule volumes for one specific case.
Args:
patient_id (str): Patient ID of desired case.
Returns:
Dataset: Dataset containing case nodules.
"""
train_cases, valid_cases = self.splits
train_pids = [case["pid"] for case in train_cases]
valid_pids = [case["pid"] for case in valid_cases]
if patient_id in train_pids:
data_dict = [{
"image": nod["image"],
"label": nod[self.target]
} for case in train_cases if case["pid"] == patient_id
for nod in case["nodules"]]
elif patient_id in valid_pids:
data_dict = [{
"image": nod["image"],
"label": nod[self.target]
} for case in valid_cases if case["pid"] == patient_id
for nod in case["nodules"]]
else:
raise ValueError("Case with given ID could not be found.")
return Dataset(data_dict, transform=self.val_transforms)
def test_anisotropic_spacing(self):
with tempfile.TemporaryDirectory() as tempdir:
pixdims = [[1.0, 1.0, 5.0], [1.0, 1.0, 4.0], [1.0, 1.0, 4.5], [1.0, 1.0, 2.0], [1.0, 1.0, 1.0]]
for i in range(5):
im, seg = create_test_image_3d(32, 32, 32, num_seg_classes=1, num_objs=3, rad_max=6, channel_dim=0)
n = nib.Nifti1Image(im, np.eye(4))
n.header["pixdim"][1:4] = pixdims[i]
nib.save(n, os.path.join(tempdir, f"img{i:d}.nii.gz"))
n = nib.Nifti1Image(seg, np.eye(4))
n.header["pixdim"][1:4] = pixdims[i]
nib.save(n, os.path.join(tempdir, f"seg{i:d}.nii.gz"))
train_images = sorted(glob.glob(os.path.join(tempdir, "img*.nii.gz")))
train_labels = sorted(glob.glob(os.path.join(tempdir, "seg*.nii.gz")))
data_dicts = [
{"image": image_name, "label": label_name} for image_name, label_name in zip(train_images, train_labels)
]
t = Compose([LoadImaged(keys=["image", "label"]), FromMetaTensord(keys=["image", "label"])])
dataset = Dataset(data=data_dicts, transform=t)
calculator = DatasetSummary(dataset, num_workers=4, meta_key_postfix=PostFix.meta())
target_spacing = calculator.get_target_spacing(anisotropic_threshold=4.0, percentile=20.0)
np.testing.assert_allclose(target_spacing, (1.0, 1.0, 1.8))
def run_interaction(self, train, compose):
data = []
for i in range(5):
data.append({"image": torch.tensor([float(i)]), "label": torch.tensor([float(i)])})
network = torch.nn.Linear(1, 1)
lr = 1e-3
opt = torch.optim.SGD(network.parameters(), lr)
loss = torch.nn.L1Loss()
dataset = Dataset(data, transform=None)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=5)
iteration_transforms = [Activationsd(keys="pred", sigmoid=True), ToNumpyd(keys="pred")]
iteration_transforms = Compose(iteration_transforms) if compose else iteration_transforms
i = Interaction(transforms=iteration_transforms, train=train, max_interactions=5)
self.assertEqual(len(i.transforms.transforms), 2, "Mismatch in expected transforms")
# set up engine
engine = SupervisedTrainer(
device=torch.device("cpu"),
max_epochs=1,
train_data_loader=data_loader,
network=network,
optimizer=opt,
loss_function=loss,
iteration_update=i,
)
engine.add_event_handler(IterationEvents.INNER_ITERATION_STARTED, add_one)
engine.add_event_handler(IterationEvents.INNER_ITERATION_COMPLETED, add_one)
engine.run()
self.assertIsNotNone(engine.state.batch.get("probability"), "Probability is missing")
self.assertEqual(engine.state.best_metric, 9)
def test_time(self):
dataset = Dataset(
data=self.datalist * 2,
transform=self.transform) # contains data for 2 batches
dataloader = DataLoader(dataset=dataset, batch_size=2, num_workers=0)
tbuffer = ThreadBuffer(dataloader)
with PerfContext() as pc:
for _ in dataloader:
time.sleep(
0.5
) # each batch takes 0.8 s to generate on top of this time
unbuffered_time = pc.total_time
with PerfContext() as pc:
for _ in tbuffer:
time.sleep(
0.5
) # while "computation" is happening the next batch is being generated, saving 0.4 s
buffered_time = pc.total_time
if sys.platform == "darwin": # skip macOS measure
print(
f"darwin: Buffered time {buffered_time} vs unbuffered time {unbuffered_time}"
)
else:
self.assertTrue(
buffered_time < unbuffered_time,
f"Buffered time {buffered_time} should be less than unbuffered time {unbuffered_time}",
)
def test_dataloader(self):
dataset = Dataset(
data=[{
"img": np.array([[[0.0, 1.0], [2.0, 3.0]]])
}, {
"img": np.array([[[0.0, 1.0], [2.0, 3.0]]])
}],
transform=IntensityStatsd(keys="img",
ops=["max", "mean"],
key_prefix="orig"),
)
# set num workers = 0 for mac / win
num_workers = 2 if sys.platform == "linux" else 0
dataloader = DataLoader(dataset=dataset,
num_workers=num_workers,
batch_size=2)
orig_method = mp.get_start_method()
mp.set_start_method("spawn", force=True)
for d in dataloader:
meta = d["img_meta_dict"]
np.testing.assert_allclose(meta["orig_max"], [3.0, 3.0], atol=1e-3)
np.testing.assert_allclose(meta["orig_mean"], [1.5, 1.5],
atol=1e-3)
# restore the mp method
mp.set_start_method(orig_method, force=True)
def test_content(self, data, trigger_event):
# set up engine
gb_count_dict = {}
def _train_func(engine, batch):
# store garbage collection counts
if trigger_event == Events.EPOCH_COMPLETED or trigger_event.lower(
) == "epoch":
if engine.state.iteration % engine.state.epoch_length == 1:
gb_count_dict[engine.state.epoch] = gc.get_count()
elif trigger_event.lower() == "iteration":
gb_count_dict[engine.state.iteration] = gc.get_count()
engine = Engine(_train_func)
# set up testing handler
dataset = Dataset(data, transform=None)
data_loader = torch.utils.data.DataLoader(dataset, batch_size=1)
GarbageCollector(trigger_event=trigger_event,
log_level=30).attach(engine)
engine.run(data_loader, max_epochs=5)
first_count = 0
for iter, gb_count in gb_count_dict.items():
# At least one zero-generation object is collected
# self.assertGreaterEqual(gb_count[0], 0)
if iter > 1:
# Since we are collecting all objects from all generations manually at each call,
# starting from the second call, there shouldn't be any 1st and 2nd
# generation objects available to collect.
self.assertEqual(gb_count[1], first_count)
self.assertEqual(gb_count[2], first_count)
def test_exception(self, datalist):
dataset = Dataset(data=datalist, transform=None)
dataloader = DataLoader(dataset=dataset, batch_size=2, num_workers=0)
with self.assertRaisesRegex((TypeError, RuntimeError),
"Collate error on the key"):
for _ in dataloader:
pass
def test_shape(self, expected_shape):
test_image = nib.Nifti1Image(
np.random.randint(0, 2, size=[128, 128, 128]), np.eye(4))
tempdir = tempfile.mkdtemp()
nib.save(test_image, os.path.join(tempdir, 'test_image1.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_label1.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_extra1.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_image2.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_label2.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_extra2.nii.gz'))
test_data = [{
'image': os.path.join(tempdir, 'test_image1.nii.gz'),
'label': os.path.join(tempdir, 'test_label1.nii.gz'),
'extra': os.path.join(tempdir, 'test_extra1.nii.gz')
}, {
'image': os.path.join(tempdir, 'test_image2.nii.gz'),
'label': os.path.join(tempdir, 'test_label2.nii.gz'),
'extra': os.path.join(tempdir, 'test_extra2.nii.gz')
}]
dataset = Dataset(
data=test_data,
transform=LoadNiftid(keys=['image', 'label', 'extra']))
data1 = dataset[0]
data2 = dataset[1]
shutil.rmtree(tempdir)
self.assertTupleEqual(data1['image'].shape, expected_shape)
self.assertTupleEqual(data1['label'].shape, expected_shape)
self.assertTupleEqual(data1['extra'].shape, expected_shape)
self.assertTupleEqual(data2['image'].shape, expected_shape)
self.assertTupleEqual(data2['label'].shape, expected_shape)
self.assertTupleEqual(data2['extra'].shape, expected_shape)
def test_shape(self, expected_shape):
test_image = nib.Nifti1Image(
np.random.randint(0, 2, size=[128, 128, 128]), np.eye(4))
tempdir = tempfile.mkdtemp()
nib.save(test_image, os.path.join(tempdir, 'test_image1.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_label1.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_extra1.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_image2.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_label2.nii.gz'))
nib.save(test_image, os.path.join(tempdir, 'test_extra2.nii.gz'))
test_data = [{
'image': os.path.join(tempdir, 'test_image1.nii.gz'),
'label': os.path.join(tempdir, 'test_label1.nii.gz'),
'extra': os.path.join(tempdir, 'test_extra1.nii.gz')
}, {
'image': os.path.join(tempdir, 'test_image2.nii.gz'),
'label': os.path.join(tempdir, 'test_label2.nii.gz'),
'extra': os.path.join(tempdir, 'test_extra2.nii.gz')
}]
test_transform = Compose([
LoadNiftid(keys=['image', 'label', 'extra']),
SimulateDelayd(keys=['image', 'label', 'extra'],
delay_time=[1e-7, 1e-6, 1e-5])
])
dataset = Dataset(data=test_data, transform=test_transform)
data1 = dataset[0]
data2 = dataset[1]
self.assertTupleEqual(data1['image'].shape, expected_shape)
self.assertTupleEqual(data1['label'].shape, expected_shape)
self.assertTupleEqual(data1['extra'].shape, expected_shape)
self.assertTupleEqual(data2['image'].shape, expected_shape)
self.assertTupleEqual(data2['label'].shape, expected_shape)
self.assertTupleEqual(data2['extra'].shape, expected_shape)
dataset = Dataset(
data=test_data,
transform=LoadNiftid(keys=['image', 'label', 'extra']))
data1_simple = dataset[0]
data2_simple = dataset[1]
self.assertTupleEqual(data1_simple['image'].shape, expected_shape)
self.assertTupleEqual(data1_simple['label'].shape, expected_shape)
self.assertTupleEqual(data1_simple['extra'].shape, expected_shape)
self.assertTupleEqual(data2_simple['image'].shape, expected_shape)
self.assertTupleEqual(data2_simple['label'].shape, expected_shape)
self.assertTupleEqual(data2_simple['extra'].shape, expected_shape)
def test_decollation_list(self, *transforms):
t_compose = Compose([AddChannel(), Compose(transforms), ToTensor()])
# If nibabel present, read from disk
if has_nib:
t_compose = _ListCompose([LoadImage(image_only=False), t_compose])
dataset = Dataset(self.data_list, t_compose)
self.check_decollate(dataset=dataset)
def test_spacing_intensity(self):
set_determinism(seed=0)
with tempfile.TemporaryDirectory() as tempdir:
for i in range(5):
im, seg = create_test_image_3d(32,
32,
32,
num_seg_classes=1,
num_objs=3,
rad_max=6,
channel_dim=0)
n = nib.Nifti1Image(im, np.eye(4))
nib.save(n, os.path.join(tempdir, f"img{i:d}.nii.gz"))
n = nib.Nifti1Image(seg, np.eye(4))
nib.save(n, os.path.join(tempdir, f"seg{i:d}.nii.gz"))
train_images = sorted(
glob.glob(os.path.join(tempdir, "img*.nii.gz")))
train_labels = sorted(
glob.glob(os.path.join(tempdir, "seg*.nii.gz")))
data_dicts = [{
"image": image_name,
"label": label_name
} for image_name, label_name in zip(train_images, train_labels)]
t = Compose([
LoadImaged(keys=["image", "label"]),
ToNumpyd(keys=[
"image", "label", "image_meta_dict", "label_meta_dict"
]),
])
dataset = Dataset(data=data_dicts, transform=t)
# test **kwargs of `DatasetSummary` for `DataLoader`
calculator = DatasetSummary(dataset,
num_workers=4,
meta_key="image_meta_dict",
collate_fn=test_collate)
target_spacing = calculator.get_target_spacing(
spacing_key="pixdim")
self.assertEqual(target_spacing, (1.0, 1.0, 1.0))
calculator.calculate_statistics()
np.testing.assert_allclose(calculator.data_mean,
0.892599,
rtol=1e-5,
atol=1e-5)
np.testing.assert_allclose(calculator.data_std,
0.131731,
rtol=1e-5,
atol=1e-5)
calculator.calculate_percentiles(sampling_flag=True, interval=2)
self.assertEqual(calculator.data_max_percentile, 1.0)
np.testing.assert_allclose(calculator.data_min_percentile,
0.556411,
rtol=1e-5,
atol=1e-5)
def test_lazy_instantiation(self):
config = {"_target_": "DataLoader", "dataset": Dataset(data=[1, 2]), "batch_size": 2}
configer = ConfigComponent(config=config, locator=None)
init_config = configer.get_config()
# modify config content at runtime
init_config["batch_size"] = 4
configer.update_config(config=init_config)
ret = configer.instantiate()
self.assertTrue(isinstance(ret, DataLoader))
self.assertEqual(ret.batch_size, 4)
def _define_prediction_data_loaders(
self,
prediction_folder_path: Union[Path, str]
) -> bool:
"""Initialize prediction datasets and data loaders.
@Note: in Windows, it is essential to set `persistent_workers=True` in the data loaders!
@return True if datasets and data loaders could be instantiated, False otherwise.
"""
# Check that the path exists
prediction_folder_path = Path(prediction_folder_path)
if not prediction_folder_path.is_dir():
return False
# Scan for images
self._prediction_image_names = natsorted(
glob(str(Path(prediction_folder_path) / "*.tif"))
)
# Optimize arguments
if sys.platform == 'win32':
persistent_workers = True
pin_memory = False
else:
persistent_workers = False
pin_memory = torch.cuda.is_available()
if len(self._prediction_image_names) == 0:
self._prediction_dataset = None
self._prediction_dataloader = None
return False
# Define the transforms
self._define_prediction_transforms()
# Prediction
self._prediction_dataset = Dataset(
self._prediction_image_names,
self._prediction_image_transforms
)
self._prediction_dataloader = DataLoader(
self._prediction_dataset,
batch_size=self._test_batch_size,
shuffle=False,
num_workers=self._test_num_workers,
persistent_workers=persistent_workers,
pin_memory=pin_memory
)
return True
def test_values(self):
dataset = Dataset(data=self.datalist, transform=self.transform)
dataloader = DataLoader(dataset=dataset, batch_size=2, num_workers=0)
tbuffer = ThreadBuffer(dataloader)
for d in tbuffer:
self.assertEqual(d["image"][0], "spleen_19.nii.gz")
self.assertEqual(d["image"][1], "spleen_31.nii.gz")
self.assertEqual(d["label"][0], "spleen_label_19.nii.gz")
self.assertEqual(d["label"][1], "spleen_label_31.nii.gz")
def test_decollate(self, dtype):
batch_size = 3
ims = [self.get_im(dtype=dtype)[0] for _ in range(batch_size * 2)]
ds = Dataset(ims)
dl = DataLoader(ds, num_workers=batch_size, batch_size=batch_size)
batch = next(iter(dl))
decollated = decollate_batch(batch)
self.assertIsInstance(decollated, list)
self.assertEqual(len(decollated), batch_size)
for elem, im in zip(decollated, ims):
self.assertIsInstance(elem, MetaTensor)
self.check(elem, im, ids=False)
def test_thread_workers(self):
dataset = Dataset(data=self.datalist, transform=self.transform)
dataloader = ThreadDataLoader(dataset=dataset,
batch_size=2,
num_workers=2,
use_thread_workers=True)
for d in dataloader:
self.assertEqual(d["image"][0], "spleen_19.nii.gz")
self.assertEqual(d["image"][1], "spleen_31.nii.gz")
self.assertEqual(d["label"][0], "spleen_label_19.nii.gz")
self.assertEqual(d["label"][1], "spleen_label_31.nii.gz")
def test_endianness(self, endianness, use_array, image_only):
hdr = nib.Nifti1Header(endianness=endianness)
nii = nib.Nifti1Image(self.im, np.eye(4), header=hdr)
nib.save(nii, self.fname)
data = [self.fname] if use_array else [{"image": self.fname}]
tr = LoadImage(image_only=image_only) if use_array else LoadImaged(
"image", image_only=image_only)
check_ds = Dataset(data, tr)
check_loader = DataLoader(check_ds, batch_size=1)
_ = next(iter(check_loader))
def test_endianness(self, endianness, use_array, image_only):
hdr = nib.Nifti1Header(endianness=endianness)
nii = nib.Nifti1Image(self.im, np.eye(4), header=hdr)
nib.save(nii, self.fname)
data = [self.fname] if use_array else [{"image": self.fname}]
tr = LoadImage(image_only=image_only) if use_array else LoadImaged("image", image_only=image_only)
check_ds = Dataset(data, tr)
check_loader = DataLoader(check_ds, batch_size=1)
ret = next(iter(check_loader))
if isinstance(ret, dict) and "image_meta_dict" in ret:
np.testing.assert_allclose(ret["image_meta_dict"]["spatial_shape"], [[100, 100]])
def test_dataloader(self, dtype):
batch_size = 5
ims = [self.get_im(dtype=dtype)[0] for _ in range(batch_size * 2)]
ds = Dataset(ims)
im_shape = tuple(ims[0].shape)
affine_shape = tuple(ims[0].affine.shape)
expected_im_shape = (batch_size,) + im_shape
expected_affine_shape = (batch_size,) + affine_shape
dl = DataLoader(ds, num_workers=batch_size, batch_size=batch_size)
for batch in dl:
self.assertIsInstance(batch, MetaTensor)
self.assertTupleEqual(tuple(batch.shape), expected_im_shape)
self.assertTupleEqual(tuple(batch.affine.shape), expected_affine_shape)
def test_with_dataloader(self, file_path, level, expected_spatial_shape, expected_shape):
train_transform = Compose(
[
LoadImaged(keys=["image"], reader=WSIReader, backend=self.backend, level=level),
ToTensord(keys=["image"]),
]
)
dataset = Dataset([{"image": file_path}], transform=train_transform)
data_loader = DataLoader(dataset)
data: dict = first(data_loader)
for s in data[PostFix.meta("image")]["spatial_shape"]:
torch.testing.assert_allclose(s, expected_spatial_shape)
self.assertTupleEqual(data["image"].shape, expected_shape)
def test_loading_array(self):
set_determinism(seed=1234)
# image dataset
images = [
np.arange(16, dtype=float).reshape(1, 4, 4),
np.arange(16, dtype=float).reshape(1, 4, 4)
]
# image patch sampler
n_samples = 8
sampler = RandSpatialCropSamples(roi_size=(3, 3),
num_samples=n_samples,
random_center=True,
random_size=False)
# image level
patch_intensity = RandShiftIntensity(offsets=1.0, prob=1.0)
image_ds = Dataset(images, transform=patch_intensity)
# patch level
ds = PatchDataset(dataset=image_ds,
patch_func=sampler,
samples_per_image=n_samples,
transform=patch_intensity)
np.testing.assert_equal(len(ds), n_samples * len(images))
# use the patch dataset, length: len(images) x samplers_per_image
for item in DataLoader(ds, batch_size=2, shuffle=False, num_workers=0):
np.testing.assert_equal(tuple(item.shape), (2, 1, 3, 3))
np.testing.assert_allclose(
item[0],
np.array([[[1.338681, 2.338681, 3.338681],
[5.338681, 6.338681, 7.338681],
[9.338681, 10.338681, 11.338681]]]),
rtol=1e-5,
)
if sys.platform != "win32":
for item in DataLoader(ds,
batch_size=2,
shuffle=False,
num_workers=2):
np.testing.assert_equal(tuple(item.shape), (2, 1, 3, 3))
np.testing.assert_allclose(
item[0],
np.array([[
[4.957847, 5.957847, 6.957847],
[8.957847, 9.957847, 10.957847],
[12.957847, 13.957847, 14.957847],
]]),
rtol=1e-5,
)
set_determinism(seed=None)
def test_with_dataloader_batch(self, file_path, level, expected_spatial_shape, expected_shape):
train_transform = Compose(
[
LoadImaged(keys=["image"], reader=WSIReader, backend=self.backend, level=level),
FromMetaTensord(keys=["image"]),
ToTensord(keys=["image"]),
]
)
dataset = Dataset([{"image": file_path}, {"image": file_path}], transform=train_transform)
batch_size = 2
data_loader = DataLoader(dataset, batch_size=batch_size)
data: dict = first(data_loader)
for s in data[PostFix.meta("image")]["spatial_shape"]:
assert_allclose(s, expected_spatial_shape, type_test=False)
self.assertTupleEqual(data["image"].shape, (batch_size, *expected_shape[1:]))
def get_task_params(args):
"""
This function is used to achieve the spacings of decathlon dataset.
In addition, for CT images (task 03, 06, 07, 08, 09 and 10), this function
also prints the mean and std values (used for normalization), and the min (0.5 percentile)
and max(99.5 percentile) values (used for clip).
"""
task_id = args.task_id
root_dir = args.root_dir
datalist_path = args.datalist_path
dataset_path = os.path.join(root_dir, task_name[task_id])
datalist_name = "dataset_task{}.json".format(task_id)
# get all training data
datalist = load_decathlon_datalist(
os.path.join(datalist_path, datalist_name), True, "training", dataset_path
)
# get modality info.
properties = load_decathlon_properties(
os.path.join(datalist_path, datalist_name), "modality"
)
dataset = Dataset(
data=datalist,
transform=LoadImaged(keys=["image", "label"]),
)
calculator = DatasetSummary(dataset, num_workers=4)
target_spacing = calculator.get_target_spacing()
print("spacing: ", target_spacing)
if properties["modality"]["0"] == "CT":
print("CT input, calculate statistics:")
calculator.calculate_statistics()
print("mean: ", calculator.data_mean, " std: ", calculator.data_std)
calculator.calculate_percentiles(
sampling_flag=True, interval=10, min_percentile=0.5, max_percentile=99.5
)
print(
"min: ",
calculator.data_min_percentile,
" max: ",
calculator.data_max_percentile,
)
else:
print("non CT input, skip calculating.")
def test_invert(self):
set_determinism(seed=0)
im_fname = make_nifti_image(create_test_image_3d(101, 100, 107, noise_max=100)[1]) # label image, discrete
data = [im_fname for _ in range(12)]
transform = Compose(
[
LoadImage(image_only=True),
EnsureChannelFirst(),
Orientation("RPS"),
Spacing(pixdim=(1.2, 1.01, 0.9), mode="bilinear", dtype=np.float32),
RandFlip(prob=0.5, spatial_axis=[1, 2]),
RandAxisFlip(prob=0.5),
RandRotate90(prob=0, spatial_axes=(1, 2)),
RandZoom(prob=0.5, min_zoom=0.5, max_zoom=1.1, keep_size=True),
RandRotate(prob=0.5, range_x=np.pi, mode="bilinear", align_corners=True, dtype=np.float64),
RandAffine(prob=0.5, rotate_range=np.pi, mode="nearest"),
ResizeWithPadOrCrop(100),
CastToType(dtype=torch.uint8),
]
)
# num workers = 0 for mac or gpu transforms
num_workers = 0 if sys.platform != "linux" or torch.cuda.is_available() else 2
dataset = Dataset(data, transform=transform)
self.assertIsInstance(transform.inverse(dataset[0]), MetaTensor)
loader = DataLoader(dataset, num_workers=num_workers, batch_size=1)
inverter = Invert(transform=transform, nearest_interp=True, device="cpu")
for d in loader:
d = decollate_batch(d)
for item in d:
orig = deepcopy(item)
i = inverter(item)
self.assertTupleEqual(orig.shape[1:], (100, 100, 100))
# check the nearest interpolation mode
torch.testing.assert_allclose(i.to(torch.uint8).to(torch.float), i.to(torch.float))
self.assertTupleEqual(i.shape[1:], (100, 101, 107))
# check labels match
reverted = i.detach().cpu().numpy().astype(np.int32)
original = LoadImage(image_only=True)(data[-1])
n_good = np.sum(np.isclose(reverted, original.numpy(), atol=1e-3))
reverted_name = i.meta["filename_or_obj"]
original_name = original.meta["filename_or_obj"]
self.assertEqual(reverted_name, original_name)
print("invert diff", reverted.size - n_good)
self.assertTrue((reverted.size - n_good) < 300000, f"diff. {reverted.size - n_good}")
set_determinism(seed=None)
def test_content(self):
data = [0] * 8
# set up engine
def _train_func(engine, batch):
pass
engine = Engine(_train_func)
# set up testing handler
val_data_loader = torch.utils.data.DataLoader(Dataset(data))
evaluator = TestEvaluator(torch.device("cpu:0"), val_data_loader)
saver = ValidationHandler(interval=2, validator=evaluator)
saver.attach(engine)
engine.run(data, max_epochs=5)
self.assertEqual(evaluator.state.max_epochs, 4)
self.assertEqual(evaluator.state.epoch_length, 8)
def test_seg_values(self):
with tempfile.TemporaryDirectory() as tempdir:
# prepare test datalist file
test_data = {
"name":
"Spleen",
"description":
"Spleen Segmentation",
"labels": {
"0": "background",
"1": "spleen"
},
"training": [
{
"image": "spleen_19.nii.gz",
"label": "spleen_19.nii.gz"
},
{
"image": "spleen_31.nii.gz",
"label": "spleen_31.nii.gz"
},
],
"test": ["spleen_15.nii.gz", "spleen_23.nii.gz"],
}
json_str = json.dumps(test_data)
file_path = os.path.join(tempdir, "test_data.json")
with open(file_path, "w") as json_file:
json_file.write(json_str)
data_list = DatasetFunc(data=file_path,
func=load_decathlon_datalist,
data_list_key="training",
base_dir=tempdir)
# partition dataset for train / validation
data_partition = DatasetFunc(
data=data_list,
func=lambda x, **kwargs: partition_dataset(x, **kwargs)[0],
num_partitions=2)
dataset = Dataset(data=data_partition, transform=None)
self.assertEqual(dataset[0]["image"],
os.path.join(tempdir, "spleen_19.nii.gz"))
self.assertEqual(dataset[0]["label"],
os.path.join(tempdir, "spleen_19.nii.gz"))
def test_dataloader_repeats(self):
dataset = Dataset(data=self.datalist, transform=self.transform)
dataloader = ThreadDataLoader(dataset=dataset,
batch_size=2,
num_workers=0,
repeats=2)
previous_batch = None
for d in dataloader:
self.assertEqual(d["image"][0], "spleen_19.nii.gz")
self.assertEqual(d["image"][1], "spleen_31.nii.gz")
if previous_batch is None:
previous_batch = d
else:
self.assertTrue(previous_batch is d,
"Batch object was not repeated")
previous_batch = None