def test_PREPROCESSOR_BASE_prepareSubfunctions(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D,
batch_size=1,
analysis="fullimage",
prepare_subfunctions=True)
pp.run_subfunctions(sample_list[0:8], training=True)
pp.run(sample_list[0:8], training=True, validation=False)
pp.run_subfunctions(sample_list[8:10], training=False)
pp.run(sample_list[8:10], training=False, validation=False)
def test_SUBFUNCTIONS_preprocessing(self):
ds = dict()
for i in range(0, 10):
img = np.random.rand(16, 16, 16) * 255
img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
seg = seg.astype(int)
sample = (img, seg)
ds["TEST.sample_" + str(i)] = sample
io_interface = Dictionary_interface(ds, classes=3, three_dim=True)
self.tmp_dir = tempfile.TemporaryDirectory(prefix="tmp.miscnn.")
tmp_batches = os.path.join(self.tmp_dir.name, "batches")
dataio = Data_IO(io_interface, input_path="", output_path="",
batch_path=tmp_batches, delete_batchDir=False)
sf = [Resize((8,8,8)), Normalization(), Clipping(min=-1.0, max=0.0)]
pp = Preprocessor(dataio, data_aug=None, batch_size=1,
prepare_subfunctions=False, analysis="fullimage",
subfunctions=sf)
sample_list = dataio.get_indiceslist()
batches = pp.run(sample_list, training=True, validation=False)
for i in range(0, 10):
img = batches[i][0]
seg = batches[i][1]
self.assertEqual(img.shape, (1,8,8,8,1))
self.assertEqual(seg.shape, (1,8,8,8,3))
self.assertTrue(np.min(img) >= -1.0 and np.max(img) <= 0.0)
self.tmp_dir.cleanup()
def test_SUBFUNCTIONS_prepare_MULTIPROCESSING(self):
ds = dict()
for i in range(0, 5):
img = np.random.rand(16, 16, 16) * 255
img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
seg = seg.astype(int)
sample = (img, seg)
ds["TEST.sample_" + str(i)] = sample
io_interface = Dictionary_interface(ds, classes=3, three_dim=True)
self.tmp_dir = tempfile.TemporaryDirectory(prefix="tmp.miscnn.")
tmp_batches = os.path.join(self.tmp_dir.name, "batches")
dataio = Data_IO(io_interface, input_path="", output_path="",
batch_path=tmp_batches, delete_batchDir=False)
sf = [Resize((8,8,8)), Normalization(), Clipping(min=-1.0, max=0.0)]
pp = Preprocessor(dataio, batch_size=1, prepare_subfunctions=True,
analysis="fullimage", subfunctions=sf,
use_multiprocessing=True)
pp.mp_threads = 4
sample_list = dataio.get_indiceslist()
pp.run_subfunctions(sample_list, training=True)
batches = pp.run(sample_list, training=True, validation=False)
self.assertEqual(len(os.listdir(tmp_batches)), 5)
for i in range(0, 5):
file_prepared_subfunctions = os.path.join(tmp_batches,
str(pp.data_io.seed) + ".TEST.sample_" + str(i) + ".pickle")
self.assertTrue(os.path.exists(file_prepared_subfunctions))
img = batches[i][0]
seg = batches[i][1]
self.assertIsNotNone(img)
self.assertIsNotNone(seg)
self.assertEqual(img.shape, (1,8,8,8,1))
self.assertEqual(seg.shape, (1,8,8,8,3))
def test_PREPROCESSOR_BASE_run(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D,
data_aug=None,
batch_size=1,
analysis="fullimage")
batches = pp.run(sample_list[8:10], training=False, validation=False)
self.assertEqual(len(batches), 2)
self.assertEqual(batches[0][0].shape, (1, 16, 16, 16, 1))
self.assertIsNone(batches[0][1])
batches = pp.run(sample_list[0:8], training=True, validation=False)
self.assertEqual(batches[0][0].shape, (1, 16, 16, 16, 1))
self.assertEqual(batches[0][1].shape, (1, 16, 16, 16, 3))
batches = pp.run(sample_list[0:8], training=True, validation=True)
self.assertEqual(batches[0][0].shape, (1, 16, 16, 16, 1))
self.assertEqual(batches[0][1].shape, (1, 16, 16, 16, 3))
def test_PREPROCESSOR_patchwisecrop_3D(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D,
data_aug=None,
batch_size=1,
analysis="patchwise-crop",
patch_shape=(4, 4, 4))
batches = pp.run(sample_list[0:3], training=True, validation=False)
self.assertEqual(len(batches), 3)
batches = pp.run(sample_list[0:1], training=False, validation=False)
self.assertEqual(len(batches), 64)
sample = self.data_io3D.sample_loader(sample_list[0], load_seg=True)
sample.seg_data = to_categorical(sample.seg_data,
num_classes=sample.classes)
ready_data = pp.analysis_patchwise_crop(sample, data_aug=False)
self.assertEqual(len(ready_data), 1)
self.assertEqual(ready_data[0][0].shape, (4, 4, 4, 1))
self.assertEqual(ready_data[0][1].shape, (4, 4, 4, 3))
def test_PREPROCESSOR_BASE_dataaugmentation(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D, batch_size=1, analysis="fullimage")
batches = pp.run(sample_list[8:10], training=False, validation=False)
self.assertEqual(len(batches), 2)
self.assertEqual(batches[0][0].shape, (1, 16, 16, 16, 1))
self.assertIsNone(batches[0][1])
sample = self.data_io3D.sample_loader(sample_list[8], load_seg=False)
self.assertFalse(np.array_equal(batches[0][0], sample.img_data))
def test_PREPROCESSOR_fullimage_3D(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D,
data_aug=None,
batch_size=2,
analysis="fullimage")
batches = pp.run(sample_list[0:3], training=True, validation=False)
self.assertEqual(len(batches), 2)
batches = pp.run(sample_list[0:1], training=False, validation=False)
self.assertEqual(len(batches), 1)
sample = self.data_io3D.sample_loader(sample_list[0], load_seg=True)
sample.seg_data = to_categorical(sample.seg_data,
num_classes=sample.classes)
ready_data = pp.analysis_fullimage(sample,
data_aug=False,
training=True)
self.assertEqual(len(ready_data), 1)
self.assertEqual(ready_data[0][0].shape, (16, 16, 16, 1))
self.assertEqual(ready_data[0][1].shape, (16, 16, 16, 3))
def test_PREPROCESSOR_postprocessing_(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D,
batch_size=1,
analysis="fullimage",
data_aug=None)
batches = pp.run(sample_list[0:3], training=True, validation=False)
for i in range(0, 3):
pred_postprec = pp.postprocessing(sample_list[i], batches[i][1])
self.assertEqual(pred_postprec.shape, (16, 16, 16))
sam = self.data_io3D.sample_loader(sample_list[i], load_seg=True)
self.assertTrue(
np.array_equal(pred_postprec,
np.reshape(sam.seg_data, (16, 16, 16))))
def test_PREPROCESSOR_BASE_batchsizes(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D, batch_size=1, analysis="fullimage")
batches = pp.run(sample_list[0:8], training=True, validation=False)
self.assertEqual(len(batches), 8)
self.assertEqual(batches[0][0].shape, (1, 16, 16, 16, 1))
pp = Preprocessor(self.data_io3D, batch_size=2, analysis="fullimage")
batches = pp.run(sample_list[0:8], training=True, validation=False)
self.assertEqual(len(batches), 4)
self.assertEqual(batches[0][0].shape, (2, 16, 16, 16, 1))
pp = Preprocessor(self.data_io3D, batch_size=3, analysis="fullimage")
batches = pp.run(sample_list[0:8], training=True, validation=False)
self.assertEqual(len(batches), 3)
self.assertEqual(batches[0][0].shape, (3, 16, 16, 16, 1))
self.assertEqual(batches[-1][0].shape, (2, 16, 16, 16, 1))
pp = Preprocessor(self.data_io3D, batch_size=8, analysis="fullimage")
batches = pp.run(sample_list[0:8], training=True, validation=False)
self.assertEqual(len(batches), 1)
self.assertEqual(batches[0][0].shape, (8, 16, 16, 16, 1))
pp = Preprocessor(self.data_io3D, batch_size=100, analysis="fullimage")
batches = pp.run(sample_list[0:8], training=True, validation=False)
self.assertEqual(len(batches), 1)
self.assertEqual(batches[0][0].shape, (8, 16, 16, 16, 1))
def test_PREPROCESSOR_BASE_prepareBatches(self):
sample_list = self.data_io3D.get_indiceslist()
pp = Preprocessor(self.data_io3D,
batch_size=1,
analysis="fullimage",
prepare_batches=True)
batch_pointer = pp.run(sample_list[0:8],
training=True,
validation=False)
self.assertEqual(batch_pointer, 7)
tmp_batches = os.path.join(self.tmp_dir3D.name, "batches")
batch_list = []
for batch_file in os.listdir(tmp_batches):
if batch_file.startswith(str(pp.data_io.seed)):
batch_list.append(batch_file)
self.assertEqual(len(batch_list), 16)
