def setUpClass(self):
np.random.seed(1234)
# Create 2D imgaging and segmentation data set
self.dataset2D = dict()
for i in range(0, 6):
img = np.random.rand(16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16) * 3
self.seg = seg.astype(int)
self.dataset2D["TEST.sample_" + str(i)] = (self.img, self.seg)
# Initialize Dictionary IO Interface
io_interface2D = Dictionary_interface(self.dataset2D,
classes=3,
three_dim=False)
# Initialize temporary directory
self.tmp_dir2D = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir2D.name, "batches")
# Initialize Data IO
self.data_io2D = Data_IO(io_interface2D,
input_path=os.path.join(self.tmp_dir2D.name),
output_path=os.path.join(self.tmp_dir2D.name),
batch_path=tmp_batches,
delete_batchDir=False)
# Initialize Preprocessor
self.pp2D = Preprocessor(self.data_io2D,
batch_size=2,
data_aug=None,
analysis="fullimage")
# Get sample list
self.sample_list2D = self.data_io2D.get_indiceslist()
# Create 3D imgaging and segmentation data set
self.dataset3D = dict()
for i in range(0, 6):
img = np.random.rand(16, 16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
self.seg = seg.astype(int)
self.dataset3D["TEST.sample_" + str(i)] = (self.img, self.seg)
# Initialize Dictionary IO Interface
io_interface3D = Dictionary_interface(self.dataset3D,
classes=3,
three_dim=True)
# Initialize temporary directory
self.tmp_dir3D = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir3D.name, "batches")
# Initialize Data IO
self.data_io3D = Data_IO(io_interface3D,
input_path=os.path.join(self.tmp_dir3D.name),
output_path=os.path.join(self.tmp_dir3D.name),
batch_path=tmp_batches,
delete_batchDir=False)
# Initialize Preprocessor
self.pp3D = Preprocessor(self.data_io3D,
batch_size=2,
data_aug=None,
analysis="fullimage")
# Get sample list
self.sample_list3D = self.data_io3D.get_indiceslist()
def setUpClass(self):
np.random.seed(1234)
# Create 2D imgaging and segmentation data set
self.dataset = dict()
for i in range(0, 2):
img = np.random.rand(16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16) * 2
self.seg = seg.astype(int)
self.dataset["TEST.sample_" + str(i)] = (self.img, self.seg)
# Initialize Dictionary IO Interface
io_interface = Dictionary_interface(self.dataset, classes=3,
three_dim=False)
# Initialize temporary directory
self.tmp_dir = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir.name, "batches")
# Initialize Data IO
self.data_io = Data_IO(io_interface,
input_path=os.path.join(self.tmp_dir.name),
output_path=os.path.join(self.tmp_dir.name),
batch_path=tmp_batches, delete_batchDir=False)
# Initialize Preprocessor
self.pp = Preprocessor(self.data_io, batch_size=1,
data_aug=None, analysis="fullimage")
# Initialize Neural Network
self.model = Neural_Network(self.pp)
# Get sample list
self.sample_list = self.data_io.get_indiceslist()
def test_SUBFUNCTIONS_preprocessing(self):
ds = dict()
for i in range(0, 10):
img = np.random.rand(16, 16, 16) * 255
img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
seg = seg.astype(int)
sample = (img, seg)
ds["TEST.sample_" + str(i)] = sample
io_interface = Dictionary_interface(ds, classes=3, three_dim=True)
self.tmp_dir = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir.name, "batches")
dataio = Data_IO(io_interface,
input_path="",
output_path="",
batch_path=tmp_batches,
delete_batchDir=False)
sf = [Resize((8, 8, 8)), Normalization(), Clipping(min=-1.0, max=0.0)]
pp = Preprocessor(dataio,
batch_size=1,
prepare_subfunctions=False,
analysis="fullimage",
subfunctions=sf)
sample_list = dataio.get_indiceslist()
batches = pp.run(sample_list, training=True, validation=False)
for i in range(0, 10):
img = batches[i][0]
seg = batches[i][1]
self.assertEqual(img.shape, (1, 8, 8, 8, 1))
self.assertEqual(seg.shape, (1, 8, 8, 8, 3))
self.assertTrue(np.min(img) >= -1.75 and np.max(img) <= 0.75)
self.tmp_dir.cleanup()
def setUpClass(self):
np.random.seed(1234)
# Create imgaging and segmentation data set
self.dataset = dict()
for i in range(0, 10):
img = np.random.rand(16, 16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
self.seg = seg.astype(int)
sample = (self.img, self.seg)
self.dataset["TEST.sample_" + str(i)] = sample
# Initialize Dictionary IO Interface
io_interface = Dictionary_interface(self.dataset,
classes=3,
three_dim=True)
# Initialize temporary directory
self.tmp_dir = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir.name, "batches")
# Initialize Data IO
self.data_io = Data_IO(io_interface,
input_path="",
output_path="",
batch_path=tmp_batches,
delete_batchDir=False)
# Initialize Data Augmentation
self.data_aug = Data_Augmentation()
# Get sample list
self.sample_list = self.data_io.get_indiceslist()
def setUpClass(self):
np.random.seed(1234)
# Create 2D imgaging and segmentation data set
self.dataset2D = dict()
for i in range(0, 10):
img = np.random.rand(16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16) * 2
self.seg = seg.astype(int)
self.dataset2D["TEST.sample_" + str(i)] = (self.img, self.seg)
# Initialize Dictionary IO Interface
io_interface2D = Dictionary_interface(self.dataset2D,
classes=3,
three_dim=False)
# Initialize temporary directory
self.tmp_dir2D = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir2D.name, "batches")
# Initialize Data IO
self.data_io2D = Data_IO(io_interface2D,
input_path="",
output_path="",
batch_path=tmp_batches,
delete_batchDir=False)
# Create 3D imgaging and segmentation data set
self.dataset3D = dict()
for i in range(0, 10):
img = np.random.rand(16, 16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
self.seg = seg.astype(int)
if i in range(8, 10): sample = (self.img, None)
else: sample = (self.img, self.seg)
self.dataset3D["TEST.sample_" + str(i)] = sample
# Initialize Dictionary IO Interface
io_interface3D = Dictionary_interface(self.dataset3D,
classes=3,
three_dim=True)
# Initialize temporary directory
self.tmp_dir3D = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir3D.name, "batches")
# Initialize Data IO
self.data_io3D = Data_IO(io_interface3D,
input_path="",
output_path="",
batch_path=tmp_batches,
delete_batchDir=False)
def test_DATAIO_BASE_getSampleList(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample_list = data_io.get_indiceslist()
self.assertEqual(len(sample_list), 10)
self.assertIn("TEST.sample_0", sample_list)
def test_DATAIO_BATCHES_sampleStorage(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=True,
load_pred=False)
data_io.backup_sample(sample)
self.assertEqual(len(os.listdir(self.tmp_batches)), 1)
data_io.batch_cleanup()
def test_DATAIO_SampleLoader_Imaging(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=False,
load_pred=False)
self.assertTrue(
np.array_equal(np.reshape(sample.img_data, (16, 16, 16)),
self.dataset["TEST.sample_0"][0]))
self.assertEqual(sample.img_data.shape, (16, 16, 16, 1))
def test_DATAIO_BATCHES_sampleLoading(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=True,
load_pred=False)
data_io.backup_sample(sample)
sample_new = data_io.load_sample_pickle(sample.index)
data_io.batch_cleanup()
self.assertTrue(np.array_equal(sample_new.img_data, sample.img_data))
self.assertTrue(np.array_equal(sample_new.seg_data, sample.seg_data))
def test_DATAIO_SampleLoader_Combined(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_3",
backup=False,
load_seg=True,
load_pred=True)
self.assertIsNotNone(sample.img_data)
self.assertIsNotNone(sample.seg_data)
self.assertIsNotNone(sample.pred_data)
self.assertEqual(sample.img_data.shape, sample.seg_data.shape)
self.assertEqual(sample.seg_data.shape, sample.pred_data.shape)
def test_DATAIO_BATCHES_loading(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=True,
load_pred=False)
data_io.backup_batches(sample.img_data, sample.seg_data, "abc")
img = data_io.batch_load(pointer="abc", img=True)
self.assertTrue(np.array_equal(sample.img_data, img))
seg = data_io.batch_load(pointer="abc", img=False)
self.assertTrue(np.array_equal(sample.seg_data, seg))
data_io.batch_cleanup()
def test_DATAIO_BATCHES_cleanup(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=True,
load_pred=False)
data_io.backup_batches(sample.img_data, sample.seg_data, "abc")
data_io.backup_batches(sample.img_data, sample.seg_data, "def")
data_io.backup_batches(sample.img_data, None, pointer="ghi")
self.assertEqual(len(os.listdir(self.tmp_batches)), 5)
data_io.batch_cleanup(pointer="def")
self.assertEqual(len(os.listdir(self.tmp_batches)), 3)
data_io.batch_cleanup()
self.assertEqual(len(os.listdir(self.tmp_batches)), 0)
def test_SUBFUNCTIONS_postprocessing(self):
ds = dict()
for i in range(0, 10):
img = np.random.rand(16, 16, 16) * 255
img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
seg = seg.astype(int)
sample = (img, seg)
ds["TEST.sample_" + str(i)] = sample
io_interface = Dictionary_interface(ds, classes=3, three_dim=True)
self.tmp_dir = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir.name, "batches")
dataio = Data_IO(io_interface,
input_path="",
output_path="",
batch_path=tmp_batches,
delete_batchDir=False)
sf = [Resize((9, 9, 9)), Normalization(), Clipping(min=-1.0, max=0.0)]
pp = Preprocessor(dataio,
batch_size=1,
prepare_subfunctions=False,
analysis="patchwise-grid",
subfunctions=sf,
patch_shape=(4, 4, 4))
sample_list = dataio.get_indiceslist()
for index in sample_list:
sample = dataio.sample_loader(index)
for sf in pp.subfunctions:
sf.preprocessing(sample, training=False)
pp.cache["shape_" + str(index)] = sample.img_data.shape
sample.seg_data = np.random.rand(9, 9, 9) * 3
sample.seg_data = sample.seg_data.astype(int)
sample.seg_data = to_categorical(sample.seg_data, num_classes=3)
data_patches = pp.analysis_patchwise_grid(sample,
training=True,
data_aug=False)
seg_list = []
for i in range(0, len(data_patches)):
seg_list.append(data_patches[i][1])
seg = np.stack(seg_list, axis=0)
self.assertEqual(seg.shape, (27, 4, 4, 4, 3))
pred = pp.postprocessing(index, seg)
self.assertEqual(pred.shape, (16, 16, 16))
self.tmp_dir.cleanup()
def test_DATAIO_BASE_savePrediction(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path=os.path.join(self.tmp_dir.name, "pred"),
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=True,
load_pred=False)
self.assertIsNone(sample.pred_data)
data_io.save_prediction(sample.seg_data, sample.index)
self.assertTrue(os.path.exists(os.path.join(self.tmp_dir.name,
"pred")))
sample = data_io.sample_loader("TEST.sample_0",
backup=False,
load_seg=True,
load_pred=True)
self.assertTrue(np.array_equal(sample.seg_data, sample.pred_data))
def test_EVALUATION_leaveOneOut(self):
# Create 3D imgaging and segmentation data set
self.dataset3D = dict()
for i in range(0, 6):
img = np.random.rand(16, 16, 16) * 255
self.img = img.astype(int)
seg = np.random.rand(16, 16, 16) * 3
self.seg = seg.astype(int)
self.dataset3D["TEST.sample_" + str(i)] = (self.img, self.seg)
# Initialize Dictionary IO Interface
io_interface3D = Dictionary_interface(self.dataset3D,
classes=3,
three_dim=True)
# Initialize temporary directory
self.tmp_dir3D = tempfile.TemporaryDirectory(prefix="tmp.CESP.")
tmp_batches = os.path.join(self.tmp_dir3D.name, "batches")
# Initialize Data IO
self.data_io3D = Data_IO(io_interface3D,
input_path=os.path.join(self.tmp_dir3D.name),
output_path=os.path.join(self.tmp_dir3D.name),
batch_path=tmp_batches,
delete_batchDir=False)
# Initialize Preprocessor
self.pp3D = Preprocessor(self.data_io3D,
batch_size=2,
data_aug=None,
analysis="fullimage")
# Initialize Neural Network
model = Neural_Network(self.pp3D)
# Get sample list
self.sample_list3D = self.data_io3D.get_indiceslist()
eval_path = os.path.join(self.tmp_dir3D.name, "evaluation")
leave_one_out(self.sample_list3D,
model,
epochs=3,
iterations=None,
evaluation_path=eval_path,
callbacks=[])
self.assertTrue(os.path.exists(eval_path))
# Cleanup stuff
self.tmp_dir3D.cleanup()
def test_DATAIO_SampleLoader_Prediction(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
sample = data_io.sample_loader("TEST.sample_5",
backup=False,
load_seg=False,
load_pred=True)
self.assertTrue(
np.array_equal(np.reshape(sample.pred_data, (16, 16, 16)),
self.dataset["TEST.sample_5"][2]))
self.assertEqual(sample.pred_data.shape, (16, 16, 16, 1))
self.assertIsNotNone(sample.img_data)
self.assertIsNone(sample.seg_data)
with self.assertRaises(Exception):
sample = data_io.sample_loader("TEST.sample_2",
backup=False,
load_seg=False,
load_pred=True)
def test_DATAIO_BASE_create(self):
data_io = Data_IO(self.io_interface,
input_path="",
output_path="",
batch_path=self.tmp_batches,
delete_batchDir=False)
