def test_DATAGENERATOR_runTraining(self): pp_fi = Preprocessor(self.data_io, batch_size=4, data_aug=self.data_aug, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=None) self.assertEqual(len(data_gen), 3) for batch in data_gen: self.assertIsInstance(batch, tuple) self.assertEqual(batch[0].shape, (4, 16, 16, 16, 1)) self.assertEqual(batch[1].shape, (4, 16, 16, 16, 3)) pp_pc = Preprocessor(self.data_io, batch_size=3, data_aug=self.data_aug, prepare_subfunctions=False, prepare_batches=False, patch_shape=(5, 5, 5), analysis="patchwise-crop") data_gen = DataGenerator(self.sample_list, pp_pc, training=True, shuffle=False, iterations=None) self.assertEqual(len(data_gen), 4) for batch in data_gen: self.assertIsInstance(batch, tuple) self.assertEqual(batch[0].shape, (3, 5, 5, 5, 1)) self.assertEqual(batch[1].shape, (3, 5, 5, 5, 3))
def evaluate(self, training_samples, validation_samples, epochs=20, iterations=None, callbacks=[]): # Initialize a Keras Data Generator for generating Training data dataGen_training = DataGenerator(training_samples, self.preprocessor, training=True, validation=False, shuffle=self.shuffle_batches, iterations=iterations) # Initialize a Keras Data Generator for generating Validation data dataGen_validation = DataGenerator(validation_samples, self.preprocessor, training=True, validation=True, shuffle=self.shuffle_batches) # Run training & validation process with the Keras fit history = self.model.fit(dataGen_training, validation_data=dataGen_validation, callbacks=callbacks, epochs=epochs, workers=self.workers, max_queue_size=self.batch_queue_size) # Clean up temporary files if necessary if self.preprocessor.prepare_batches or self.preprocessor.prepare_subfunctions: self.preprocessor.data_io.batch_cleanup() # Return the training & validation history return history
def test_DATAGENERATOR_iterations(self): pp_fi = Preprocessor(self.data_io, batch_size=1, data_aug=None, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=None) self.assertEqual(10, len(data_gen)) data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=5) self.assertEqual(5, len(data_gen)) data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=50) self.assertEqual(50, len(data_gen)) data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=100) self.assertEqual(100, len(data_gen))
def predict(self, sample_list, return_output=False, activation_output=False): # Initialize result array for direct output if return_output: results = [] # Iterate over each sample for sample in sample_list: # Initialize Keras Data Generator for generating batches dataGen = DataGenerator([sample], self.preprocessor, training=False, validation=False, shuffle=False, iterations=None) # Run prediction process with Keras predict pred_list = [] for batch in dataGen: pred_batch = self.model.predict_on_batch(batch) pred_list.append(pred_batch) pred_seg = np.concatenate(pred_list, axis=0) # Postprocess prediction pred_seg = self.preprocessor.postprocessing( sample, pred_seg, activation_output) # Backup predicted segmentation if return_output: results.append(pred_seg) else: self.preprocessor.data_io.save_prediction(pred_seg, sample) # Clean up temporary files if necessary if self.preprocessor.prepare_batches or self.preprocessor.prepare_subfunctions: self.preprocessor.data_io.batch_cleanup() # Output predictions results if direct output modus is active if return_output: return results
def test_DATAGENERATOR_create(self): pp_fi = Preprocessor(self.data_io, batch_size=4, data_aug=self.data_aug, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=False, validation=False, shuffle=False, iterations=None) self.assertIsInstance(data_gen, DataGenerator)
def test_DATAGENERATOR_augcyling(self): data_aug = Data_Augmentation(cycles=20) pp_fi = Preprocessor(self.data_io, batch_size=4, data_aug=data_aug, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=None) self.assertEqual(50, len(data_gen))
def train(self, sample_list, epochs=20, iterations=None, callbacks=[]): # Initialize Keras Data Generator for generating batches dataGen = DataGenerator(sample_list, self.preprocessor, training=True, validation=False, shuffle=self.shuffle_batches, iterations=iterations) # Run training process with Keras fit_generator self.model.fit_generator(generator=dataGen, epochs=epochs, callbacks=callbacks, workers=self.workers, max_queue_size=self.batch_queue_size) # Clean up temporary files if necessary if self.preprocessor.prepare_batches or self.preprocessor.prepare_subfunctions: self.preprocessor.data_io.batch_cleanup()
def test_DATAGENERATOR_prepareData(self): pp_fi = Preprocessor(self.data_io, batch_size=4, data_aug=None, prepare_subfunctions=True, prepare_batches=True, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=True, iterations=None) self.assertEqual(len(data_gen), 3) for batch in data_gen: self.assertIsInstance(batch, tuple) self.assertEqual(batch[0].shape[1:], (16, 16, 16, 1)) self.assertEqual(batch[1].shape[1:], (16, 16, 16, 3)) self.assertIn(batch[0].shape[0], [2, 4])
def predict_augmentated(self, sample): if self.preprocessor.data_augmentation is None: raise ValueError("Inference Augmentation requires a " + \ "Data Augmentation class instance!") else: data_aug = self.preprocessor.data_augmentation # Initialize result array for the augmentated predictions results = [] # Activate augmentation inferene data_aug.infaug = True if self.three_dim: flip_list = data_aug.infaug_flip_list else: flip_list = data_aug.infaug_flip_list[:-1] # Compute inference for each flip augmentation / for each axis for flip_axis in flip_list: # Update flip axis data_aug.infaug_flip_current = flip_axis # Initialize Keras Data Generator for generating batches dataGen = DataGenerator([sample], self.preprocessor, training=False, validation=False, shuffle=False, iterations=None) sampleObj = self.preprocessor.data_io.sample_loader(sample, load_seg=False) #TODO optimize # Run prediction process with Keras predict pred_list = [] for batch in dataGen: pred_batch = self.model.predict_on_batch(batch) pred_list.append(pred_batch) pred_seg = np.concatenate(pred_list, axis=0) # Postprocess prediction pred_seg = self.preprocessor.postprocessing(sampleObj, pred_seg, activation_output=True) # Backup predicted segmentation for current augmentation results.append(pred_seg) # Reset inference augmentation modus data_aug.infaug = False data_aug.infaug_flip_current = None # Return result array return results
def test_DATAGENERATOR_shuffle(self): pp_fi = Preprocessor(self.data_io, batch_size=1, data_aug=None, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=None) list_ordered = [] for batch in data_gen: list_ordered.append(batch) for batch in data_gen: list_ordered.append(batch) data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=True, iterations=None) list_shuffled = [] for batch in data_gen: list_shuffled.append(batch) data_gen.on_epoch_end() for batch in data_gen: list_shuffled.append(batch) size = len(data_gen) o_counter = 0 s_counter = 0 for i in range(0, size): oa_img = list_ordered[i][0] oa_seg = list_ordered[i][1] ob_img = list_ordered[i + size][0] ob_seg = list_ordered[i + size][1] sa_img = list_shuffled[i][0] sa_seg = list_shuffled[i][1] sb_img = list_shuffled[i + size][0] sb_seg = list_shuffled[i + size][1] if np.array_equal(oa_img, ob_img) and \ np.array_equal(oa_seg, ob_seg): o_counter += 1 if not np.array_equal(sa_img, sb_img) and \ not np.array_equal(sa_seg, sb_seg): s_counter += 1 o_ratio = o_counter / size self.assertTrue(o_ratio == 1.0) s_ratio = s_counter / size self.assertTrue(1.0 >= s_ratio and s_ratio >= 0.5)
def test_DATAGENERATOR_consistency(self): pp_fi = Preprocessor(self.data_io, batch_size=1, data_aug=None, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator(self.sample_list, pp_fi, training=True, shuffle=False, iterations=None) i = 0 for batch in data_gen: sample = self.data_io.sample_loader(self.sample_list[i], load_seg=True) self.assertTrue(np.array_equal(batch[0][0], sample.img_data)) seg = to_categorical(sample.seg_data, num_classes=3) self.assertTrue(np.array_equal(batch[1][0], seg)) i += 1
def test_DATAGENERATOR_inferenceAug(self): data_aug = Data_Augmentation() pp_fi = Preprocessor(self.data_io, batch_size=4, data_aug=data_aug, prepare_subfunctions=False, prepare_batches=False, analysis="fullimage") data_gen = DataGenerator([self.sample_list[0]], pp_fi, training=False, shuffle=False, iterations=None) pred_list_inactive = [] for batch in data_gen: pred_list_inactive.append(batch) data_aug.infaug = True pred_list_active = [] for batch in data_gen: pred_list_active.append(batch) for i in range(0, len(pred_list_active)): ba = pred_list_active[i] bi = pred_list_inactive[i] self.assertFalse(np.array_equal(ba, bi))
# Create a pixel value normalization Subfunction for z-score scaling sf_zscore = Normalization(mode="z-score") # Assemble Subfunction classes into a list sf = [sf_clipping, sf_normalize, sf_resample, sf_zscore] # Create and configure the Preprocessor class pp = Preprocessor(data_io, data_aug=data_aug, batch_size=2, subfunctions=sf, prepare_subfunctions=True, prepare_batches=False, analysis="fullimage", patch_shape=(160, 160, 80)) # Adjust the patch overlap for predictions pp.patchwise_overlap = (80, 80, 40) # Initialize Keras Data Generator for generating batches from miscnn.neural_network.data_generator import DataGenerator dataGen = DataGenerator(sample_list, pp, training=False, validation=False, shuffle=False) x = [] y = [] z = [] for batch in dataGen: print("Batch:", batch.shape) x.append(batch.shape[1]) y.append(batch.shape[2]) z.append(batch.shape[3]) print("Mean:") print(np.mean(x), np.mean(y), np.mean(z)) print(np.median(x), np.median(y), np.median(z))