def test_get_ilastik_weights(self): img_path = os.path.join(base_path, '../test_data/ilastik') lbl_path = os.path.join(base_path, '../test_data/ilastik/ilastik-1.2.ilp') c = IlastikConnector(img_path, lbl_path) d = Dataset(c) size = (8, 2, 4) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) m.augment_by_flipping(False) mini = next(m) weights = mini.weights() # label 1 at position (4,1,3) self.assertTrue(weights[0, 0, 4, 1, 3] == 1) # label 2 at position (1,1,1) self.assertTrue(weights[0, 1, 1, 1, 1] == 1) # label 3 at position (1,1,1) self.assertTrue(weights[0, 2, 7, 1, 1] == 1)
def test_set_augmentation(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 3, 4) pad = (1, 2, 2) m = TrainingBatch(d, size, padding_zxy=pad) self.assertEqual(m.augmentation, {'flip'}) m.augment_by_rotation(True) self.assertEqual(m.augmentation, {'flip', 'rotate'}) self.assertEqual(m.rotation_range, (-45, 45)) m.augment_by_shear(True) self.assertEqual(m.augmentation, {'flip', 'rotate', 'shear'}) self.assertEqual(m.shear_range, (-5, 5)) m.augment_by_flipping(False) self.assertEqual(m.augmentation, {'rotate', 'shear'})
def test_getitem_multichannel_labels(self): # define data loacations pixel_image_dir = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_image_dir = os.path.join( base_path, '../test_data/tiffconnector_1/labels_multichannel/') savepath = tempfile.TemporaryDirectory() tile_size = (1, 5, 4) # size of network output layer in zxy # padding of network input layer in zxy, in respect to output layer padding = (0, 2, 2) # make training_batch mb and prediction interface p with # TiffConnector binding c = TiffConnector(pixel_image_dir, label_image_dir, savepath=savepath.name) m = TrainingBatch(Dataset(c), tile_size, padding_zxy=padding) for counter, mini in enumerate(m): # shape is (6, 6, 1, 5, 4): # batchsize 6 , 6 label-classes, 1 z, 5 x, 4 y weights = mini.weights() # shape is (6, 3, 1, 9, 8): # batchsize 6, 6 channels, 1 z, 9 x, 4 y (more xy due to padding) pixels = mini.pixels() self.assertEqual(weights.shape, (6, 6, 1, 5, 4)) self.assertEqual(pixels.shape, (6, 3, 1, 9, 8)) # apply training on mini.pixels and mini.weights goes here if counter > 10: # m is infinite break
def test_split(self): img_path = os.path.join( base_path, '../test_data/ilastik/pixels_ilastik-multiim-1.2') label_path = os.path.join( base_path, '../test_data/ilastik/ilastik-multiim-1.2.ilp') c = IlastikConnector(img_path, label_path) d = Dataset(c) size = (1, 1, 1) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) m2 = m.split(0.3) self.assertTrue( len(m.tile_pos_for_label[1]) > len(m2.tile_pos_for_label[1])) self.assertTrue( len(m.tile_pos_for_label[2]) > len(m2.tile_pos_for_label[2])) # check for no overlap self.assertEqual( 0, len(set(m.tile_pos_for_label[1]) & set(m2.tile_pos_for_label[1]))) self.assertEqual( 0, len(set(m2.tile_pos_for_label[1]) & set(m.tile_pos_for_label[1])))
def test_get_ilastik_weights2(self): pth = os.path.join(base_path, '../test_data/ilastik/dimensionstest') img_path = os.path.join(pth, 'images') lbl_path = os.path.join(pth, 'x15_y10_z2_c4_classes2.ilp') c = IlastikConnector(img_path, lbl_path) d = Dataset(c) size = (2, 15, 10) # this is the size of the whole image pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) m.augment_by_flipping(False) mini = next(m) weights = mini.weights() print(weights[0, :, :, :, :]) # all samples of the batch should be identical, # because tilesize=imsize assert_array_equal(weights[0, :, :, :, :], weights[1, :, :, :, :]) # weight positions from labelvalue 1 and 2 pos = [[0, 0, 2, 1], [0, 0, 2, 1], [0, 0, 8, 6], [0, 0, 9, 6], [0, 1, 4, 3], [1, 0, 2, 2], [1, 0, 3, 2], [1, 0, 8, 7], [1, 0, 9, 7]] for p in pos: self.assertEqual(weights[0, p[0], p[1], p[2], p[3]], 1) self.assertEqual(weights[1, p[0], p[1], p[2], p[3]], 1)
def test_tile_positions_decay(self): img_path = os.path.join( base_path, '../test_data/ilastik/pixels_ilastik-multiim-1.2') label_path = os.path.join( base_path, '../test_data/ilastik/ilastik-multiim-1.2.ilp') c = IlastikConnector(img_path, label_path) d = Dataset(c) size = (2, 6, 4) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) n_pos_lbl_1 = len(m.tile_pos_for_label[1]) n_pos_lbl_2 = len(m.tile_pos_for_label[2]) assert not m.tile_pos_for_label[1] is m.tile_pos_for_label[2] for _ in range(800): next(m) n_pos_lbl_1_after = len(m.tile_pos_for_label[1]) n_pos_lbl_2_after = len(m.tile_pos_for_label[2]) self.assertTrue(n_pos_lbl_1 > n_pos_lbl_1_after) self.assertTrue(n_pos_lbl_2 > n_pos_lbl_2_after)
def test_get_pixels_dimension_order(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (2, 5, 4) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) b = next(m) p = m.pixels() w = m.weights() self.assertEqual(p.shape, (3, 3, 2, 5, 4)) self.assertEqual(w.shape, (3, 3, 2, 5, 4)) m.set_pixel_dimension_order('bczxy') p = m.pixels() w = m.weights() self.assertEqual(p.shape, (3, 3, 2, 5, 4)) self.assertEqual(w.shape, (3, 3, 2, 5, 4)) m.set_pixel_dimension_order('bzxyc') p = m.pixels() w = m.weights() self.assertEqual(p.shape, (3, 2, 5, 4, 3)) self.assertEqual(w.shape, (3, 2, 5, 4, 3))
def test_normalize_zscore(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 2, 2) pad = (0, 0, 0) batchsize = 2 nr_channels = 3 nz = 1 nx = 4 ny = 5 m = TrainingBatch(d, size, padding_zxy=pad, batch_size=len(d.label_values())) m.set_normalize_mode('local_z_score') pixels = np.zeros((batchsize, nr_channels, nz, nx, ny)) pixels[:, 0, :, :, :] = 1 pixels[:, 1, :, :, :] = 2 pixels[:, 2, :, :, :] = 3 p_norm = m._normalize(pixels) self.assertTrue((p_norm == 0).all()) # add variation pixels[:, 0, 0, 0, 0] = 2 pixels[:, 0, 0, 0, 1] = 0 pixels[:, 1, 0, 0, 0] = 3 pixels[:, 1, 0, 0, 1] = 1 pixels[:, 2, 0, 0, 0] = 4 pixels[:, 2, 0, 0, 1] = 2 p_norm = m._normalize(pixels) assert_array_equal(p_norm[:, 0, :, :, :], p_norm[:, 1, :, :, :]) assert_array_equal(p_norm[:, 0, :, :, :], p_norm[:, 2, :, :, :]) val = np.array([[[3.16227766, -3.16227766, 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]]]) assert_array_almost_equal(val, p_norm[0, 0, :, :, :])
def test_random_tile(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 3, 4) pad = (1, 2, 2) m = TrainingBatch(d, size, padding_zxy=pad) m._random_tile(for_label=1)
def test_pixel_format_is_float(self): p = os.path.join(base_path, '../test_data/ilastik/dimensionstest') img_path = os.path.join(p, 'images') label_path = os.path.join(p, 'x15_y10_z2_c4_classes2.ilp') c = IlastikConnector(img_path, label_path) d = Dataset(c) size = (2, 4, 3) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) mini = next(m) p = mini._pixels self.assertTrue(np.issubdtype(p.dtype, np.float))
def _configure_minibatch_data(self, input_tile_size_zxy, output_tile_size_zxy): padding_zxy = tuple( ((np.array(input_tile_size_zxy) - np.array(output_tile_size_zxy)) / 2).astype(np.int)) self.data_val = None self.data = TrainingBatch(self.dataset, output_tile_size_zxy, padding_zxy=padding_zxy) self.data.set_normalize_mode('local') self.data.set_pixel_dimension_order('bzxyc') next(self.data) self.output_tile_size_zxy = output_tile_size_zxy self.padding_zxy = padding_zxy
def test_shape_data_split(self): import logging logging.basicConfig(level=logging.INFO) img_path = os.path.join(base_path, '../test_data/shapes/pixels/*') label_path = os.path.join(base_path, '../test_data/shapes/labels.ilp') c = IlastikConnector(img_path, label_path) d = Dataset(c) size = (1, 50, 50) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) m2 = m.split(0.001)
def test_normalize_global(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 2, 2) pad = (0, 0, 0) batchsize = 2 nr_channels = 3 nz = 1 nx = 4 ny = 5 val = np.array( [[[0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333], [0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333], [0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333], [0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333]], [[0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667], [0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667], [0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667], [0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667]], [[1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.]]]) m = TrainingBatch(d, size, padding_zxy=pad, batch_size=len(d.label_values())) m.set_normalize_mode('global', minmax=[0, 3]) pixels = np.zeros((batchsize, nr_channels, nz, nx, ny)) pixels[:, 0, :, :, :] = 1 pixels[:, 1, :, :, :] = 2 pixels[:, 2, :, :, :] = 3 p_norm = m._normalize(pixels) pprint(p_norm) print(pixels.shape) print(p_norm.shape) assert_array_almost_equal(val, p_norm[0, :, 0, :, :])
def test_normalize_global_auto(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 5, 4) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad, batch_size=len(d.label_values())) assert m.global_norm_minmax is None m.set_normalize_mode('global') assert len(m.global_norm_minmax) == 3
def test_set_pixel_dimension_order(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 3, 4) pad = (1, 2, 2) m = TrainingBatch(d, size, padding_zxy=pad) m.set_pixel_dimension_order('bczxy') self.assertEqual([0, 1, 2, 3, 4], m.pixel_dimension_order) m.set_pixel_dimension_order('bzxyc') self.assertEqual([0, 4, 1, 2, 3], m.pixel_dimension_order)
def test_getitem(self): # define data locations pixel_image_dir = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_image_dir = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') tile_size = (1, 5, 4) # size of network output layer in zxy # padding of network input layer in zxy, in respect to output layer padding = (0, 2, 2) c = TiffConnector(pixel_image_dir, label_image_dir, savepath=self.tmpdir) d = Dataset(c) m = TrainingBatch(d, tile_size, padding_zxy=padding, batch_size=len(d.label_values())) for counter, mini in enumerate(m): # shape is (6, 3, 1, 5, 4): # batchsize 6, 3 label-classes, 1 z, 5 x, 4 y weights = mini.weights() # shape is (6, 3, 1, 9, 8): # batchsize 6, 3 channels, 1 z, 9 x, 4 y (more xy due to padding) pixels = mini.pixels() self.assertEqual(weights.shape, (3, 3, 1, 5, 4)) self.assertEqual(pixels.shape, (3, 3, 1, 9, 8)) # apply training on mini.pixels and mini.weights goes here if counter > 10: # m is infinite break
def test_remove_unlabeled_tiles(self): img_path = os.path.join( base_path, '../test_data/ilastik/pixels_ilastik-multiim-1.2') label_path = os.path.join( base_path, '../test_data/ilastik/ilastik-multiim-1.2.ilp') c = IlastikConnector(img_path, label_path) d = Dataset(c) size = (2, 6, 4) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) n_pos_lbl_1 = len(m.tile_pos_for_label[1]) n_pos_lbl_2 = len(m.tile_pos_for_label[2]) m.remove_unlabeled_tiles() n_pos_lbl_1_after = len(m.tile_pos_for_label[1]) n_pos_lbl_2_after = len(m.tile_pos_for_label[2]) self.assertTrue(n_pos_lbl_1 > n_pos_lbl_1_after) self.assertTrue(n_pos_lbl_2 > n_pos_lbl_2_after)