def test_sync_label_counts(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) c1, c2 = c.split(1. / 3.) d1 = Dataset(c1) d2 = Dataset(c2) d1.sync_label_counts(d2) np.testing.assert_array_equal(set(d1.label_counts.keys()), set(d2.label_counts.keys())) assert_array_equal(d1.label_counts[1], np.array([4])) assert_array_equal(d1.label_counts[2], np.array([3])) assert_array_equal(d1.label_counts[3], np.array([3])) assert_array_equal(d2.label_counts[1], np.array([0, 0])) assert_array_equal(d2.label_counts[2], np.array([0, 11])) assert_array_equal(d2.label_counts[3], np.array([0, 3])) d1 = Dataset(c1) d2 = Dataset(c2) d2.sync_label_counts(d1) np.testing.assert_array_equal(set(d1.label_counts.keys()), set(d2.label_counts.keys())) assert_array_equal(d1.label_counts[1], np.array([4])) assert_array_equal(d1.label_counts[2], np.array([3])) assert_array_equal(d1.label_counts[3], np.array([3])) assert_array_equal(d2.label_counts[1], np.array([0, 0])) assert_array_equal(d2.label_counts[2], np.array([0, 11])) assert_array_equal(d2.label_counts[3], np.array([0, 3]))
def test_check_label_matrix_dimensions(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join( base_path, '../test_data/tiffconnector_1/labels_multichannel/*.tif') c = TiffConnector(img_path, label_path) c.check_label_matrix_dimensions()
def test_map_label_values_3(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) original_labels = c.original_label_values_for_all_images() res = c.calc_label_values_mapping(original_labels) self.assertEqual(res, [{91: 1, 109: 2, 150: 3}])
def test_put_tile_multichannel(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/*.tif') # savepath = os.path.join( # base_path, '../test_data') path = os.path.join(self.tmpdir, '6width4height3slices.tif') c = TiffConnector(img_path, label_path, savepath=self.tmpdir) pixels = np.array([[[.1, .2, .3], [.4, .5, .6]]], dtype=np.float32) label_value = 3 c.put_tile(pixels, pos_zxy=(0, 1, 1), image_nr=2, label_value=label_value, multichannel=3) slices = c._open_probability_map_file(2, 3, multichannel=3) print(slices.shape) probim = np.moveaxis(slices, (0, 1, 2, 3), (1, 3, 2, 0)) probim = probim[2:3, :, :, :] val = \ np.array([[[[0., 0., 0., 0.], [0., 0.1, 0.2, 0.3], [0., 0.4, 0.5, 0.6], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]]]], dtype=np.float32) np.testing.assert_array_equal(val, probim) try: os.remove(path) except FileNotFoundError: pass
def test_original_label_values(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join( base_path, '../test_data/tiffconnector_1/labels_multichannel/*.tif') c = TiffConnector(img_path, label_path) res = c.original_label_values_for_all_images() self.assertEqual(res, [{91, 109, 150}, {91, 109, 150}])
def test_label_count_for_image(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/*.tif') c = TiffConnector(img_path, label_path) count = c.label_count_for_image(2) print(c.labelvalue_mapping) self.assertEqual(count, {2: 11, 3: 3})
def test_map_label_values_4(self): img_path = os.path.join( base_path, '../test_data/tiffconnector_1/im/6width4height3slices_rgb.tif') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/*.tif') c = TiffConnector(img_path, label_path) original_labels = c.original_label_values_for_all_images() c.calc_label_values_mapping(original_labels) self.assertEqual(c.labelvalue_mapping, [{109: 1, 150: 2}])
def test_put_tile_1(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/*.tif') savepath = tempfile.TemporaryDirectory() c = TiffConnector(img_path, label_path, savepath=savepath.name) pixels = np.array([[[.1, .2, .3], [.4, .5, .6]]], dtype=np.float32) path = os.path.join(savepath.name, '6width4height3slices_rgb_class_3.tif') try: os.remove(path) except FileNotFoundError: pass c.put_tile(pixels, pos_zxy=(0, 1, 1), image_nr=2, label_value=3) slices = c._open_probability_map_file(2, 3) probim = np.array([[slices[0, 0, z].T for z in range(3)]]) val = \ np.array([[[[0., 0., 0., 0.], [0., 0.1, 0.2, 0.3], [0., 0.4, 0.5, 0.6], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]]]], dtype=np.float32) np.testing.assert_array_equal(val, probim) try: os.remove(path) except FileNotFoundError: pass
def test_get_weight_tile_for_label_2(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) d.label_weights[3] = 1.2 img_nr = 2 pos_czxy = np.array((0, 0, 0)) size_czxy = np.array((1, 6, 4)) label_value = 3 mat = d._get_weights_tile(img_nr, pos_czxy, size_czxy, label_value) val = np.zeros(size_czxy) print('valshae') print(val.shape) val[0, 0, 1] = 1.2 val[0, 4, 1] = 1.2 val[0, 5, 1] = 1.2 pprint('mat') pprint(mat) pprint(val) self.assertTrue((val == mat).all())
def io_connector(image_path, label_path, *args, **kwds): ''' Returns either a TiffConnector or an NapariConnector, depending on input files. Parameters ---------- image_path : str or list of str Either wildcard or list of paths to pixel data in tiff format. label_path : str of list of str Either wildcard or list of paths to pixel data in tiff format returns a TiffConnector. If a path to a single Ilastik ilp file is given, an NapariConnector is returned. Returns ------- Connector Either NapariConnector or TiffConnector See Also -------- yapic_io.tiff_connector.TiffConnector yapic_io.napari_connector.NapariConnector ''' from yapic_io.tiff_connector import TiffConnector from yapic_io.napari_connector import NapariConnector if label_path.endswith('.h5'): logger.info('Napari project file detected') return NapariConnector(image_path, label_path) else: logger.info('Tiff files detected.') return TiffConnector(image_path, label_path, *args, **kwds)
def test_multichannel_pixel_tile_3(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) img = 2 pos_zxy = (0, 0, 0) size_zxy = (1, 4, 3) channels = [1] pd = (0, 2, 3) val = np.array([[[[73, 73, 43, 89, 89, 102, 81, 87, 78], [82, 68, 78, 37, 102, 102, 102, 37, 68], [161, 153, 78, 87, 81, 102, 89, 82, 78], [180, 180, 107, 87, 87, 37, 82, 87, 87], [143, 121, 121, 107, 78, 68, 78, 87, 87], [111, 111, 121, 180, 125, 73, 73, 78, 82], [121, 111, 143, 161, 106, 82, 73, 68, 43], [121, 180, 147, 123, 106, 106, 125, 68, 78]]]]) tile = d.multichannel_pixel_tile(img, pos_zxy, size_zxy, channels, pixel_padding=pd, augment_params={'rotation_angle': 45}) self.assertTrue((tile == val).all())
def test_n_images(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') c = TiffConnector(img_path, 'path/to/nowhere/') d = Dataset(c) np.testing.assert_array_equal(d.n_images, 3)
def test_random_pos_izxy(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join( base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) np.random.seed(42) img_nr, pos_zxy = d._random_pos_izxy(label_value=1, tile_size_zxy=(1, 2, 2)) assert_array_equal(img_nr, 0) assert_array_equal(pos_zxy, [2, 7, 20]) # this tile has same size as image, pos must always be 0 img_nr, pos_zxy = d._random_pos_izxy(label_value=1, tile_size_zxy=(3, 40, 26)) assert_array_equal(img_nr, 0) assert_array_equal(pos_zxy, [0, 0, 0]) # this tile is larger than the image with self.assertRaises(AssertionError): d._random_pos_izxy(label_value=1, tile_size_zxy=(3, 40, 27)) np.random.seed(None)
def test_random_training_tile(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 3, 4) pad = (1, 2, 2) channels = [0, 1, 2] pixel_shape_val = (3, 3, 7, 8) weight_shape_val = (3, 1, 3, 4) # mapping [{91: 1, 109: 2, 150: 3}] labels_val = [1, 2, 3] tile = d.random_training_tile(size, channels, pixel_padding=pad, augment_params={'rotation_angle': 45}) np.testing.assert_array_equal(tile.pixels.shape, pixel_shape_val) np.testing.assert_array_equal(tile.channels, channels) np.testing.assert_array_equal(tile.weights.shape, weight_shape_val) np.testing.assert_array_equal(tile.labels, labels_val)
def test_getitem_multichannel_labels(self): # define data loacations pixel_image_dir = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_image_dir = os.path.join( base_path, '../test_data/tiffconnector_1/labels_multichannel/') savepath = tempfile.TemporaryDirectory() tile_size = (1, 5, 4) # size of network output layer in zxy # padding of network input layer in zxy, in respect to output layer padding = (0, 2, 2) # make training_batch mb and prediction interface p with # TiffConnector binding c = TiffConnector(pixel_image_dir, label_image_dir, savepath=savepath.name) m = TrainingBatch(Dataset(c), tile_size, padding_zxy=padding) for counter, mini in enumerate(m): # shape is (6, 6, 1, 5, 4): # batchsize 6 , 6 label-classes, 1 z, 5 x, 4 y weights = mini.weights() # shape is (6, 3, 1, 9, 8): # batchsize 6, 6 channels, 1 z, 9 x, 4 y (more xy due to padding) pixels = mini.pixels() self.assertEqual(weights.shape, (6, 6, 1, 5, 4)) self.assertEqual(pixels.shape, (6, 3, 1, 9, 8)) # apply training on mini.pixels and mini.weights goes here if counter > 10: # m is infinite break
def test_set_augmentation(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 3, 4) pad = (1, 2, 2) m = TrainingBatch(d, size, padding_zxy=pad) self.assertEqual(m.augmentation, {'flip'}) m.augment_by_rotation(True) self.assertEqual(m.augmentation, {'flip', 'rotate'}) self.assertEqual(m.rotation_range, (-45, 45)) m.augment_by_shear(True) self.assertEqual(m.augmentation, {'flip', 'rotate', 'shear'}) self.assertEqual(m.shear_range, (-5, 5)) m.augment_by_flipping(False) self.assertEqual(m.augmentation, {'rotate', 'shear'})
def test_get_pixels_dimension_order(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (2, 5, 4) pad = (0, 0, 0) m = TrainingBatch(d, size, padding_zxy=pad) b = next(m) p = m.pixels() w = m.weights() self.assertEqual(p.shape, (3, 3, 2, 5, 4)) self.assertEqual(w.shape, (3, 3, 2, 5, 4)) m.set_pixel_dimension_order('bczxy') p = m.pixels() w = m.weights() self.assertEqual(p.shape, (3, 3, 2, 5, 4)) self.assertEqual(w.shape, (3, 3, 2, 5, 4)) m.set_pixel_dimension_order('bzxyc') p = m.pixels() w = m.weights() self.assertEqual(p.shape, (3, 2, 5, 4, 3)) self.assertEqual(w.shape, (3, 2, 5, 4, 3))
def test_get_tile(self): img_path = os.path.join( base_path, '../test_data/tiffconnector_1/together/img*.tif') c = TiffConnector(img_path, 'path/to/nowhere/') image_nr = 0 pos = (0, 0, 0, 0) size = (1, 1, 1, 2) tile = c.get_tile(image_nr=image_nr, pos=pos, size=size) val = np.empty(shape=size) val[0, 0, 0, 0] = 151 val[0, 0, 0, 1] = 151 val = val.astype(int) print(val) print(tile) np.testing.assert_array_equal(tile, val)
def test_check_label_matrix_dimensions_2(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join( base_path, '../test_data/tiffconnector_1/labels_multichannel_not_valid/') self.assertRaises(AssertionError, lambda: TiffConnector(img_path, label_path))
def test_random_training_tile_by_polling(self): img_path = os.path.join( base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join( base_path, '../test_data/tiffconnector_1/labels/') size = (1, 4, 3) channels = [0, 1, 2] labels = set([1, 2, 3]) ensure_labelvalue = 2 c = TiffConnector(img_path, label_path) d = Dataset(c) # weights_val = np.array( weights_val = np.array([[[[0., 0., 0.], [0., 0., 0.], [0., 0., 0.], [0., 0., 0.]]], [[[0., 0., 0.], [0., 0., 0.], [1., 1., 1.], [1., 1., 1.]]], [[[1., 0., 0.], [0., 0., 0.], [0., 0., 0.], [0., 0., 0.]]]]) np.random.seed(43) training_tile = d._random_training_tile_by_polling( size, channels, labels, ensure_labelvalue=ensure_labelvalue) assert_array_equal(training_tile.weights, weights_val) weights_val = np.array([[[[0., 0., 0.], [0., 0., 0.], [0., 0., 0.], [0., 0., 0.]]], [[[0., 0., 0.], [0., 1., 1.], [0., 1., 1.], [0., 0., 0.]]], [[[0., 0., 0.], [0., 0., 0.], [0., 0., 0.], [0., 1., 0.]]]]) training_tile = d._random_training_tile_by_polling( size, channels, labels, ensure_labelvalue=None) assert_array_equal(training_tile.weights, weights_val) np.random.seed(None)
def test_split(self): img_path = os.path.join( base_path, '../test_data/tiffconnector_1/together/img*.tif') lbl_path = os.path.join( base_path, '../test_data/tiffconnector_1/together/lbl*.tif') c = TiffConnector(img_path, lbl_path) c1, c2 = c.split(0.5) expected_names1 = [(Path('img_40width26height3slices_rgb.tif'), Path('lbl_40width26height3slices_rgb.tif'))] expected_names2 = [(Path('img_40width26height6slices_rgb.tif'), None), (Path('img_6width4height3slices_rgb.tif'), Path('lbl_6width4height3slices_rgb.tif'))] self.assertEqual(set(c1.filenames), set(expected_names1)) self.assertEqual(set(c2.filenames), set(expected_names2)) # test for issue #1 self.assertEqual(c1.labelvalue_mapping, c.labelvalue_mapping) self.assertEqual(c2.labelvalue_mapping, c.labelvalue_mapping)
def test_get_label_probs(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) res = d._get_label_probs(label_value=None) assert_array_almost_equal(res, [0.416667, 0., 0.583333]) res = d._get_label_probs(label_value=1) assert_array_almost_equal(res, [1., 0., 0.])
def test_get_tile2(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/c2z2y2x2.tif') conn = TiffConnector(img_path, 'path/to/nowhere/') image_nr = 0 pos = (0, 0, 0, 0) size = (2, 2, 2, 2) tile = conn.get_tile(image_nr=image_nr, pos=pos, size=size) expected = [[[[ c * 2**3 + z * 2**2 + y * 2**1 + x * 2**0 for y in range(2) ] for x in range(2)] for z in range(2)] for c in range(2)] np.testing.assert_array_equal(tile, expected) size = (1, 1, 1, 1) for c, z, x, y in itertools.product(range(2), repeat=4): pos = (c, z, x, y) tile = conn.get_tile(image_nr=image_nr, pos=pos, size=size) v = c * 2**3 + z * 2**2 + y * 2**1 + x * 2**0 np.testing.assert_array_equal(tile, [[[[v]]]])
def test_put_tile_1(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/*.tif') c = TiffConnector(img_path, label_path, savepath=self.tmpdir) pixels = np.array([[[.1, .2, .3], [.4, .5, .6]]], dtype=np.float32) path = os.path.join(self.tmpdir, '6width4height3slices_rgb_class_3.tif') c.put_tile(pixels, pos_zxy=(0, 1, 1), image_nr=2, label_value=3) slices = c._open_probability_map_file(2, 3) probim = np.moveaxis(slices, (0, 1, 2, 3), (1, 3, 2, 0)) val = \ np.array([[[[0., 0., 0., 0.], [0., 0.1, 0.2, 0.3], [0., 0.4, 0.5, 0.6], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]], [[0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.], [0., 0., 0., 0.]]]], dtype=np.float32) np.testing.assert_array_equal(val, probim)
def test_normalize_zscore(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 2, 2) pad = (0, 0, 0) batchsize = 2 nr_channels = 3 nz = 1 nx = 4 ny = 5 m = TrainingBatch(d, size, padding_zxy=pad, batch_size=len(d.label_values())) m.set_normalize_mode('local_z_score') pixels = np.zeros((batchsize, nr_channels, nz, nx, ny)) pixels[:, 0, :, :, :] = 1 pixels[:, 1, :, :, :] = 2 pixels[:, 2, :, :, :] = 3 p_norm = m._normalize(pixels) self.assertTrue((p_norm == 0).all()) # add variation pixels[:, 0, 0, 0, 0] = 2 pixels[:, 0, 0, 0, 1] = 0 pixels[:, 1, 0, 0, 0] = 3 pixels[:, 1, 0, 0, 1] = 1 pixels[:, 2, 0, 0, 0] = 4 pixels[:, 2, 0, 0, 1] = 2 p_norm = m._normalize(pixels) assert_array_equal(p_norm[:, 0, :, :, :], p_norm[:, 1, :, :, :]) assert_array_equal(p_norm[:, 0, :, :, :], p_norm[:, 2, :, :, :]) val = np.array([[[3.16227766, -3.16227766, 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]]]) assert_array_almost_equal(val, p_norm[0, 0, :, :, :])
def test_load_label_filenames(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/*.tif') c = TiffConnector(img_path, label_path) self.assertEqual(c.filenames[0][1], Path('40width26height3slices_rgb.tif')) self.assertIsNone(c.filenames[1][1]) self.assertEqual(c.filenames[2][1], Path('6width4height3slices_rgb.tif'))
def test_load_filenames(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/*.tif') c = TiffConnector(img_path, 'path/to/nowhere/') img_filenames = [ Path('6width4height3slices_rgb.tif'), Path('40width26height3slices_rgb.tif'), Path('40width26height6slices_rgb.tif') ] fnames = [e[0] for e in c.filenames] self.assertEqual(set(img_filenames), set(fnames))
def test_set_label_weight(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) d.label_weights[3] = 0.7 self.assertTrue( (d.label_weights[3] == np.array([0.7, 0.7, 0.7, 0.7, 0.7, 0.7])).all()) self.assertTrue((d.label_weights[1] == np.array([1, 1, 1, 1])).all())
def test_random_tile(self): img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/') label_path = os.path.join(base_path, '../test_data/tiffconnector_1/labels/') c = TiffConnector(img_path, label_path) d = Dataset(c) size = (1, 3, 4) pad = (1, 2, 2) m = TrainingBatch(d, size, padding_zxy=pad) m._random_tile(for_label=1)
def test_filter_labeled(self): img_path = os.path.join( base_path, '../test_data/tiffconnector_1/together/img*.tif') lbl_path = os.path.join( base_path, '../test_data/tiffconnector_1/together/lbl*.tif') c = TiffConnector(img_path, lbl_path).filter_labeled() expected_names = \ [(Path('img_40width26height3slices_rgb.tif'), Path('lbl_40width26height3slices_rgb.tif')), (Path('img_6width4height3slices_rgb.tif'), Path('lbl_6width4height3slices_rgb.tif'))] self.assertEqual(set(c.filenames), set(expected_names))