def test_get_pixels_dimension_order(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (2, 5, 4)
pad = (0, 0, 0)
m = TrainingBatch(d,
size,
padding_zxy=pad,
batch_size=len(d.label_values()))
next(m)
p = m.pixels()
w = m.weights()
self.assertEqual(p.shape, (3, 3, 2, 5, 4))
self.assertEqual(w.shape, (3, 3, 2, 5, 4))
m.set_pixel_dimension_order('bczxy')
p = m.pixels()
w = m.weights()
self.assertEqual(p.shape, (3, 3, 2, 5, 4))
self.assertEqual(w.shape, (3, 3, 2, 5, 4))
m.set_pixel_dimension_order('bzxyc')
p = m.pixels()
w = m.weights()
self.assertEqual(p.shape, (3, 2, 5, 4, 3))
self.assertEqual(w.shape, (3, 2, 5, 4, 3))
def test_random_training_tile(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (1, 3, 4)
pad = (1, 2, 2)
channels = [0, 1, 2]
pixel_shape_val = (3, 3, 7, 8)
weight_shape_val = (3, 1, 3, 4)
# mapping [{91: 1, 109: 2, 150: 3}]
labels_val = [1, 2, 3]
tile = d.random_training_tile(size,
channels,
pixel_padding=pad,
augment_params={'rotation_angle': 45})
np.testing.assert_array_equal(tile.pixels.shape, pixel_shape_val)
np.testing.assert_array_equal(tile.channels, channels)
np.testing.assert_array_equal(tile.weights.shape, weight_shape_val)
np.testing.assert_array_equal(tile.labels, labels_val)
def test_set_augmentation(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (1, 3, 4)
pad = (1, 2, 2)
m = TrainingBatch(d,
size,
padding_zxy=pad,
batch_size=len(d.label_values()))
self.assertEqual(m.augmentation, {'flip'})
m.augment_by_rotation(True)
self.assertEqual(m.augmentation, {'flip', 'rotate'})
self.assertEqual(m.rotation_range, (-45, 45))
m.augment_by_shear(True)
self.assertEqual(m.augmentation, {'flip', 'rotate', 'shear'})
self.assertEqual(m.shear_range, (-5, 5))
m.augment_by_flipping(False)
self.assertEqual(m.augmentation, {'rotate', 'shear'})
def test_multichannel_pixel_tile_3(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
img = 2
pos_zxy = (0, 0, 0)
size_zxy = (1, 4, 3)
channels = [1]
pd = (0, 2, 3)
val = np.array([[[[73, 73, 43, 89, 89, 102, 81, 87, 78],
[82, 68, 78, 37, 102, 102, 102, 37, 68],
[161, 153, 78, 87, 81, 102, 89, 82, 78],
[180, 180, 107, 87, 87, 37, 82, 87, 87],
[143, 121, 121, 107, 78, 68, 78, 87, 87],
[111, 111, 121, 180, 125, 73, 73, 78, 82],
[121, 111, 143, 161, 106, 82, 73, 68, 43],
[121, 180, 147, 123, 106, 106, 125, 68, 78]]]])
tile = d.multichannel_pixel_tile(img,
pos_zxy,
size_zxy,
channels,
pixel_padding=pd,
augment_params={'rotation_angle': 45})
self.assertTrue((tile == val).all())
def test_get_weight_tile_for_label_2(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
d.label_weights[3] = 1.2
img_nr = 2
pos_czxy = np.array((0, 0, 0))
size_czxy = np.array((1, 6, 4))
label_value = 3
mat = d._get_weights_tile(img_nr, pos_czxy, size_czxy, label_value)
val = np.zeros(size_czxy)
print('valshae')
print(val.shape)
val[0, 0, 1] = 1.2
val[0, 4, 1] = 1.2
val[0, 5, 1] = 1.2
pprint('mat')
pprint(mat)
pprint(val)
self.assertTrue((val == mat).all())
def load_training_data(self, image_path, label_path, batch_size=None):
'''
Connect to a training dataset.
Parameters
----------
image_path : string
Path to folder with tiff images.
label_path : string
Path to folder with label tiff images or path to ilastik project
file (.ilp file).
'''
_connector = io_connector(image_path, label_path)
self.lbl_map = _connector.labelvalue_mapping[0]
self.dataset = Dataset(_connector)
if batch_size is None:
self.batch_size = len(self.dataset.label_values())
else:
self.batch_size = batch_size
msg = '\n\nImport taining dataset:\n{}\n'.format(
self.dataset.pixel_connector.__repr__())
sys.stdout.write(msg)
def test_channels_are_consistent(self):
data_dir = os.path.join(base_path, '../test_data/napari')
c = NapariConnector(data_dir, os.path.join(data_dir, 'connector_test.h5'))
d = Dataset(c)
is_consistent, channel_cnt = d.channels_are_consistent()
assert is_consistent
assert channel_cnt == [3]
def test_random_training_tile_by_polling(self):
img_path = os.path.join(
base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(
base_path, '../test_data/tiffconnector_1/labels/')
size = (1, 4, 3)
channels = [0, 1, 2]
labels = set([1, 2, 3])
ensure_labelvalue = 2
c = TiffConnector(img_path, label_path)
d = Dataset(c)
# weights_val = np.array(
weights_val = np.array([[[[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.]]],
[[[0., 0., 0.],
[0., 0., 0.],
[1., 1., 1.],
[1., 1., 1.]]],
[[[1., 0., 0.],
[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.]]]])
np.random.seed(43)
training_tile = d._random_training_tile_by_polling(
size, channels, labels,
ensure_labelvalue=ensure_labelvalue)
assert_array_equal(training_tile.weights, weights_val)
weights_val = np.array([[[[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.]]],
[[[0., 0., 0.],
[0., 1., 1.],
[0., 1., 1.],
[0., 0., 0.]]],
[[[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.],
[0., 1., 0.]]]])
training_tile = d._random_training_tile_by_polling(
size, channels, labels,
ensure_labelvalue=None)
assert_array_equal(training_tile.weights, weights_val)
np.random.seed(None)
def test_get_label_probs(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
res = d._get_label_probs(label_value=None)
assert_array_almost_equal(res, [0.416667, 0., 0.583333])
res = d._get_label_probs(label_value=1)
assert_array_almost_equal(res, [1., 0., 0.])
def test_normalize_zscore(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (1, 2, 2)
pad = (0, 0, 0)
batchsize = 2
nr_channels = 3
nz = 1
nx = 4
ny = 5
m = TrainingBatch(d,
size,
padding_zxy=pad,
batch_size=len(d.label_values()))
m.set_normalize_mode('local_z_score')
pixels = np.zeros((batchsize, nr_channels, nz, nx, ny))
pixels[:, 0, :, :, :] = 1
pixels[:, 1, :, :, :] = 2
pixels[:, 2, :, :, :] = 3
p_norm = m._normalize(pixels)
self.assertTrue((p_norm == 0).all())
# add variation
pixels[:, 0, 0, 0, 0] = 2
pixels[:, 0, 0, 0, 1] = 0
pixels[:, 1, 0, 0, 0] = 3
pixels[:, 1, 0, 0, 1] = 1
pixels[:, 2, 0, 0, 0] = 4
pixels[:, 2, 0, 0, 1] = 2
p_norm = m._normalize(pixels)
assert_array_equal(p_norm[:, 0, :, :, :], p_norm[:, 1, :, :, :])
assert_array_equal(p_norm[:, 0, :, :, :], p_norm[:, 2, :, :, :])
val = np.array([[[3.16227766, -3.16227766, 0., 0., 0.],
[0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.]]])
assert_array_almost_equal(val, p_norm[0, 0, :, :, :])
def test_pixel_statistics(self):
data_dir = os.path.join(base_path, '../test_data/napari')
c = NapariConnector(data_dir, os.path.join(data_dir, 'connector_test.h5'))
d = Dataset(c)
channels = [0, 1, 2]
stats = d.pixel_statistics([0, 1, 2], n_tiles=1000)
assert len(stats) == len(channels)
for stat in stats:
assert len(stat) == 2
assert stat[0] < stat[1]
def test_set_label_weight(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
d.label_weights[3] = 0.7
self.assertTrue(
(d.label_weights[3] == np.array([0.7, 0.7, 0.7, 0.7, 0.7,
0.7])).all())
self.assertTrue((d.label_weights[1] == np.array([1, 1, 1, 1])).all())
def test_n_images(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
c = TiffConnector(img_path, 'path/to/nowhere/')
d = Dataset(c)
np.testing.assert_array_equal(d.n_images, 3)
def test_equalize_label_weights(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(
base_path, '../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
d.equalize_label_weights()
print(d.label_weights)
val = {1: 0.5121951219512196,
2: 0.14634146341463417,
3: 0.34146341463414637}
np.testing.assert_array_equal(d.label_weights, val)
def load_prediction_data(self, image_path, save_path):
'''
Connect to a prediction dataset.
Parameters
----------
image_path : string
Path to folder with tiff images to predict.
save_path : string
Path to folder for saving prediction images.
'''
self.dataset = Dataset(
io_connector(image_path,
'/tmp/this_should_not_exist',
savepath=save_path))
def test_split(self):
img_path = os.path.join(
base_path, '../test_data/ilastik/pixels_ilastik-multiim-1.2')
label_path = os.path.join(
base_path, '../test_data/ilastik/ilastik-multiim-1.2.ilp')
c = IlastikConnector(img_path, label_path)
d = Dataset(c)
size = (1, 1, 1)
pad = (0, 0, 0)
m = TrainingBatch(d, size, padding_zxy=pad)
m2 = m.split(0.3)
self.assertTrue(
len(m.tile_pos_for_label[1]) > len(m2.tile_pos_for_label[1]))
self.assertTrue(
len(m.tile_pos_for_label[2]) > len(m2.tile_pos_for_label[2]))
# check for no overlap
self.assertEqual(
0,
len(set(m.tile_pos_for_label[1]) & set(m2.tile_pos_for_label[1])))
self.assertEqual(
0,
len(set(m2.tile_pos_for_label[1]) & set(m.tile_pos_for_label[1])))
def test_load_label_counts_from_ilastik(self):
img_path = os.path.join(base_path, '../test_data/ilastik')
lbl_path = os.path.join(base_path,
'../test_data/ilastik/ilastik-1.2.ilp')
c = io_connector(img_path, lbl_path)
d = Dataset(c)
actual_counts = c.label_count_for_image(0)
print(actual_counts)
label_counts = d.load_label_counts()
print(label_counts)
assert_array_equal(label_counts[1], np.array([1]))
assert_array_equal(label_counts[2], np.array([1]))
assert_array_equal(label_counts[3], np.array([1]))
def test_get_ilastik_weights2(self):
pth = os.path.join(base_path, '../test_data/ilastik/dimensionstest')
img_path = os.path.join(pth, 'images')
lbl_path = os.path.join(pth, 'x15_y10_z2_c4_classes2.ilp')
c = IlastikConnector(img_path, lbl_path)
d = Dataset(c)
size = (2, 15, 10) # this is the size of the whole image
pad = (0, 0, 0)
m = TrainingBatch(d, size, padding_zxy=pad)
m.augment_by_flipping(False)
mini = next(m)
weights = mini.weights()
print(weights[0, :, :, :, :])
# all samples of the batch should be identical,
# because tilesize=imsize
assert_array_equal(weights[0, :, :, :, :], weights[1, :, :, :, :])
# weight positions from labelvalue 1 and 2
pos = [[0, 0, 2, 1], [0, 0, 2, 1], [0, 0, 8, 6], [0, 0, 9, 6],
[0, 1, 4, 3], [1, 0, 2, 2], [1, 0, 3, 2], [1, 0, 8, 7],
[1, 0, 9, 7]]
for p in pos:
self.assertEqual(weights[0, p[0], p[1], p[2], p[3]], 1)
self.assertEqual(weights[1, p[0], p[1], p[2], p[3]], 1)
def test_get_ilastik_weights(self):
img_path = os.path.join(base_path, '../test_data/ilastik')
lbl_path = os.path.join(base_path,
'../test_data/ilastik/ilastik-1.2.ilp')
c = IlastikConnector(img_path, lbl_path)
d = Dataset(c)
size = (8, 2, 4)
pad = (0, 0, 0)
m = TrainingBatch(d, size, padding_zxy=pad)
m.augment_by_flipping(False)
mini = next(m)
weights = mini.weights()
# label 1 at position (4,1,3)
self.assertTrue(weights[0, 0, 4, 1, 3] == 1)
# label 2 at position (1,1,1)
self.assertTrue(weights[0, 1, 1, 1, 1] == 1)
# label 3 at position (1,1,1)
self.assertTrue(weights[0, 2, 7, 1, 1] == 1)
def test_tile_positions_decay(self):
img_path = os.path.join(
base_path, '../test_data/ilastik/pixels_ilastik-multiim-1.2')
label_path = os.path.join(
base_path, '../test_data/ilastik/ilastik-multiim-1.2.ilp')
c = IlastikConnector(img_path, label_path)
d = Dataset(c)
size = (2, 6, 4)
pad = (0, 0, 0)
m = TrainingBatch(d, size, padding_zxy=pad)
n_pos_lbl_1 = len(m.tile_pos_for_label[1])
n_pos_lbl_2 = len(m.tile_pos_for_label[2])
assert not m.tile_pos_for_label[1] is m.tile_pos_for_label[2]
for _ in range(800):
next(m)
n_pos_lbl_1_after = len(m.tile_pos_for_label[1])
n_pos_lbl_2_after = len(m.tile_pos_for_label[2])
self.assertTrue(n_pos_lbl_1 > n_pos_lbl_1_after)
self.assertTrue(n_pos_lbl_2 > n_pos_lbl_2_after)
def test_getitem_multichannel_labels(self):
# define data loacations
pixel_image_dir = os.path.join(base_path,
'../test_data/tiffconnector_1/im/')
label_image_dir = os.path.join(
base_path, '../test_data/tiffconnector_1/labels_multichannel/')
savepath = tempfile.TemporaryDirectory()
tile_size = (1, 5, 4) # size of network output layer in zxy
# padding of network input layer in zxy, in respect to output layer
padding = (0, 2, 2)
# make training_batch mb and prediction interface p with
# TiffConnector binding
c = TiffConnector(pixel_image_dir,
label_image_dir,
savepath=savepath.name)
m = TrainingBatch(Dataset(c), tile_size, padding_zxy=padding)
for counter, mini in enumerate(m):
# shape is (6, 6, 1, 5, 4):
# batchsize 6 , 6 label-classes, 1 z, 5 x, 4 y
weights = mini.weights()
# shape is (6, 3, 1, 9, 8):
# batchsize 6, 6 channels, 1 z, 9 x, 4 y (more xy due to padding)
pixels = mini.pixels()
self.assertEqual(weights.shape, (6, 6, 1, 5, 4))
self.assertEqual(pixels.shape, (6, 3, 1, 9, 8))
# apply training on mini.pixels and mini.weights goes here
if counter > 10: # m is infinite
break
def test_normalize_global(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (1, 2, 2)
pad = (0, 0, 0)
batchsize = 2
nr_channels = 3
nz = 1
nx = 4
ny = 5
val = np.array(
[[[0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333],
[0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333],
[0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333],
[0.33333333, 0.33333333, 0.33333333, 0.33333333, 0.33333333]],
[[0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667],
[0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667],
[0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667],
[0.66666667, 0.66666667, 0.66666667, 0.66666667, 0.66666667]],
[[1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.],
[1., 1., 1., 1., 1.]]])
m = TrainingBatch(d,
size,
padding_zxy=pad,
batch_size=len(d.label_values()))
m.set_normalize_mode('global', minmax=[0, 3])
pixels = np.zeros((batchsize, nr_channels, nz, nx, ny))
pixels[:, 0, :, :, :] = 1
pixels[:, 1, :, :, :] = 2
pixels[:, 2, :, :, :] = 3
p_norm = m._normalize(pixels)
pprint(p_norm)
print(pixels.shape)
print(p_norm.shape)
assert_array_almost_equal(val, p_norm[0, :, 0, :, :])
def load_training_data(self, image_path, label_path):
'''
Connect to a training dataset.
Parameters
----------
image_path : string
Path to folder with tiff images.
label_path : string
Path to folder with label tiff images or path to ilastik project
file (.ilp file).
'''
print(image_path)
print(label_path)
self.dataset = Dataset(io_connector(image_path, label_path))
def test_random_tile(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (1, 3, 4)
pad = (1, 2, 2)
m = TrainingBatch(d,
size,
padding_zxy=pad,
batch_size=len(d.label_values()))
m._random_tile(for_label=1)
def test_training_tile_1(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(
base_path, '../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
img = 2
pos_zxy = (0, 0, 0)
size_zxy = (1, 4, 3)
channels = [1]
labels = [2, 3]
tr = d.training_tile(img, pos_zxy, size_zxy, channels, labels,
pixel_padding=(0, 1, 2))
np.testing.assert_array_equal(tr.pixels.shape, (1, 1, 6, 7))
np.testing.assert_array_equal(tr.weights.shape, (2, 1, 4, 3))
def test_multichannel_pixel_tile_1(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
img = 2
pos_zxy = (0, 0, 0)
size_zxy = (1, 4, 3)
channels = [1]
val = np.array([[[[102, 89, 82], [81, 37, 43], [87, 78, 68],
[107, 153, 125]]]])
tile = d.multichannel_pixel_tile(img, pos_zxy, size_zxy, channels)
self.assertTrue((tile == val).all())
def test_normalize_global_auto(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
size = (1, 5, 4)
pad = (0, 0, 0)
m = TrainingBatch(d,
size,
padding_zxy=pad,
batch_size=len(d.label_values()))
assert m.global_norm_minmax is None
m.set_normalize_mode('global')
assert len(m.global_norm_minmax) == 3
def test_get_weight_tile_for_label(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
img_nr = 2
pos_czxy = (0, 0, 0)
size_czxy = (1, 6, 4)
label_value = 3
mat = d._get_weights_tile(img_nr, pos_czxy, size_czxy, label_value)
val = np.zeros(size_czxy)
val[0, 0, 1] = 1
val[0, 4, 1] = 1
val[0, 5, 1] = 1
np.testing.assert_array_equal(val, mat)
def test_load_label_counts(self):
img_path = os.path.join(base_path, '../test_data/tiffconnector_1/im/')
label_path = os.path.join(base_path,
'../test_data/tiffconnector_1/labels/')
c = TiffConnector(img_path, label_path)
d = Dataset(c)
t = d.load_label_counts()
# labelcounts for each image for labelvalue 1
expected_1 = np.array([4, 0, 0])
# labelcounts for each image for labelvalue 2
expected_2 = np.array([3, 0, 11])
# labelcounts for each image for labelvalue 3
expected_3 = np.array([3, 0, 3])
assert_array_equal(expected_1, t[1])
assert_array_equal(expected_2, t[2])
assert_array_equal(expected_3, t[3])
self.assertTrue(sorted(list(t.keys())), [1, 2, 3])
def test_random_training_tile_by_polling_ilastik(self):
p = os.path.join(base_path, '../test_data/ilastik/dimensionstest')
img_path = os.path.join(p, 'images')
label_path = os.path.join(p, 'x15_y10_z2_c4_classes2.ilp')
size = (1, 1, 1)
channels = [0, 1, 2, 3]
labels = set([1, 2])
ensure_labelvalue = 2
c = IlastikConnector(img_path, label_path)
d = Dataset(c)
np.random.seed(43)
training_tile = d._random_training_tile_by_polling(
size, channels, labels, ensure_labelvalue=ensure_labelvalue)
print(training_tile)
weights_val = np.array([[[[0.]]], [[[1.]]]])
assert_array_equal(training_tile.weights, weights_val)
