class DSBTestAugmentations(Dataset):
def __init__(self,
dataset,
scale=1.,
output_shape=None):
self.root_dataset = dataset
self.scale = scale
self.output_shape = output_shape
self._test_transforms = None
@property
def test_transforms(self):
if self._test_transforms is None:
self._test_transforms = Compose(QuantileNormalize(apply_to=[0]),
Scale(self.scale))
if self.output_shape is not None:
self._test_transforms.add(CenterCrop(self.output_shape))
return self._test_transforms
def __len__(self):
return len(self.root_dataset)
def __getitem__(self, index):
x, y = self.root_dataset[index]
y = y.astype(np.double)
x, y = self.test_transforms(x, y)
return x, y
def get_transforms(self):
transforms = Compose()
for transform in self.master_config:
transforms.add(transform)
return transforms
def get_transforms(self):
transforms = Compose()
if self.master_config.get('random_flip', False):
transforms.add(RandomFlip3D())
transforms.add(RandomRotate())
# Elastic transforms can be skipped by
# setting elastic_transform to false in the
# yaml config file.
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get(
'elastic_transform')
if elastic_transform_config.get('apply', False):
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
# random slide augmentation
if self.master_config.get('random_slides', False):
# TODO slide probability
ouput_shape = self.master_config.get('shape_after_slide', None)
max_misalign = self.master_config.get('max_misalign', None)
transforms.add(
RandomSlide(output_image_size=ouput_shape,
max_misalign=max_misalign))
# affinity transforms for affinity targets
# we apply the affinity target calculation only to the segmentation (1)
assert self.affinity_config is not None
# transforms.add(Segmentation2AffinitiesDynamicOffsets(apply_to=[1], **self.affinity_config))
transforms.add(
affinity_config_to_transform(apply_to=[1], **self.affinity_config))
# TODO: add clipping transformation
# crop invalid affinity labels and elastic augment reflection padding assymetrically
crop_config = self.master_config.get('crop_after_target', {})
if crop_config:
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(VolumeAsymmetricCrop(**crop_config))
# transforms.add(InvertTarget(self.affinity_config.get("retain_segmentation", False)))
# transforms.add(HackyHacky())
#transforms.add(ComputeVAETarget()), HackyHacky
return transforms
def get_transforms(self):
transforms = Compose()
if self.master_config.get('random_flip', False):
transforms.add(RandomFlip3D())
transforms.add(RandomRotate())
# # Elastic transforms can be skipped by
# # setting elastic_transform to false in the
# # yaml config file.
# if self.master_config.get('elastic_transform'):
# elastic_transform_config = self.master_config.get('elastic_transform')
# if elastic_transform_config.get('apply', False):
# transforms.add(ElasticTransform(alpha=elastic_transform_config.get('alpha', 2000.),
# sigma=elastic_transform_config.get('sigma', 50.),
# order=elastic_transform_config.get('order', 0)))
# random slide augmentation
if self.master_config.get('random_slides', False):
# TODO slide probability
ouput_shape = self.master_config.get('shape_after_slide', None)
max_misalign = self.master_config.get('max_misalign', None)
transforms.add(
RandomSlide(output_image_size=ouput_shape,
max_misalign=max_misalign))
# affinity transforms for affinity targets
# we apply the affinity target calculation only to the segmentation (1)
assert self.affinity_config is not None
transforms.add(
affinity_config_to_transform(apply_to=[1], **self.affinity_config))
# crop invalid affinity labels and elastic augment reflection padding assymetrically
crop_config = self.master_config.get('crop_after_target', {})
if crop_config:
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(VolumeAsymmetricCrop(**crop_config))
from vaeAffs.transforms import PassGTBoundaries_HackyHackyReloaded
transforms.add(PassGTBoundaries_HackyHackyReloaded())
return transforms
def get_transforms(self):
transforms = Compose(RandomFlip3D(), RandomRotate())
# Elastic transforms can be skipped by setting elastic_transform to false in the
# yaml config file.
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get(
'elastic_transform')
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
for_validation = self.master_config.get('for_validation', False)
# if we compute the affinities on the gpu, or use the feeder for validation only,
# we don't need to add the affinity transform here
if not for_validation:
assert self.affinity_config is not None
# we apply the affinity target calculation only to the segmentation (1)
transforms.add(
affinity_config_to_transform(apply_to=[1],
**self.affinity_config))
transforms.add(InvertAffinities(apply_to=[1]))
return transforms
def make_transforms(self):
transforms = Compose(PadTo(self.window_size), RandomFlip3D(), RandomRotate())
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get('elastic_transform')
transforms.add(ElasticTransform(alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
# affinity transforms for affinity targets
# we apply the affinity target calculation only to the segmentation (1)
affinity_config = self.master_config.get('affinity_config', None)
# Do we also train with semantic labels ?
train_semantic = self.master_config.get('train_semantic', False)
if affinity_config is None:
if train_semantic:
transforms.add(Semantics(apply_to=[1]))
self.label_transforms = None
else:
self.label_transforms = Cast('float32')
elif affinity_config == 'distances':
# TODO read the bandwidths from the config
self.label_transforms = Compose(Cast('int64'), ConnectedComponents3D())
from ..transforms.distance_transform import SignedDistanceTransform
transforms.add(SignedDistanceTransform(fg_bandwidth=8,
bg_bandwidth=32,
apply_to=[1]))
else:
if train_semantic:
# we can't apply connected components yet if we train semantics and affinities
self.label_transforms = Cast('int64')
transforms.add(SemanticsAndAffinities(affinity_config, apply_to=[1]))
else:
self.label_transforms = Compose(Cast('int64'), ConnectedComponents3D())
transforms.add(affinity_config_to_transform(apply_to=[1], **affinity_config))
self.transforms = transforms
sigma = 0.025
self.raw_transforms = Compose(Cast('float32'), Normalize(), AdditiveNoise(sigma=sigma))
def get_transforms(self, mean, std, sigma, p_augment_ws,
zero_mean_unit_variance):
transforms = Compose(Cast(self.dtype))
# add normalization (zero mean / unit variance)
if zero_mean_unit_variance:
transforms.add(Normalize(mean=mean, std=std))
else:
transforms.add(Normalize01())
# add noise transform if specified
if sigma is not None:
transforms.add(AdditiveNoise(sigma=sigma))
# add watershed super-pixel augmentation is specified
if p_augment_ws > 0.:
assert WatershedAugmentation is not None
transforms.add(WatershedAugmentation(p_augment_ws, invert=True))
return transforms
def get_transforms(self):
transforms = Compose(RandomFlip3D(), RandomRotate())
# Elastic transforms can be skipped by setting elastic_transform to false in the
# yaml config file.
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get(
'elastic_transform')
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
# TODO doesn't look like we have misalignment, so should be fine
# if we do not use random slides
# random slide augmentation
if self.master_config.get('random_slides', False):
assert False, "No random slides for now"
ouput_shape = self.master_config.get('shape_after_slide', None)
max_misalign = self.master_config.get('max_misalign', None)
transforms.add(
RandomSlide(output_image_size=ouput_shape,
max_misalign=max_misalign))
# if we compute the affinities on the gpu, or use the feeder for validation only,
# we don't need to add the affinity transform here
if self.affinity_config is not None:
# we apply the affinity target calculation only to the segmentation (1)
transforms.add(
affinity_config_to_transform(apply_to=[1],
**self.affinity_config))
# Next: crop invalid affinity labels and elastic augment reflection padding assymetrically
crop_config = self.master_config.get('crop_after_target', {})
if crop_config:
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(VolumeAsymmetricCrop(**crop_config))
return transforms
def get_transforms(self):
# The Segmentation2Affinities adds a channel dimension. Now depending on how many
# orders were requested, we dispatch Segmentation2Affinities or
# Segmentation2MultiOrderAffinities.
transforms = Compose()
# Cast to the right dtype
transforms.add(Cast(self.dtype))
# Run connected components to shuffle the labels
transforms.add(ConnectedComponents3D(label_segmentation=True))
# Make affinity maps
transforms.add(
Segmentation2MultiOrderAffinities(
dim=self.affinity_dim,
orders=pyu.to_iterable(self.affinity_order),
add_singleton_channel_dimension=True,
retain_segmentation=self.retain_segmentation))
return transforms
def get_transforms(self):
transforms = Compose(RandomFlip3D(), RandomRotate())
if 'elastic_transform' in self.master_config:
# Elastic transforms can be skipped by setting elastic_transform to false in the
# yaml config file.
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get(
'elastic_transform')
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 1)))
else:
# Preserve legacy behaviour
transforms.add(ElasticTransform(alpha=2000., sigma=50.))
if self.master_config.get('crop_after_elastic_transform', False):
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(
CenterCrop(**self.master_config.get(
'crop_after_elastic_transform', {})))
return transforms
def get_transforms(self):
transforms = Compose()
if self.transform_config.get('random_flip', False):
transforms.add(RandomFlip3D())
transforms.add(RandomRotate())
# Elastic transforms can be skipped by
# setting elastic_transform to false in the
# yaml config file.
if self.transform_config.get('elastic_transform'):
elastic_transform_config = self.transform_config.get(
'elastic_transform')
if elastic_transform_config.get('apply', False):
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
# Replicate and downscale batch:
nb_inputs = 1
if self.transform_config.get("downscale_and_crop") is not None:
ds_config = self.transform_config.get("downscale_and_crop")
apply_to = [conf.pop('apply_to') for conf in ds_config]
nb_inputs = (np.array(apply_to) == 0).sum()
transforms.add(ReplicateTensorsInBatch(apply_to))
for indx, conf in enumerate(ds_config):
transforms.add(
DownSampleAndCropTensorsInBatch(apply_to=[indx],
order=None,
**conf))
# Check if to compute binary-affinity-targets from GT labels:
if self.transform_config.get("affinity_config") is not None:
affs_config = deepcopy(
self.transform_config.get("affinity_config"))
global_kwargs = affs_config.pop("global", {})
aff_transform = Segmentation2AffinitiesDynamicOffsets if affs_config.pop("use_dynamic_offsets", False) \
else affinity_config_to_transform
for input_index in affs_config:
affs_kwargs = deepcopy(global_kwargs)
affs_kwargs.update(affs_config[input_index])
transforms.add(
aff_transform(apply_to=[input_index + nb_inputs],
**affs_kwargs))
# crop invalid affinity labels and elastic augment reflection padding assymetrically
crop_config = self.transform_config.get('crop_after_target', {})
if crop_config:
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(VolumeAsymmetricCrop(**crop_config))
transforms.add(AsTorchBatch(3))
return transforms
def get_transforms(self):
transforms = Compose()
if self.master_config.get('random_flip', False):
transforms.add(RandomFlip3D())
transforms.add(RandomRotate())
# Elastic transforms can be skipped by
# setting elastic_transform to false in the
# yaml config file.
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get(
'elastic_transform')
if elastic_transform_config.get('apply', False):
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
# random slide augmentation
if self.master_config.get('random_slides', False):
# TODO slide probability
ouput_shape = self.master_config.get('shape_after_slide', None)
max_misalign = self.master_config.get('max_misalign', None)
transforms.add(
RandomSlide(output_image_size=ouput_shape,
max_misalign=max_misalign))
# affinity transforms for affinity targets
# we apply the affinity target calculation only to the segmentation (1)
#assert self.affinity_config is not None
#transforms.add(affinity_config_to_transform(apply_to=[1], **self.affinity_config))
# TODO: add transfrom for directional DT
if self.master_config.get('compute_directions', False):
direction_config = self.master_config.get('compute_directions')
transforms.add(
LabelToDirections(
n_directions=direction_config.get('n_directions'),
compute_z=direction_config.get('z_direction')))
# TODO: add clipping transformation
if self.master_config.get('clip', False):
clipping_config = self.master_config.get('clip')
transforms.add(Clip(**clipping_config))
if self.master_config.get('multiply', False):
mult_config = self.master_config.get('multiply')
transforms.add(Multiply(**mult_config))
# crop invalid affinity labels and elastic augment reflection padding assymetrically
crop_config = self.master_config.get('crop_after_target', False)
if crop_config:
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(VolumeAsymmetricCrop(**crop_config))
return transforms
def get_transforms(self):
transforms = Compose()
if self.master_config.get('random_flip', False):
transforms.add(
AdjustBatch(
defected_label=self.master_config.get('defects_label', 3)))
transforms.add(RandomFlip3D())
transforms.add(RandomRotate())
transforms.add(
DuplicateGtDefectedSlices(
defected_label=self.master_config.get('defects_label', 3),
ignore_label=self.master_config.get('ignore_label', 0)))
# Elastic transforms can be skipped by
# setting elastic_transform to false in the
# yaml config file.
if self.master_config.get('elastic_transform'):
elastic_transform_config = self.master_config.get(
'elastic_transform')
if elastic_transform_config.get('apply', False):
transforms.add(
ElasticTransform(
alpha=elastic_transform_config.get('alpha', 2000.),
sigma=elastic_transform_config.get('sigma', 50.),
order=elastic_transform_config.get('order', 0)))
# random slide augmentation
if self.master_config.get('random_slides') is not None:
# TODO slide probability
random_slides_config = deepcopy(
self.master_config.get('random_slides'))
ouput_shape = random_slides_config.pop('shape_after_slide', None)
max_misalign = random_slides_config.pop('max_misalign', None)
transforms.add(
RandomSlide(
output_image_size=ouput_shape,
max_misalign=max_misalign,
# defects_label=self.master_config.get('defects_label', 2),
**random_slides_config))
# Replicate and downscale batch:
nb_inputs = 1
if self.master_config.get("downscale_and_crop") is not None:
ds_config = self.master_config.get("downscale_and_crop")
apply_to = [conf.pop('apply_to') for conf in ds_config]
nb_inputs = (np.array(apply_to) == 0).sum()
transforms.add(ReplicateBatchGeneralized(apply_to))
for indx, conf in enumerate(ds_config):
transforms.add(
DownsampleAndCrop3D(apply_to=[indx], order=None, **conf))
# # affinity transforms for affinity targets
# # we apply the affinity target calculation only to the segmentation (1)
if self.master_config.get("affinity_config") is not None:
affs_config = deepcopy(self.master_config.get("affinity_config"))
global_kwargs = affs_config.pop("global", {})
use_dynamic_offsets = affs_config.pop("use_dynamic_offsets", False)
if use_dynamic_offsets:
raise ValueError(
"The class neurofire.transform.affinities.Segmentation2AffinitiesDynamicOffsets has been deprecated."
)
aff_transform = affinity_config_to_transform
for input_index in affs_config:
affs_kwargs = deepcopy(global_kwargs)
affs_kwargs.update(affs_config[input_index])
transforms.add(
aff_transform(apply_to=[input_index + nb_inputs],
**affs_kwargs))
# crop invalid affinity labels and elastic augment reflection padding assymetrically
crop_config = self.master_config.get('crop_after_target', {})
if crop_config:
# One might need to crop after elastic transform to avoid edge artefacts of affinity
# computation being warped into the FOV.
transforms.add(VolumeAsymmetricCrop(**crop_config))
return transforms