def test_init_and_eq_and_ne():
t1 = Transform()
assert t1.rotation == Quat.identity()
assert t1.translation == Vec3.zero()
rot = Quat.from_rotation_on_axis(2, np.pi)
trans = Vec3(1, 2, 3)
t2 = Transform(rot, trans)
assert t2.rotation == rot
assert t2.translation == trans
t3 = Transform.identity()
assert t1 == t3
assert t1 != t2
assert t3 != t2
def create_patches(
raw_data,
patch_size,
n_patches_per_image,
patch_axes = None,
save_file = None,
transforms = None,
patch_filter = no_background_patches(),
normalization = norm_percentiles(),
shuffle = True,
verbose = True,
):
"""Create normalized training data to be used for neural network training.
Parameters
----------
raw_data : :class:`RawData`
Object that yields matching pairs of raw images.
patch_size : tuple
Shape of the patches to be extraced from raw images.
Must be compatible with the number of dimensions and axes of the raw images.
As a general rule, use a power of two along all XYZT axes, or at least divisible by 8.
n_patches_per_image : int
Number of patches to be sampled/extracted from each raw image pair (after transformations, see below).
patch_axes : str or None
Axes of the extracted patches. If ``None``, will assume to be equal to that of transformed raw data.
save_file : str or None
File name to save training data to disk in ``.npz`` format (see :func:`csbdeep.io.save_training_data`).
If ``None``, data will not be saved.
transforms : list or tuple, optional
List of :class:`Transform` objects that apply additional transformations to the raw images.
This can be used to augment the set of raw images (e.g., by including rotations).
Set to ``None`` to disable. Default: ``None``.
patch_filter : function, optional
Function to determine for each image pair which patches are eligible to be extracted
(default: :func:`no_background_patches`). Set to ``None`` to disable.
normalization : function, optional
Function that takes arguments `(patches_x, patches_y, x, y, mask, channel)`, whose purpose is to
normalize the patches (`patches_x`, `patches_y`) extracted from the associated raw images
(`x`, `y`, with `mask`; see :class:`RawData`). Default: :func:`norm_percentiles`.
shuffle : bool, optional
Randomly shuffle all extracted patches.
verbose : bool, optional
Display overview of images, transforms, etc.
Returns
-------
tuple(:class:`numpy.ndarray`, :class:`numpy.ndarray`, str)
Returns a tuple (`X`, `Y`, `axes`) with the normalized extracted patches from all (transformed) raw images
and their axes.
`X` is the array of patches extracted from source images with `Y` being the array of corresponding target patches.
The shape of `X` and `Y` is as follows: `(n_total_patches, n_channels, ...)`.
For single-channel images, `n_channels` will be 1.
Raises
------
ValueError
Various reasons.
Example
-------
>>> raw_data = RawData.from_folder(basepath='data', source_dirs=['source1','source2'], target_dir='GT', axes='ZYX')
>>> X, Y, XY_axes = create_patches(raw_data, patch_size=(32,128,128), n_patches_per_image=16)
Todo
----
- Save created patches directly to disk using :class:`numpy.memmap` or similar?
Would allow to work with large data that doesn't fit in memory.
"""
## images and transforms
if transforms is None:
transforms = []
transforms = list(transforms)
if patch_axes is not None:
transforms.append(permute_axes(patch_axes))
if len(transforms) == 0:
transforms.append(Transform.identity())
image_pairs, n_raw_images = raw_data.generator(), raw_data.size
tf = Transform(*zip(*transforms)) # convert list of Transforms into Transform of lists
image_pairs = compose(*tf.generator)(image_pairs) # combine all transformations with raw images as input
n_transforms = np.prod(tf.size)
n_images = n_raw_images * n_transforms
n_patches = n_images * n_patches_per_image
n_required_memory_bytes = 2 * n_patches*np.prod(patch_size) * 4
## memory check
_memory_check(n_required_memory_bytes)
## summary
if verbose:
print('='*66)
print('%5d raw images x %4d transformations = %5d images' % (n_raw_images,n_transforms,n_images))
print('%5d images x %4d patches per image = %5d patches in total' % (n_images,n_patches_per_image,n_patches))
print('='*66)
print('Input data:')
print(raw_data.description)
print('='*66)
print('Transformations:')
for t in transforms:
print('{t.size} x {t.name}'.format(t=t))
print('='*66)
print('Patch size:')
print(" x ".join(str(p) for p in patch_size))
print('=' * 66)
sys.stdout.flush()
## sample patches from each pair of transformed raw images
X = np.empty((n_patches,)+tuple(patch_size),dtype=np.float32)
Y = np.empty_like(X)
for i, (x,y,_axes,mask) in tqdm(enumerate(image_pairs),total=n_images,disable=(not verbose)):
if i >= n_images:
warnings.warn('more raw images (or transformations thereof) than expected, skipping excess images.')
break
if i==0:
axes = axes_check_and_normalize(_axes,len(patch_size))
channel = axes_dict(axes)['C']
# checks
# len(axes) >= x.ndim or _raise(ValueError())
axes == axes_check_and_normalize(_axes) or _raise(ValueError('not all images have the same axes.'))
x.shape == y.shape or _raise(ValueError())
mask is None or mask.shape == x.shape or _raise(ValueError())
(channel is None or (isinstance(channel,int) and 0<=channel<x.ndim)) or _raise(ValueError())
channel is None or patch_size[channel]==x.shape[channel] or _raise(ValueError('extracted patches must contain all channels.'))
_Y,_X = sample_patches_from_multiple_stacks((y,x), patch_size, n_patches_per_image, mask, patch_filter)
s = slice(i*n_patches_per_image,(i+1)*n_patches_per_image)
X[s], Y[s] = normalization(_X,_Y, x,y,mask,channel)
if shuffle:
shuffle_inplace(X,Y)
axes = 'SC'+axes.replace('C','')
if channel is None:
X = np.expand_dims(X,1)
Y = np.expand_dims(Y,1)
else:
X = np.moveaxis(X, 1+channel, 1)
Y = np.moveaxis(Y, 1+channel, 1)
if save_file is not None:
print('Saving data to %s.' % str(Path(save_file)))
save_training_data(save_file, X, Y, axes)
return X,Y,axes
