def process(function,
source,
sink=None,
axes=None,
size_max=None,
size_min=None,
overlap=None,
optimization=True,
optimization_fix='all',
neighbours=False,
function_type=None,
as_memory=False,
return_result=False,
return_blocks=False,
processes=None,
verbose=False,
**kwargs):
"""Create blocks and process a function on them in parallel.
Arguments
---------
function : function
The main data processing script.
source : str, Source, or list
The source or list of sources to apply a function to
sink : str, Source, list, or None
The sink or list of sinks to write the result to.
If None, return single array.
axes : int, list of ints, or None
Axes along which to split the source. If None, the
splitting is determined automaticlly from the order of the array.
size_max : int, list of ints or None
Maximal size of a block along the axes.
If None, :const:`default_size_max` is used.
size_min : int or list of ints
Minial size of a block along the axes.
If None, :const:`default_size_min` is used.
overlap : int, list of ints or None
Minimal overlap between blocks along the axes.
If None, :const:`default_overlap` is used.
optimization : bool or list of bools
If True, optimize block sizes to best fit number of processes.
optimization_fix : 'increase', 'decrease', 'all' or None or list
Increase, decrease or optimally change the block size when optimization
is active.
neighbours : bool
If True, also include information about the neighbourhood in the blocks.
function_type : 'array', 'source', 'block' or None
The function type passed. If None, 'array' is used.
* 'array'
Reading and writing the valid slices from the blocks is automatic
and the function gets passed numpy arrays.
* 'source'
Reading and writing the valid slices from the blocks is automatic
and the function gets passed Source classes as inputs.
* 'block'
The function is assumed to act on and update blocks itself.
as_memory : bool
If True, load full blocks into memory before applying the function.
Can be useful to reduce frequent reading and writing operations of memmaps.
return_result : bool
If True, return the results of the proceessing functions.
return_blocks : bool
If True, return the block information used to distribute the processing.
processes : int
The number of parallel processes, if 'serial', use serial processing.
verbose : bool
Print information on sub-stack generation.
Returns
-------
sink : str, Source, list or array
The results of the processing.
Note
----
This implementation only supports processing into sinks with the same shape as the source.
"""
#sources and sinks
if isinstance(source, list):
sources = source
else:
sources = [source]
sources = [io.as_source(s).as_virtual() for s in sources]
#if sink is None:
# sink = sma.Source(shape=sources[0].shape, dtype=sources[0].dtype, order=sources[0].order);
if isinstance(sink, list):
sinks = sink
elif sink is None:
sinks = []
else:
sinks = [sink]
sinks = [io.initialize(s, hint=sources[0]) for s in sinks]
sinks = [io.as_source(s).as_virtual() for s in sinks]
axes = block_axes(sources[0], axes=axes)
split = ft.partial(split_into_blocks,
processes=processes,
axes=axes,
size_max=size_max,
size_min=size_min,
overlap=overlap,
optimization=optimization,
optimization_fix=optimization_fix,
neighbours=neighbours,
verbose=False)
source_blocks = [split(s) for s in sources]
sink_blocks = [split(s) for s in sinks]
n_blocks = len(source_blocks[0])
source_blocks = [[blocks[i] for blocks in source_blocks]
for i in range(n_blocks)]
sink_blocks = [[blocks[i] for blocks in sink_blocks]
for i in range(n_blocks)]
if function_type is None:
function_type = 'array'
if function_type == 'block':
func = ft.partial(process_block_block,
function=function,
as_memory=as_memory,
return_result=return_result,
verbose=verbose,
**kwargs)
elif function_type == 'source':
func = ft.partial(process_block_source,
function=function,
as_memory=as_memory,
as_array=False,
verbose=verbose,
**kwargs)
elif function_type == 'array':
func = ft.partial(process_block_source,
function=function,
as_memory=as_memory,
as_array=True,
verbose=verbose,
**kwargs)
else:
raise ValueError(
"function type %r not 'array', 'source', 'block' or None!")
if not isinstance(processes, int) and processes != "serial":
processes = mp.cpu_count()
if verbose:
timer = tmr.Timer()
print("Processing %d blocks with function %r." %
(n_blocks, function.__name__))
if isinstance(processes, int):
#from bounded_pool_executor import BoundedProcessPoolExecutor
with cf.ProcessPoolExecutor(max_workers=processes) as executor:
#with BoundedProcessPoolExecutor(max_workers=processes) as executor:
futures = [
executor.submit(func, *args)
for args in zip(source_blocks, sink_blocks)
]
result = [f.result() for f in futures]
#executor.map(function, source_blocks, sink_blocks)
else:
result = [func(*args) for args in zip(source_blocks, sink_blocks)]
#analysis:ignore
if verbose:
timer.print_elapsed_time("Processed %d blocks with function %r" %
(n_blocks, function.__name__))
#gc.collect();
if return_result:
ret = result
else:
ret = sink
if return_blocks:
ret = (ret, [source_blocks, sink_blocks])
return ret
def resample_inverse(source, sink = None,
resample_source = None, resample_sink = None,
orientation = None,
source_shape = None, source_resolution = None,
sink_shape = None, sink_resolution = None,
axes_order = None, method = 'memmap',
interpolation = 'linear',
processes = None, verbose = True, **args):
"""Resample data inversely to :func:`resample` routine.
Arguments
---------
source : str, array
Source to be inversly resampled (e.g. sink in :func:`resample`).
sink : str or None
Sink to write the inversly resampled image to.
resample_source : str, array or None
Optional source in :func:`resample`.
resmaple_sink: str, array or None
Optional sink used in :func:`resample`.
orientation : tuple
Orientation as specified as in :func:`resample`.
source_shape : tuple or None
Optional value of source_shape as in :func:`resample`.
source_resolution : tuple or None
Optional value of source_resolution as in :func:`resample`.
sink_resolution : tuple or None
Optional value of sink_resolution as in :func:`resample`.
processing_directory : str or None
Optional directory in which to perform resmapling in parallel.
If None, a temporary directry will be created.
axis_order : list of tuples of int or None
The axes pairs along which to resample the data as in :func:`resample`.
method : 'shared' or 'memmap'
Method to handle intermediate resampling results. If 'shared' use shared
memory, otherwise use a memory map on disk.
interpolation : str
Method to use for interpolating to the resmapled image.
processes int or None
Number of processes to use for parallel resampling.
verbose : bool
If True, print progress information.
Returns
-------
resampled : array or str
Data or file name of inversly resampled image.
Notes
-----
* All arguments, except source and sink should be passed as :func:`resample`
to invert the resmapling.
"""
source = io.as_source(source);
ndim = source.ndim;
dtype = source.dtype;
#orientation
orientation = format_orientation(orientation);
orientation_inverse = inverse_orientation(orientation);
#original source info
if source_shape is None:
if source_resolution is None and resample_source is None:
raise ValueError('Either source_shape, source_resolution or resample_source must to be given!')
if resample_source is not None:
source_shape = io.shape(resample_source);
#original sink info
if sink_shape is None and sink_resolution is None:
if resample_sink is None:
sink_shape = io.shape(source);
else:
sink_shape = io.shape(resample_sink);
source_shape, sink_shape, source_resolution, sink_resolution = \
resample_shape(source_shape=source_shape, sink_shape=sink_shape,
source_resolution=source_resolution, sink_resolution=sink_resolution,
orientation=orientation);
sink_shape_in_source_orientation = orient_shape(sink_shape, orientation, inverse=True);
axes_order, shape_order = _axes_order(axes_order, source, source_shape, sink_shape_in_source_orientation);
interpolation = _interpolation_to_cv2(interpolation);
if processes is None or not processes == 'serial':
processes = io.mp.cpu_count();
#reversed orientation
if not orientation is None:
#reverse axes
slicing = [slice(None)] * ndim;
reslice = False;
for d,o in enumerate(orientation):
if o < 0:
slicing[d] = slice(None, None, -1);
reslice = True;
if reslice:
source = source[slicing];
#re-orient
per = orientation_to_permuation(orientation_inverse);
source = io.read(source);
source = source.transpose(per);
source = io.sma.as_shared(source);
#reverse resampling steps
axes_order = axes_order[::-1];
shape_order = shape_order[:-1];
shape_order = shape_order[::-1];
shape_order = shape_order + [source_shape]
#print(axes_order, shape_order)
#reverse resampling
n_steps = len(axes_order);
last_source = source;
delete_files = [];
#print(last_source)
for step, axes, shape in zip(range(n_steps), axes_order, shape_order):
if step == n_steps-1:
resampled = io.initialize(source=sink, shape=shape, dtype=dtype, memory='shared', as_source=True);
else:
if method == 'shared':
resampled = io.sma.create(shape, dtype=dtype, order='C', as_source=True);
else:
location = tempfile.mktemp() + '.npy';
resampled = io.mmp.create(location, shape=shape, dtype=dtype, order='C', as_source=True);
delete_files.append(location);
#indices for non-resampled axes
indices = tuple([range(s) for d,s in enumerate(shape) if d not in axes]);
indices = [i for i in itertools.product(*indices)];
n_indices = len(indices);
#resample step
last_source_virtual = last_source.as_virtual();
resampled_virtual = resampled.as_virtual();
_resample = ft.partial(_resample_2d, source=last_source_virtual, sink=resampled_virtual, axes=axes, shape=shape,
interpolation=interpolation, n_indices=n_indices, verbose=verbose)
if processes == 'serial':
for index in indices:
_resample(index=index);
else:
with concurrent.futures.ProcessPoolExecutor(processes) as executor:
executor.map(_resample, indices);
last_source = resampled;
for f in delete_files:
io.delete_file(f);
sink = resampled.as_real();
return sink;
def smooth_by_configuration(source, sink = None, iterations = 1,
processing_parameter = None,
processes = None, verbose = False):
"""Smooth a binary source using the local configuration around each pixel.
Arguments
---------
source : array or Source
The binary source to smooth.
sink : array, Source or None
The sink to write result of smoothing. If None, return array.
iterations : int
Number of smoothing iterations.
processing_parameter : None or dict
The parameter passed to
:func:`ClearMap.ParallelProcessing.BlockProcessing.process`.
processes : int or None
number of processes to use.
verbose : bool
If True, print progress information.
Returns
-------
smoothed : array or Source
Thre smoothed binary array.
Note
----
The algorithm is based on a topological smoothing operation defined by adding
or removing forground pixels based on the local topology of the binary array.
"""
if verbose:
print('Binary smoothing: initialized!');
timer = tmr.Timer();
#smoothing function
smooth = functools.partial(smooth_by_configuration_block, iterations=iterations, verbose=False);
smooth.__name__ = 'smooth_by_configuration'
#initialize sources and sinks
source = io.as_source(source);
sink = io.initialize(sink, shape=source.shape, dtype=bool, order=source.order);
#block processing parameter
block_processing_parameter = dict(axes = bp.block_axes(source),
as_memory=True,
overlap=None,
function_type='source',
processes=processes,
verbose=verbose);
if processing_parameter is not None:
block_processing_parameter.update(processing_parameter);
if not 'overlap' in block_processing_parameter or block_processing_parameter['overlap'] is None:
block_processing_parameter['overlap'] = 2 + 2 * iterations;
if not 'size_min' in block_processing_parameter or block_processing_parameter['size_min'] is None:
block_processing_parameter['size_min'] = 2 + 2 * iterations + 1;
if not 'axes' in block_processing_parameter or block_processing_parameter['axes'] is None:
block_processing_parameter['axes'] = bp.block_axes(source);
#print(block_processing_parameter)
#block process
bp.process(smooth, source, sink, **block_processing_parameter);
if verbose:
timer.print_elapsed_time('Binary smoothing: done');
return sink;
def resample(source, sink = None, orientation = None,
sink_shape = None, source_resolution = None, sink_resolution = None,
interpolation = 'linear', axes_order = None, method = 'shared',
processes = None, verbose = True):
"""Resample data of source in new shape/resolution and orientation.
Arguments
---------
source : str or array
The source to be resampled.
sink : str or None
The sink for the resampled image.
orientation : tuple or None:
The orientation specified by permuation and change in sign of (1,2,3).
sink_shape : tuple or None
The target shape of the resampled sink.
source_resolution : tuple or None
The resolution of the source (in length per pixel).
sink_resolution : tuple or None
The resolution of the resampled source (in length per pixel).
interpolation : str
The method to use for interpolating to the resmapled array.
axis_order : str, list of tuples of int or None
The axes pairs along which to resample the data at each step.
If None, this is detertmined automatically. For a FileList source,
setting the first tuple should point to axis not indicating files.
If 'size' the axis order is determined automatically to maximally reduce
the size of the array in each resmapling step.
If 'order' the axis order is chosed automatically to optimize io speed.
method : 'shared' or 'memmap'
Method to handle intermediate resampling results. If 'shared' use shared
memory, otherwise use a memory map on disk.
processes : int, None or 'serial'
Number of processes to use for parallel resampling, if None use maximal
processes avaialable, if 'serial' process in serial.
verbose : bool
If True, display progress information.
Returns
-------
sink : array or str
The data or filename of resampled sink.
Notes
-----
* Resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference (as when viewed by ImageJ).
* Orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes.
* Only a minimal set of information to determine the resampling parameter
has to be given, e.g. source_shape and sink_shape.
* The resampling is done by iterating two dimensional resampling steps.
"""
#TODO: write full nd resampling routine extending cv2 lib.
if verbose:
timer = tmr.Timer();
source = io.as_source(source);
source_shape = source.shape;
ndim = len(source_shape);
dtype = source.dtype;
order = source.order;
orientation = format_orientation(orientation);
source_shape, sink_shape, source_resolution, sink_resolution = \
resample_shape(source_shape=source_shape, sink_shape=sink_shape,
source_resolution=source_resolution, sink_resolution=sink_resolution,
orientation=orientation);
sink_shape_in_source_orientation = orient_shape(sink_shape, orientation, inverse=True);
interpolation = _interpolation_to_cv2(interpolation);
if not isinstance(processes, int) and processes != 'serial':
processes = io.mp.cpu_count();
#detemine order of resampling
axes_order, shape_order = _axes_order(axes_order, source, sink_shape_in_source_orientation, order=order);
#print(axes_order, shape_order)
if len(axes_order) == 0:
if verbose:
print('resampling: no resampling necessary, source has same size as sink!');
if sink != source:
return io.write(sink, source);
else:
return source;
#resample
n_steps = len(axes_order);
last_source = source;
delete_files = [];
for step, axes, shape in zip(range(n_steps), axes_order, shape_order):
if step == n_steps-1 and orientation is None:
resampled = io.initialize(source=sink, shape=sink_shape, dtype=dtype, as_source=True);
else:
if method == 'shared':
resampled = io.sma.create(shape, dtype=dtype, order=order, as_source=True);
else:
location = tempfile.mktemp() + '.npy';
resampled = io.mmp.create(location, shape=shape, dtype=dtype, order=order, as_source=True);
delete_files.append(location);
#print(resampled)
#indices for non-resampled axes
indices = tuple([range(s) for d,s in enumerate(shape) if d not in axes]);
indices = [i for i in itertools.product(*indices)];
n_indices = len(indices);
#resample step
last_source_virtual = last_source.as_virtual();
resampled_virtual = resampled.as_virtual();
_resample = ft.partial(_resample_2d, source=last_source_virtual, sink=resampled_virtual, axes=axes, shape=shape,
interpolation=interpolation, n_indices=n_indices, verbose=verbose)
if processes == 'serial':
for index in indices:
_resample(index=index);
else:
#print(processes);
with concurrent.futures.ProcessPoolExecutor(processes) as executor:
executor.map(_resample, indices);
last_source = resampled;
#fix orientation
if not orientation is None:
#permute
per = orientation_to_permuation(orientation);
resampled = resampled.transpose(per);
#reverse axes
reslice = False;
slicing = [slice(None)] * ndim;
for d,o in enumerate(orientation):
if o < 0:
slicing[d] = slice(None, None, -1);
reslice = True;
if reslice:
resampled = resampled[slicing];
if verbose:
print("resample: re-oriented shape %r!" % (resampled.shape,))
sink = io.write(sink, resampled);
else:
sink = resampled;
for f in delete_files:
io.delete_file(f);
if verbose:
timer.print_elapsed_time('Resampling')
return sink;
def initialize_sink(sink=None,
shape=None,
dtype=None,
order=None,
memory=None,
location=None,
mode=None,
source=None,
return_buffer=True,
as_1d=False,
return_shape=False,
return_strides=False):
"""Initialze or create a sink.
Arguments
---------
sink : sink specification
The source to initialize.
shape : tuple of int
Optional shape of the sink. If None, inferred from the source.
dtype : dtype
Optional dtype of the sink. If None, inferred from the source.
order : 'C', 'F' or None
Optonal order of the sink. If None, inferred from the source.
memory : 'shared' or None
If 'shared' create a shared memory sink.
location : str
Optional location specification of the sink.
source : Source or None
Optional source to infer sink specifictions from.
return_buffer : bool
If True, return alos a buffer compatible with cython memory views.
return_shape : bool
If True, also return shape of the sink.
return_strides : bool
If True, also return the element strides of the sink.
Returns
-------
sink : Source
The intialized sink.
buffer : array
Buffer of the sink.
shape : tuple of int
Shape of the source.
strides : tuple of int
Element strides of the source.
"""
sink = io.initialize(sink,
shape=shape,
dtype=dtype,
order=order,
memory=memory,
location=location,
mode=mode,
like=source,
as_source=True)
if return_buffer:
buffer = sink.as_buffer()
if buffer.dtype == bool:
buffer = sink.view('uint8')
if as_1d:
buffer = buffer.reshape(-1, order='A')
result = (sink, )
if return_buffer:
result += (buffer, )
if return_shape:
result += (np.array(sink.shape, dtype=int), )
if return_strides:
result += (np.array(sink.element_strides, dtype=int), )
if len(result) == 1:
return result[0]
else:
return result
