def _initialize_sink(sink = None, source = None, return_1d = False, return_strides = False, shape = None, dtype = None, order = None, memory = None, location = None):
"""Helper to initialize sinks."""
if sink is None:
if shape is None:
if source is None:
raise ValueError("Cannot determine sink without source.");
else:
shape = source.shape;
if dtype is None:
if source is None:
raise ValueError("Cannot determine sink without source.");
else:
dtype = source.dtype;
if order is None and source is not None:
order = io.order(source);
#print('shape=%r, dtype=%r, order=%r, memory=%r, location=%r' % (shape, dtype, order, memory, location));
sink = io.prepare_sink(sink, shape=shape, dtype=dtype, order=order, memory=memory, location=location);
if sink.dtype == bool:
sink = sink.view('uint8');
if return_strides:
strides = io.element_strides(sink);
if return_1d:
sink1d = sink.reshape(-1, order = 'A');
if not return_strides and not return_1d:
return sink;
result = (sink,)
if return_strides:
result += (strides,);
if return_1d:
result += (sink1d,);
return result;
def detect_cells(
source,
sink=None,
cell_detection_parameter=default_cell_detection_parameter,
processing_parameter=default_cell_detection_processing_parameter):
"""Cell detection pipeline.
Arguments
---------
source : source specification
The source of the stitched raw data.
sink : sink specification or None
The sink to write the result to. If None, an array is returned.
cell_detection_parameter : dict
Parameter for the binarization. See below for details.
processing_parameter : dict
Parameter for the parallel processing.
See :func:`ClearMap.ParallelProcessing.BlockProcesing.process` for
description of all the parameter.
verbose : bool
If True, print progress output.
Returns
-------
sink : Source
The result of the cell detection.
Notes
-----
Effectively this function performs the following steps:
* illumination correction via :func:`~ClearMap.ImageProcessing.IlluminationCorrection.correct_illumination`
* background removal
* difference of Gaussians (DoG) filter
* maxima detection via :func:`~ClearMap.Analysis.Measurements.MaximaDetection.find_extended_maxima`
* cell shape detection via :func:`~ClearMap.Analysis.Measurements.ShapeDetection.detect_shape`
* cell intensity and size measurements via: :func:`~ClearMap.ImageProcessing.Measurements.ShapeDetection.find_intensity`,
:func:`~ClearMap.ImageProcessing.Measurements.ShapeDetection.find_size`.
The parameters for each step are passed as sub-dictionaries to the
cell_detection_parameter dictionary.
* If None is passed for one of the steps this step is skipped.
* Each step also has an additional parameter 'save' that enables saving of
the result of that step to a file to inspect the pipeline.
Illumination correction
-----------------------
illumination_correction : dict or None
Illumination correction step parameter.
flatfield : array or str
The flat field estimate for the image planes.
background : array or None
A background level to assume for the flatfield correction.
scaling : float, 'max', 'mean' or None
Optional scaling after the flat field correction.
save : str or None
Save the result of this step to the specified file if not None.
See also :func:`ClearMap.ImageProcessing.IlluminationCorrection.correct_illumination`
Background removal
------------------
background_correction : dict or None
Background removal step parameter.
shape : tuple
The shape of the structure lement to estimate the background.
This should be larger than the typical cell size.
form : str
The form of the structur element (e.g. 'Disk')
save : str or None
Save the result of this step to the specified file if not None.
Equalization
------------
equalization : dict or None
Equalization step parameter.
See also :func:`ClearMap.ImageProcessing.LocalStatistics.local_percentile`
precentile : tuple
The lower and upper percentiles used to estimate the equalization.
The lower percentile is used for normalization, the upper to limit the
maximal boost to a maximal intensity above this percentile.
max_value : float
The maximal intensity value in the equalized image.
selem : tuple
The structural element size to estimate the percentiles.
Should be larger than the larger vessels.
spacing : tuple
The spacing used to move the structural elements.
Larger spacings speed up processing but become locally less precise.
interpolate : int
The order of the interpoltation used in constructing the full
background estimate in case a non-trivial spacing is used.
save : str or None
Save the result of this step to the specified file if not None.
DoG Filter
----------
dog_filter : dict or None
Difference of Gaussian filter step parameter.
shape : tuple
The shape of the filter.
This should be near the typical cell size.
sigma : tuple or None
The std of the inner Gaussian.
If None, detemined automatically from shape.
sigma2 : tuple or None
The std of the outer Gaussian.
If None, detemined automatically from shape.
save : str or None
Save the result of this step to the specified file if not None.
Maxima detection
----------------
maxima_detection : dict or None
Extended maxima detection step parameter.
h_max : float or None
The 'height'for the extended maxima.
If None, simple local maxima detection isused.
shape : tuple
The shape of the structural element for extended maxima detection.
This should be near the typical cell size.
threshold : float or None
Only maxima above this threshold are detected. If None, all maxima
are detected.
valid : bool
If True, only detect cell centers in the valid range of the blocks with
overlap.
save : str or None
Save the result of this step to the specified file if not None.
Shape detection
---------------
shape_detection : dict or None
Shape detection step parameter.
threshold : float
Cell shape is expanded from maxima if pixles are above this threshold
and not closer to another maxima.
save : str or None
Save the result of this step to the specified file if not None.
Intensity detection
-------------------
intensity_detection : dict or None
Intensity detection step parameter.
method : {'max'|'min','mean'|'sum'}
The method to use to measure the intensity of a cell.
shape : tuple or None
If no cell shapes are detected a disk of this shape is used to measure
the cell intensity.
save : str or None
Save the result of this step to the specified file if not None.
References
----------
[1] Renier, Adams, Kirst, Wu et al., "Mapping of Brain Activity by Automated Volume Analysis of Immediate Early Genes.", Cell 165, 1789 (2016)
[1] Kirst et al., "Mapping the Fine-Scale Organization and Plasticity of the Brain Vasculature", Cell 180, 780 (2020)
"""
#initialize sink
shape = io.shape(source)
order = io.order(source)
for key in cell_detection_parameter.keys():
par = cell_detection_parameter[key]
if isinstance(par, dict):
filename = par.get('save', None)
if filename:
ap.initialize_sink(filename,
shape=shape,
order=order,
dtype='float')
cell_detection_parameter.update(
verbose=processing_parameter.get('verbose', False))
results, blocks = bp.process(detect_cells_block,
source,
sink=None,
function_type='block',
return_result=True,
return_blocks=True,
parameter=cell_detection_parameter,
**processing_parameter)
#merge results
results = np.vstack([np.hstack(r) for r in results])
#create column headers
header = ['x', 'y', 'z']
dtypes = [int, int, int]
if cell_detection_parameter['shape_detection'] is not None:
header += ['size']
dtypes += [int]
measures = cell_detection_parameter['intensity_detection']['measure']
header += measures
dtypes += [float] * len(measures)
dt = {
'names': header,
'formats': dtypes
}
cells = np.zeros(len(results), dtype=dt)
for i, h in enumerate(header):
cells[h] = results[:, i]
#save results
return io.write(sink, cells)
def binarize(source, sink = None, binarization_parameter = default_binarization_parameter, processing_parameter = default_binarization_processing_parameter):
"""Multi-path binarization of iDISCO+ cleared vasculature data.
Arguments
---------
source : source specification
The source of the stitched raw data.
sink : sink specification or None
The sink to write the result to. If None, an array is returned.
binarization_parameter : dict
Parameter for the binarization. See below for details.
processing_parameter : dict
Parameter for the parallel processing.
See :func:`ClearMap.ParallelProcessing.BlockProcesing.process` for
description of all the parameter.
verbose : bool
If True, print progress output.
Returns
-------
sink : Source
The result of the binarization.
Notes
-----
* The binarization pipeline is composed of several steps. The parameters for
each step are passed as sub-dictionaries to the binarization_parameter
dictionary.
* If None is passed for one of the steps this step is skipped.
* Each step also has an additional parameter 'save' that enables saving of
the result of that step to a file to inspect the pipeline.
General parameter
-----------------
binary_status : str or None
File name to save the information about which part of the multi-path
binarization contributed to the final result.
max_bin : int
Number of intensity levels to use for the data after preprocessing.
Higher values will increase the intensity resolution but slow down
processing.
For the vasculature a typical value is 2**12.
Clipping
--------
clip : dict or None
Clipping and mask generation step parameter.
clip_range : tuple
The range to clip the raw data as (lowest, highest)
Voxels above lowest define the foregournd mask used
in the following steps.
For the vasculature a typical value is (400,60000).
save : str or None
Save the result of this step to the specified file if not None.
See also :mod:`ClearMap.ImageProcessing.Clipping.Clipping`
Lightsheet correction
---------------------
lightsheet : dict or None
Lightsheet correction step parameter.
percentile : float
Percentile in [0,1] used to estimate the lightshieet artifact.
For the vasculature a typical value is 0.25.
lightsheet : dict
Parameter for the ligthsheet artifact percentile estimation.
See :func:`ClearMap.ImageProcessing.LightsheetCorrection.correct_lightsheet`
for list of all parameters. The crucial parameter is
selem : tuple
The structural element shape used to estimate the stripe artifact.
It should match the typical lenght, width, and depth of the artifact
in the data.
For the vasculature a typical value is (150,1,1).
background : dict
Parameter for the background estimation in the light sheet correction.
See :func:`ClearMap.ImageProcessing.LightsheetCorrection.correct_lightsheet`
for list of all parameters. The crucial parameters are
selem : tuple
The structural element shape used to estimate the background.
It should be bigger than the largest vessels,
For the vasculature a typical value is (200,200,1).
spacing : tuple
The spacing to use to estimate the background. Larger spacings speed up
processing but become less local estimates.
For the vasculature a typical value is (25,25,1)
step : tuple
This parameter enables to subsample from the entire array defined by
the structural element using larger than single voxel steps.
For the vasculature a typical value is (2,2,1).
interpolate : int
The order of the interpoltation used in constructing the full
background estimate in case a non-trivial spacing is used.
For the vasculature a typical value is 1.
lightsheet_vs_background : float
The background is multiplied by this weight before comparing to the
lightsheet artifact estimate.
For the vasculature a typical value is 2.
save : str or None
Save the result of this step to the specified file if not None.
Median filter
-------------
median : dict or None
Median correction step parameter.
See :func:`ClearMap.ImageProcessing.Filter.Rank.median` for all parameter.
The important parameters are
selem : tuple
The structural element size for the median filter.
For the vascualture a typical value is (3,3,3).
save : str or None
Save the result of this step to the specified file if not None.
Pseudo Deconvolution
--------------------
deconvolve : dict
The deconvolution step parameter.
sigma : float
The std of a Gaussina filter applied to the high intensity pixel image.
The number should reflect the scale of the halo effect seen around high
intensity structures.
For the vasculature a typical value is 10.
save : str or None
Save the result of this step to the specified file if not None.
threshold : float
Voxels above this threshold will be added to the binarization result
in the multi-path biniarization.
For the vasculature a typical value is 750.
Adaptive Thresholding
---------------------
adaptive : dict or None
Adaptive thresholding step parameter.
A local ISODATA threshold is estimated.
See also :mod:`ClearMap.ImageProcessing.LocalStatistics`.
selem : tuple
The structural element size to estimate the percentiles.
Should be larger than the larger vessels.
For the vasculature a typical value is (200,200,5).
spacing : tuple
The spacing used to move the structural elements.
Larger spacings speed up processing but become locally less precise.
For the vasculature a typical value is (50,50,5)
interpolate : int
The order of the interpoltation used in constructing the full
background estimate in case a non-trivial spacing is used.
For the vasculature a typical value is 1.
save : str or None
Save the result of this step to the specified file if not None.
Equalization
------------
equalize : dict or None
Equalization step parameter.
See also :func:`ClearMap.ImageProcessing.LocalStatistics.local_percentile`
precentile : tuple
The lower and upper percentiles used to estimate the equalization.
The lower percentile is used for normalization, the upper to limit the
maximal boost to a maximal intensity above this percentile.
For the vasculature a typical value is (0.4, 0.975).
max_value : float
The maximal intensity value in the equalized image.
For the vasculature a typical value is 1.5.
selem : tuple
The structural element size to estimate the percentiles.
Should be larger than the larger vessels.
For the vasculature a typical value is (200,200,5).
spacing : tuple
The spacing used to move the structural elements.
Larger spacings speed up processing but become locally less precise.
For the vasculature a typical value is (50,50,5)
interpolate : int
The order of the interpoltation used in constructing the full
background estimate in case a non-trivial spacing is used.
For the vasculature a typical value is 1.
save : str or None
Save the result of this step to the specified file if not None.
threshold : float
Voxels above this threshold will be added to the binarization result
in the multi-path biniarization.
For the vasculature a typical value is 1.1.
Tube filter
-----------
vesselize : dict
The tube filter step parameter.
background : dict or None
Parameters to correct for local background. See
:func:`ClearMap.ImageProcessing.Filter.Rank.percentile`.
If None, no background correction is done before the tube filter.
selem : tuple
The structural element specification to estimate the percentiles.
Should be larger than the largest vessels intended to be
boosted by the tube filter.
For the vasculature a typical value is ('disk', (30,30,1)) .
percentile : float
Percentile in [0,1] used to estimate the background.
For the vasculature a typical value is 0.5.
tubness : dict
Parameters used for the tube filter. See
:func:`ClearMap.ImageProcessing.Differentiation.Hessian.lambda123`.
sigma : float
The scale of the vessels to boos in the filter.
For the vasculature a typical value is 1.0.
save : str or None
Save the result of this step to the specified file if not None.
threshold : float
Voxels above this threshold will be added to the binarization result
in the multi-path biniarization.
For the vasculature a typical value is 120.
Binary filling
--------------
fill : dict or None
If not None, apply a binary filling the binarized result.
For the vasculature this step is set to None and done globally
in the postprocessing step.
Binary smoothing
----------------
smooth : dict or None
The smoothing parameter passed to
:func:`ClearMap.ImageProcessing.Binary.Smoothing.smooth_by_configuration`.
For the vasculature this step is set to None and done globally
in the postprocessing step.
References
----------
[1] C. Kirst et al., "Mapping the Fine-Scale Organization and Plasticity of the Brain Vasculature", Cell 180, 780 (2020)
"""
#initialize sink
shape = io.shape(source);
order = io.order(source);
sink, sink_buffer = ap.initialize_sink(sink=sink, shape=shape, order=order, dtype=bool); #, memory='shared');
#initialize addition output sinks
binary_status = binarization_parameter.get('binary_status', None);
if binary_status:
ap.initialize_sink(binary_status, source=sink, shape=shape, order=order, dtype='uint16');
for key in binarization_parameter.keys():
par = binarization_parameter[key];
if isinstance(par, dict):
filename = par.get('save', None);
if filename:
ap.initialize_sink(filename, shape=shape, order=order, dtype='float');
binarization_parameter.update(verbose=processing_parameter.get('verbose', False));
bp.process(binarize_block, source, sink, function_type='block', parameter=binarization_parameter, **processing_parameter)
return sink;
def threshold(source,
sink=None,
threshold=None,
hysteresis_threshold=None,
seeds=None,
background=None):
"""Hysteresis thresholding.
Arguments
---------
source : array
Input source.
sink : array or None
If None, a new array is allocated.
threshold : float
The threshold for the intial seeds.
hysteresis_threshold : float or None
The hysteresis threshold to extend the initial seeds.
If None, no hysteresis thresholding is performed.
seeds : array or None
The seeds from which to start the hysteresis thresholds
background : array or None
Exclude this area from the hysteresis thresholding.
Returns
-------
sink : array
Thresholded output.
"""
if threshold is None and seeds is None:
raise ValueError('The threshold and seeds cannot both be None!')
if source is sink:
raise NotImplementedError("Cannot perform operation in place.")
if sink is None:
sink = np.zeros(source.shape, dtype='int8', order=io.order(source))
source_flat = source.reshape(-1, order=io.order(source))
sink_flat = sink.reshape(-1, order=io.order(sink))
strides = np.array(io.element_strides(source))
if seeds is None:
seeds = np.where(source_flat >= threshold)[0]
else:
seeds_flat = seeds.reshape(-1, order=io.order(seeds))
seeds = np.where(seeds_flat)[0]
if hysteresis_threshold is not None:
parameter_index = np.zeros(0, dtype=int)
parameter_double = np.array([hysteresis_threshold], dtype=float)
if background is not None:
background_flat = background.reshape(-1,
order=io.order(background))
background_flat = background_flat.view(dtype='uint8')
code.threshold_to_background(source_flat, sink_flat,
background_flat, strides, seeds,
parameter_index, parameter_double)
else:
code.threshold(source_flat, sink_flat, strides, seeds,
parameter_index, parameter_double)
else:
sink_flat[seeds] = 1
return sink
def fill(source, sink=None, seeds=None, processes=None, verbose=False):
"""Fill binary holes.
Arguments
---------
source : array
Input source.
sink : array or None
If None, a new array is allocated.
Returns
-------
sink : array
Binary image with filled holes.
"""
if source is sink:
raise NotImplementedError("Cannot perform operation in place.")
if verbose:
print('Binary filling: initialized!')
timer = tmr.Timer()
#create temporary shared array
order = io.order(source)
#temp = sma.empty(source.shape, dtype='int8', order=order);
temp = np.empty(source.shape, dtype='int8', order=order)
source_flat = source.reshape(-1, order='A')
temp_flat = temp.reshape(-1, order='A')
if source_flat.dtype == bool:
source_flat = source_flat.view(dtype='uint8')
if processes is None:
processes = mp.cpu_count()
if not isinstance(processes, int):
processes = 1
#prepare flood fill
code.prepare_temp(source_flat, temp_flat, processes=processes)
#flood fill in parallel using mp
if seeds is None:
seeds = border_indices(source)
else:
seeds = np.where(seeds.reshape(-1, order=order))[0]
strides = np.array(io.element_strides(source))
code.label_temp(temp_flat, strides, seeds, processes=processes)
if sink is None:
#sink = sma.empty(source.shape, dtype=bool, order=order);
sink = np.empty(source.shape, dtype=bool, order=order)
sink_flat = sink.reshape(-1, order=order)
if sink_flat.dtype == 'bool':
sink_flat = sink_flat.view(dtype='uint8')
code.fill(source_flat, temp_flat, sink_flat, processes=processes)
if verbose:
timer.print_elapsed_time('Binary filling')
del temp, temp_flat
gc.collect()
return sink
