def _initialize_processing(processes = None, verbose = False, function = None):
"""Initialize parallel array processing.
Arguments
---------
processes : int, 'seial' or None
The number of processes to use. If None use number of cpus.
verbose : bool
If True, print progress information.
function : str or None
The nae of the function.
Returns
-------
processes : int
The number of processes.
timer : Timer
A timer for the processing.
"""
if processes is None:
processes = default_processes;
if processes == 'serial':
processes = 1;
if verbose:
if function:
print('%s: initialized!' % function);
timer = tmr.Timer();
else:
timer = None;
return processes, timer
def find_size(label, max_label=None, verbose=False):
"""Find size given object shapes as a labled image
Arguments
---------
label : array, str or Source
Labeled image in which each object has its own label.
max_label : int or None
Maximal label to include, if None use all label.
verbose : bool
Print progress info.
Returns
-------
sizes : array
Measured intensities
"""
if verbose:
timer = tmr.Timer()
hdict.pprint(head='Size detection:', max_label=max_label)
label = io.as_source(label)
if max_label is None:
max_label = int(label.max())
sizes = scipy.ndimage.measurements.sum(np.ones(label.shape, dtype=bool),
labels=label,
index=np.arange(1, max_label + 1))
if verbose:
timer.print_elapsed_time(head='Size detection')
return sizes
def grey_reconstruct(source,
mask=None,
sink=None,
method=None,
shape=3,
verbose=False):
"""Calculates the grey reconstruction of the image
Arguments
---------
source : array
The source image data.
method : 'dilation' or 'erosion' or None
The mehtjod to use, if None return original image.
shape : in or tuple
Shape of the strucuturing element for the grey reconstruction.
verbose : boo;
If True, print progress info.
Returns
-------
reconstructed: array
Grey reconstructed image.
Note
----
The reconstruction is done slice by slice along the z-axis.
"""
if verbose:
timer = tmr.Timer()
hdict.pprint(head='Grey reconstruction', method=method, shape=shape)
if method is None:
return source
if sink is None:
sink = np.empty(source.shape, dtype=source.dtype)
# background subtraction in each slice
selem = se.structure_element(form='Disk', shape=shape,
ndim=2).astype('uint8')
for z in range(source.shape[2]):
#img[:,:,z] = img[:,:,z] - grey_opening(img[:,:,z], structure = structureElement('Disk', (30,30)));
#img[:,:,z] = img[:,:,z] - morph.grey_opening(img[:,:,z], structure = self.structureELement('Disk', (150,150)));
sink[:, :, z] = source[:, :, z] - reconstruct(
source[:, :, z], mask=mask[:, :, z], method=method, selem=selem)
if verbose:
timer.print_elapsed_time('Grey reconstruction')
return sink
def _test():
import numpy as np
import ClearMap.Utils.Timer as tmr
import ClearMap.ImageProcessing.Clipping.Clipping as clp
data = np.random.rand(1000,1000,2000);
for p in [1, 10, None]:
print('Clipping: processes = %r' % p);
timer = tmr.Timer();
clipped = clp.clip(data, clip_max = 0.5, processes = p);
timer.print_elapsed_time('Clipping');
def find_intensity(source, label, max_label=None, method='sum', verbose=False):
"""Find integrated intensity given object shapes as labled image.
Arguments
---------
source : array, str, or Source
Source to measure intensities from.
label : array, str, or Source
Labeled image with a separate label for each object.
max_label : int or None
Maximal label to include. If None use all.
method : {'sum', 'mean', 'max', 'min'}
Method to use to measure the intensities in each object's area.
verbose : bool
If True, print progress information.
Returns
-------
intensities : array
Measured intensities.
"""
if verbose:
timer = tmr.Timer()
hdict.pprint(head='Intensity detection:',
max_label=max_label,
method=method)
source = io.as_source(source).array
label = io.as_source(label)
if max_label is None:
max_label = label.max()
if method.lower() == 'sum':
measure = scipy.ndimage.measurements.sum
elif method.lower() == 'mean':
measure = scipy.ndimage.measurements.mean
elif method.lower() == 'max':
measure = scipy.ndimage.measurements.maximum
elif method.lower() == 'min':
measure = scipy.ndimage.measurements.minimum
else:
raise RuntimeError('Unkown method %r!' % (method, ))
intensities = measure(label,
labels=label,
index=np.arange(1, max_label + 1))
if verbose:
timer.print_elapsed_time(head='Intensity detection')
return intensities
def find_maxima(source, h_max=None, shape=5, threshold=None, verbose=None):
"""Find local and extended maxima in an image.
Arguments
---------
source : array
The source data.
h_max : float or None
H parameter for the initial h-Max transform.
If None, do not perform a h-max transform.
shape : int or tuple
Shape for the structure element for the local maxima filter.
threshold : float or None
If float, include only maxima larger than this threshold.
verbose : bool
Print progress info.
Returns
-------
maxima : array
Binary image with True pixel at extended maxima.
Notes
-----
This routine performs a h-max transfrom, followed by a local maxima search
and thresholding of the maxima.
See also
--------
:func:`h_max_transform`, :func:`local_max`
"""
if verbose:
timer = tmr.Timer()
hdict.pprint(head='Find Maxima:',
h_max=h_max,
shape=shape,
threshold=threshold)
# extended maxima
maxima = h_max_transform(source, h_max=h_max)
#local maxima
maxima = local_max(maxima, shape=shape)
#thresholding
if not threshold is None:
maxima = np.logical_and(maxima, source >= threshold)
if verbose:
timer.print_elapsed_time(head='Find Maxima')
return maxima
def convert_files(filenames, extension = None, path = None, processes = None, verbose = False):
"""Transforms list of files to their sink format in parallel.
Arguments
---------
filenames : list of str
The filenames to convert
extension : str
The new file format extension.
path : str or None
Optional path speicfication.
processes : int, 'serial' or None
The number of processes to use for parallel conversion.
verbose : bool
If True, print progress information.
Returns
-------
filenames : list of str
The new file names.
"""
if not isinstance(filenames, (tuple, list)):
filenames = [filenames];
if len(filenames) == 0:
return [];
n_files = len(filenames);
if path is not None:
filenames = [fu.join(path, fu.split(f)[1]) for f in filenames];
sinks = ['.'.join(f.split('.')[:-1] + [extension]) for f in filenames];
if verbose:
timer = tmr.Timer()
print('Converting %d files to %s!' % (n_files, extension));
if not isinstance(processes, int) and processes != 'serial':
processes = mp.cpu_count();
#print(n_files, extension, filenames, sinks)
_convert = functools.partial(_convert_files, n_files=n_files, extension=extension, verbose=verbose);
if processes == 'serial':
[_convert(source,sink,i) for i,source,sink in zip(range(n_files), filenames, sinks)];
else:
with concurrent.futures.ProcessPoolExecutor(processes) as executor:
executor.map(_convert, filenames, sinks, range(n_files));
if verbose:
timer.print_elapsed_time('Converting %d files to %s' % (n_files, extension));
return sinks;
def rank(source,
sink=None,
function=rnk.median,
resample=None,
verbose=False,
out=sys.stdout,
**kwargs):
"""Rank filter inbetween reshaping."""
timer = tmr.Timer()
sink, sink_buffer = ap.initialize_sink(sink=sink, source=source, order='F')
if resample:
interpolation = cv2.INTER_NEAREST
new_shape = np.round(np.array(sink.shape, dtype=float) *
resample).astype(int)
new_shape[2] = sink.shape[2]
data = np.zeros(tuple(new_shape), order='F', dtype=source.dtype)
new_shape = tuple(new_shape[1::-1])
for z in range(source.shape[2]):
data[:, :, z] = cv2.resize(src=source[:, :, z],
dsize=new_shape,
interpolation=interpolation)
#print data.shape, data.dtype
out.write(timer.elapsed_time(head='Rank filter: Resampling') + '\n')
else:
data = source
#keys = inspect.getargspec(function).args;
#kwargs = { k : v for k,v in kwargs.iteritems() if k in keys};
data = function(data, **kwargs)
out.write(
timer.elapsed_time(head='Rank filter: %s' % function.__name__) + '\n')
if resample:
#interpolation = cv2.INTER_LINEAR;
interpolation = cv2.INTER_AREA
for z in range(sink.shape[2]):
sink_buffer[:, :, z] = cv2.resize(src=data[:, :, z],
dsize=sink.shape[1::-1],
interpolation=interpolation)
out.write(timer.elapsed_time(head='Rank filter: Upsampling') + '\n')
else:
sink_buffer[:] = data
return sink
def process_block_source(sources,
sinks,
function,
as_memory=False,
as_array=False,
verbose=False,
**kwargs):
"""Process a block with full traceback.
Arguments
---------
sources : source specifications
Sources passed to the function.
sinks : sourcespecifications
Sinks where data is written to.
function func : function
The function to call.
"""
if verbose:
timer = tmr.Timer()
print('Processing block %s' % (sources[0].info(), ))
#sources = [s.as_real() for s in sources];
sources_input = sources
if as_memory:
sources = [s.as_memory() for s in sources]
if as_array:
sources = [s.array for s in sources]
results = function(*sources, **kwargs)
if not isinstance(results, (list, tuple)):
results = [results]
if len(sources_input) != len(sinks):
sources_input = sources_input + [sources_input[0]
] * (len(sinks) - len(sources))
for sink, source, result in zip(sinks, sources_input, results):
#sink = sink.as_real();
sink.valid[:] = result[source.valid.slicing]
if verbose:
timer.print_elapsed_time('Processing block %s' %
(sources_input[0].info(), ))
gc.collect()
return None
def detect_shape(source, seeds, threshold=None, verbose=False):
"""Detect object shapes by generatng a labeled image from seeds.
Arguments
---------
source : array, str or Source
Source image.
seeds : array, str or Source
Cell centers as point coordinates.
threshold : float or None
Threshold to determine mask for watershed, pixel below this are
treated as background. If None, the seeds are expanded indefinately.
verbose :bool
If True, print progress info.
Returns
-------
shapes : array
Labeled image, where each label indicates a object.
"""
if verbose:
timer = tmr.Timer()
hdict.pprint(head='Shape detection', threshold=threshold)
source = io.as_source(source).array
seeds = io.as_source(seeds)
if threshold is None:
mask = None
else:
mask = source > threshold
peaks = vox.voxelize(seeds,
shape=source.shape,
weights=np.arange(1, seeds.shape[0] + 1)).array
shapes = skimage.morphology.watershed(-source, peaks, mask=mask)
#shapes = watershed_ift(-source.astype('uint16'), peaks);
#shapes[numpy.logical_not(mask)] = 0;
if verbose:
timer.print_elapsed_time('Shape detection')
return shapes
def initialize_processing(processes=None,
verbose=False,
function=None,
blocks=None,
return_blocks=False):
"""Initialize parallel array processing.
Arguments
---------
processes : int, 'seial' or None
The number of processes to use. If None use number of cpus.
verbose : bool
If True, print progress information.
function : str or None
The nae of the function.
Returns
-------
processes : int
The number of processes.
timer : Timer
A timer for the processing.
"""
if processes is None:
processes = default_processes
if processes == 'serial':
processes = 1
if verbose:
if function:
print('%s: initialized!' % function)
timer = tmr.Timer()
else:
timer = None
results = (processes, timer)
if return_blocks:
if blocks is None:
blocks = processes * default_blocks_per_process
results += (blocks, )
return results
def process_block_block(sources,
sinks,
function,
as_memory=False,
return_result=False,
verbose=False,
**kwargs):
"""Process a block with full traceback.
Arguments
---------
sources : source specifications
Sources passed to the function.
sinks : sourcespecifications
Sinks where data is written to.
function func : function
The function to call.
"""
if verbose:
timer = tmr.Timer()
print('Processing block %s' % (sources[0].info(), ))
if as_memory:
sinks = sinks
sinks_memory = [s.as_memory_block() for s in sinks]
sources_and_sinks = [s.as_memory_block()
for s in sources] + sinks_memory
else:
sources_and_sinks = sources + sinks
result = function(*sources_and_sinks, **kwargs)
if as_memory:
for sink, sink_memory in zip(sinks, sinks_memory):
sink.valid[:] = sink_memory.valid[:]
if verbose:
timer.print_elapsed_time('Processing block %s' % (sources[0].info(), ))
gc.collect()
if return_result:
return result
else:
return None
def find_center_of_maxima(source, maxima=None, label=None, verbose=False):
"""Find center of detected maxima weighted by intensity
Arguments
---------
source : array
Intensity image data.
maxima : array or None
Binary array indicating the maxima. I label is not None this can be None.
label : array or None
Labeled image of the shapes of the maxima.
If None, determined from maxima.
verbose : bool
Print progress info.
Returns
-------
coordinates : array
Coordinates of the n centers of maxima as (n,d)-array.
"""
if verbose:
timer = tmr.Timer()
print('Center of Maxima initialized!')
#center of maxima
if label is None:
label, n_label = ndm.label(maxima)
else:
n_label = label.max()
if n_label > 0:
centers = np.array(
ndm.center_of_mass(source, label, index=np.arange(1, n_label)))
else:
centers = np.zeros((0, source.ndim))
if verbose:
timer.print_elapsed_time('Center of Maxima: %d maxima detected' %
centers.shape[0])
return centers
def resample(source, sink = None, orientation = None,
sink_shape = None, source_resolution = None, sink_resolution = None,
interpolation = 'linear', axes_order = None, method = 'shared',
processes = None, verbose = True):
"""Resample data of source in new shape/resolution and orientation.
Arguments
---------
source : str or array
The source to be resampled.
sink : str or None
The sink for the resampled image.
orientation : tuple or None:
The orientation specified by permuation and change in sign of (1,2,3).
sink_shape : tuple or None
The target shape of the resampled sink.
source_resolution : tuple or None
The resolution of the source (in length per pixel).
sink_resolution : tuple or None
The resolution of the resampled source (in length per pixel).
interpolation : str
The method to use for interpolating to the resmapled array.
axis_order : str, list of tuples of int or None
The axes pairs along which to resample the data at each step.
If None, this is detertmined automatically. For a FileList source,
setting the first tuple should point to axis not indicating files.
If 'size' the axis order is determined automatically to maximally reduce
the size of the array in each resmapling step.
If 'order' the axis order is chosed automatically to optimize io speed.
method : 'shared' or 'memmap'
Method to handle intermediate resampling results. If 'shared' use shared
memory, otherwise use a memory map on disk.
processes : int, None or 'serial'
Number of processes to use for parallel resampling, if None use maximal
processes avaialable, if 'serial' process in serial.
verbose : bool
If True, display progress information.
Returns
-------
sink : array or str
The data or filename of resampled sink.
Notes
-----
* Resolutions are assumed to be given for the axes of the intrinsic
orientation of the data and reference (as when viewed by ImageJ).
* Orientation: permuation of 1,2,3 with potential sign, indicating which
axes map onto the reference axes, a negative sign indicates reversal
of that particular axes.
* Only a minimal set of information to determine the resampling parameter
has to be given, e.g. source_shape and sink_shape.
* The resampling is done by iterating two dimensional resampling steps.
"""
#TODO: write full nd resampling routine extending cv2 lib.
if verbose:
timer = tmr.Timer();
source = io.as_source(source);
source_shape = source.shape;
ndim = len(source_shape);
dtype = source.dtype;
order = source.order;
orientation = format_orientation(orientation);
source_shape, sink_shape, source_resolution, sink_resolution = \
resample_shape(source_shape=source_shape, sink_shape=sink_shape,
source_resolution=source_resolution, sink_resolution=sink_resolution,
orientation=orientation);
sink_shape_in_source_orientation = orient_shape(sink_shape, orientation, inverse=True);
interpolation = _interpolation_to_cv2(interpolation);
if not isinstance(processes, int) and processes != 'serial':
processes = io.mp.cpu_count();
#detemine order of resampling
axes_order, shape_order = _axes_order(axes_order, source, sink_shape_in_source_orientation, order=order);
#print(axes_order, shape_order)
if len(axes_order) == 0:
if verbose:
print('resampling: no resampling necessary, source has same size as sink!');
if sink != source:
return io.write(sink, source);
else:
return source;
#resample
n_steps = len(axes_order);
last_source = source;
delete_files = [];
for step, axes, shape in zip(range(n_steps), axes_order, shape_order):
if step == n_steps-1 and orientation is None:
resampled = io.initialize(source=sink, shape=sink_shape, dtype=dtype, as_source=True);
else:
if method == 'shared':
resampled = io.sma.create(shape, dtype=dtype, order=order, as_source=True);
else:
location = tempfile.mktemp() + '.npy';
resampled = io.mmp.create(location, shape=shape, dtype=dtype, order=order, as_source=True);
delete_files.append(location);
#print(resampled)
#indices for non-resampled axes
indices = tuple([range(s) for d,s in enumerate(shape) if d not in axes]);
indices = [i for i in itertools.product(*indices)];
n_indices = len(indices);
#resample step
last_source_virtual = last_source.as_virtual();
resampled_virtual = resampled.as_virtual();
_resample = ft.partial(_resample_2d, source=last_source_virtual, sink=resampled_virtual, axes=axes, shape=shape,
interpolation=interpolation, n_indices=n_indices, verbose=verbose)
if processes == 'serial':
for index in indices:
_resample(index=index);
else:
#print(processes);
with concurrent.futures.ProcessPoolExecutor(processes) as executor:
executor.map(_resample, indices);
last_source = resampled;
#fix orientation
if not orientation is None:
#permute
per = orientation_to_permuation(orientation);
resampled = resampled.transpose(per);
#reverse axes
reslice = False;
slicing = [slice(None)] * ndim;
for d,o in enumerate(orientation):
if o < 0:
slicing[d] = slice(None, None, -1);
reslice = True;
if reslice:
resampled = resampled[slicing];
if verbose:
print("resample: re-oriented shape %r!" % (resampled.shape,))
sink = io.write(sink, resampled);
else:
sink = resampled;
for f in delete_files:
io.delete_file(f);
if verbose:
timer.print_elapsed_time('Resampling')
return sink;
def smooth_by_configuration(source, sink = None, iterations = 1,
processing_parameter = None,
processes = None, verbose = False):
"""Smooth a binary source using the local configuration around each pixel.
Arguments
---------
source : array or Source
The binary source to smooth.
sink : array, Source or None
The sink to write result of smoothing. If None, return array.
iterations : int
Number of smoothing iterations.
processing_parameter : None or dict
The parameter passed to
:func:`ClearMap.ParallelProcessing.BlockProcessing.process`.
processes : int or None
number of processes to use.
verbose : bool
If True, print progress information.
Returns
-------
smoothed : array or Source
Thre smoothed binary array.
Note
----
The algorithm is based on a topological smoothing operation defined by adding
or removing forground pixels based on the local topology of the binary array.
"""
if verbose:
print('Binary smoothing: initialized!');
timer = tmr.Timer();
#smoothing function
smooth = functools.partial(smooth_by_configuration_block, iterations=iterations, verbose=False);
smooth.__name__ = 'smooth_by_configuration'
#initialize sources and sinks
source = io.as_source(source);
sink = io.initialize(sink, shape=source.shape, dtype=bool, order=source.order);
#block processing parameter
block_processing_parameter = dict(axes = bp.block_axes(source),
as_memory=True,
overlap=None,
function_type='source',
processes=processes,
verbose=verbose);
if processing_parameter is not None:
block_processing_parameter.update(processing_parameter);
if not 'overlap' in block_processing_parameter or block_processing_parameter['overlap'] is None:
block_processing_parameter['overlap'] = 2 + 2 * iterations;
if not 'size_min' in block_processing_parameter or block_processing_parameter['size_min'] is None:
block_processing_parameter['size_min'] = 2 + 2 * iterations + 1;
if not 'axes' in block_processing_parameter or block_processing_parameter['axes'] is None:
block_processing_parameter['axes'] = bp.block_axes(source);
#print(block_processing_parameter)
#block process
bp.process(smooth, source, sink, **block_processing_parameter);
if verbose:
timer.print_elapsed_time('Binary smoothing: done');
return sink;
def postprocess(source, sink = None, postprocessing_parameter = default_postprocessing_parameter, processing_parameter = default_postprocessing_processing_parameter, processes = None, verbose = True):
"""Postprocess a binarized image.
Arguments
---------
source : source specification
The binary source.
sink : sink specification or None
The sink to write the postprocesses result to.
If None, an array is returned.
postprocessing_parameter : dict
Parameter for the postprocessing.
processing_parameter : dict
Parameter for the parallel processing.
verbose : bool
If True, print progress output.
Returns
-------
sink : Source
The result of the binarization.
Notes
-----
* The postporcessing pipeline is composed of several steps. The parameters
for each step are passed as sub-dictionaries to the
postprocessing_parameter dictionary.
* If None is passed for one of the steps the step is skipped.
Smoothing
---------
smooth : dict or None
Smoothing step parameter. See
:func:`ClearMap.ImageProcessing.Binary.Smoothing.smooth_by_configuration`
iterations : int
Number of smoothing iterations.
For the vasculature a typical value is 6.
Filling
-------
fill : bool or None
If True, fill holes in the binary data.
"""
source = io.as_source(source);
sink = ap.initialize_sink(sink, shape=source.shape, dtype=source.dtype, order=source.order, return_buffer=False);
if verbose:
timer = tmr.Timer();
print('Binary post processing: initialized.');
postprocessing_parameter = postprocessing_parameter.copy();
parameter_smooth = postprocessing_parameter.pop('smooth', None);
parameter_fill = postprocessing_parameter.pop('fill', None);
#print(parameter_smooth, parameter_fill)
#smoothing
save = None;
if parameter_smooth:
#intialize temporary files if needed
if parameter_fill:
save = parameter_smooth.pop('save', None);
temporary_filename = save;
if temporary_filename is None:
temporary_filename = postprocessing_parameter['temporary_filename'];
if temporary_filename is None:
temporary_filename = tmpf.mktemp(prefix='TubeMap_Vasculature_postprocessing', suffix='.npy');
sink_smooth = ap.initialize_sink(temporary_filename, shape=source.shape, dtype=source.dtype, order=source.order, return_buffer=False);
else:
sink_smooth = sink;
#run smoothing
source_fill = bs.smooth_by_configuration(source, sink=sink_smooth, processing_parameter=processing_parameter, processes=processes, verbose=verbose, **parameter_smooth);
else:
source_fill = source;
if parameter_fill:
sink = bf.fill(source_fill, sink=sink, processes=processes, verbose=verbose);
if parameter_smooth and save is None:
io.delete_file(temporary_filename);
else:
sink = source_fill;
if verbose:
timer.print_elapsed_time('Binary post processing');
gc.collect()
return None;
def binarize_block(source, sink, parameter = default_binarization_parameter):
"""Binarize a Block."""
#initialize parameter and slicings
verbose = parameter.get('verbose', False);
if verbose:
prefix = 'Block %s: ' % (source.info(),);
total_time = tmr.Timer(prefix);
max_bin = parameter.get('max_bin', MAX_BIN);
base_slicing = sink.valid.base_slicing;
valid_slicing = source.valid.slicing;
#initialize binary status for inspection
binary_status = parameter.get('binary_status', None);
if binary_status:
binary_status = io.as_source(binary_status);
binary_status = binary_status[base_slicing];
#clipping
parameter_clip = parameter.get('clip', None);
if parameter_clip:
parameter_clip = parameter_clip.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_clip, head = prefix + 'Clipping:')
parameter_clip.update(norm=max_bin, dtype=DTYPE);
save = parameter_clip.pop('save', None);
clipped, mask, high, low = clip(source, **parameter_clip);
not_low = np.logical_not(low);
if save:
save = io.as_source(save);
save[base_slicing] = clipped[valid_slicing];
if binary_status is not None:
binary_status[high[valid_slicing]] += BINARY_STATUS['High']
else:
sink[valid_slicing] = high[valid_slicing];
del high, low
if verbose:
timer.print_elapsed_time('Clipping');
else:
clipped = source
mask = not_low = np.ones(source.shape, dtype=bool);
#high = low = np.zeros(source.shape, dtype=bool);
#low = np.zeros(source.shape, dtype=bool);
#not_low = np.logical_not(low);
#active arrays: clipped, mask, not_low
#lightsheet correction
parameter_lightsheet = parameter.get('lightsheet', None);
if parameter_lightsheet:
parameter_lightsheet = parameter_lightsheet.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_lightsheet, head = prefix + 'Lightsheet:')
#parameter_lightsheet.update(max_bin=max_bin);
save = parameter_lightsheet.pop('save', None);
corrected = lc.correct_lightsheet(clipped, mask=mask, max_bin=max_bin, **parameter_lightsheet);
if save:
save = io.as_source(save);
save[base_slicing] = corrected[valid_slicing];
if verbose:
timer.print_elapsed_time('Lightsheet');
else:
corrected = clipped;
del clipped
#active arrays: corrected, mask, not_low
#median filter
parameter_median = parameter.get('median', None);
if parameter_median:
parameter_median = parameter_median.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_median, head = prefix + 'Median:')
save = parameter_median.pop('save', None);
median = rnk.median(corrected, max_bin=max_bin, mask=not_low, **parameter_median);
if save:
save = io.as_source(save);
save[base_slicing] = median[valid_slicing];
if verbose:
timer.print_elapsed_time('Median');
else:
median = corrected;
del corrected, not_low;
#active arrays: median, mask
#pseudo deconvolution
parameter_deconvolution = parameter.get('deconvolve', None);
if parameter_deconvolution:
parameter_deconvolution = parameter_deconvolution.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_deconvolution, head = prefix + 'Deconvolution:')
save = parameter_deconvolution.pop('save', None);
threshold = parameter_deconvolution.pop('threshold', None);
if binary_status is not None:
binarized = binary_status > 0;
else:
binarized = sink[:];
deconvolved = deconvolve(median, binarized[:], **parameter_deconvolution)
del binarized
if save:
save = io.as_source(save);
save[base_slicing] = deconvolved[valid_slicing];
if verbose:
timer.print_elapsed_time('Deconvolution');
if threshold:
binary_deconvolved = deconvolved > threshold;
if binary_status is not None:
binary_status[binary_deconvolved[valid_slicing]] += BINARY_STATUS['Deconvolved'];
else:
sink[valid_slicing] += binary_deconvolved[valid_slicing];
del binary_deconvolved
if verbose:
timer.print_elapsed_time('Deconvolution: binarization');
else:
deconvolved = median;
#active arrays: median, mask, deconvolved
#adaptive
parameter_adaptive = parameter.get('adaptive', None);
if parameter_adaptive:
parameter_adaptive = parameter_adaptive.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_adaptive, head = prefix + 'Adaptive:')
save = parameter_adaptive.pop('save', None);
adaptive = threshold_adaptive(deconvolved, **parameter_adaptive)
if save:
save = io.as_source(save);
save[base_slicing] = adaptive[valid_slicing];
binary_adaptive = deconvolved > adaptive;
if binary_status is not None:
binary_status[binary_adaptive[valid_slicing]] += BINARY_STATUS['Adaptive'];
else:
sink[valid_slicing] += binary_adaptive[valid_slicing];
del binary_adaptive, adaptive;
if verbose:
timer.print_elapsed_time('Adaptive');
del deconvolved
#active arrays: median, mask
# equalize
parameter_equalize = parameter.get('equalize', None);
if parameter_equalize:
parameter_equalize = parameter_equalize.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_equalize, head = prefix + 'Equalization:')
save = parameter_equalize.pop('save', None);
threshold = parameter_equalize.pop('threshold', None);
equalized = equalize(median, mask=mask, **parameter_equalize);
if save:
save = io.as_source(save);
save[base_slicing] = equalized[valid_slicing];
if verbose:
timer.print_elapsed_time('Equalization');
if threshold:
binary_equalized = equalized > threshold
if binary_status is not None:
binary_status[binary_equalized[valid_slicing]] += BINARY_STATUS['Equalized'];
else:
sink[valid_slicing] += binary_equalized[valid_slicing];
#prepare equalized for use in vesselization
parameter_vesselization = parameter.get('vesselize', None);
if parameter_vesselization and parameter_vesselization.get('background', None):
equalized[binary_equalized] = threshold;
equalized = float(max_bin-1) / threshold * equalized;
del binary_equalized
if verbose:
timer.print_elapsed_time('Equalization: binarization');
else:
equalized = median;
del median
#active arrays: mask, equalized
# smaller vessels /capilarries
parameter_vesselization = parameter.get('vesselize', None);
if parameter_vesselization:
parameter_vesselization = parameter_vesselization.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_vesselization, head = prefix + 'Vesselization:')
parameter_background = parameter_vesselization.get('background', None)
parameter_background = parameter_background.copy();
if parameter_background:
save = parameter_background.pop('save', None);
equalized = np.array(equalized, dtype = 'uint16');
background = rnk.percentile(equalized, max_bin=max_bin, mask=mask, **parameter_background);
tubeness = equalized - np.minimum(equalized, background);
del background
if save:
save = io.as_source(save);
save[base_slicing] = tubeness[valid_slicing];
else:
tubeness = equalized;
parameter_tubeness = parameter_vesselization.get('tubeness', {})
tubeness = tubify(tubeness, **parameter_tubeness);
save = parameter_vesselization.get('save', None);
if save:
save = io.as_source(save);
save[base_slicing] = tubeness[valid_slicing];
if verbose:
timer.print_elapsed_time('Vesselization');
threshold = parameter_vesselization.get('threshold', None);
if threshold:
binary_vesselized = tubeness > threshold;
if binary_status is not None:
binary_status[binary_vesselized[valid_slicing]] += BINARY_STATUS['Tube'];
else:
sink[valid_slicing] += binary_vesselized[valid_slicing];
del binary_vesselized
if verbose:
timer.print_elapsed_time('Vesselization: binarization');
del tubeness
del equalized, mask
#active arrays: None
#fill holes
parameter_fill = parameter.get('fill', None);
if parameter_fill:
parameter_fill = parameter_fill.copy();
if verbose:
timer = tmr.Timer(prefix);
#hdict.pprint(parameter_fill, head = 'Filling:')
if binary_status is not None:
foreground = binary_status > 0;
filled = ndi.morphology.binary_fill_holes(foreground);
binary_status[np.logical_and(filled, np.logical_not(foreground))] += BINARY_STATUS['Fill'];
del foreground, filled
else:
filled = ndi.morphology.binary_fill_holes(sink[:]);
sink[valid_slicing] += filled[valid_slicing];
del filled
if verbose:
timer.print_elapsed_time('Filling');
if binary_status is not None:
sink[valid_slicing] = binary_status[valid_slicing] > 0;
#smooth binary
parameter_smooth = parameter.get('smooth', None);
if parameter_smooth:
parameter_smooth = parameter_smooth.copy();
if verbose:
timer = tmr.Timer(prefix);
hdict.pprint(parameter_smooth, head = prefix + 'Smoothing:')
smoothed = bs.smooth_by_configuration(sink, sink=None, processes=1, **parameter_smooth);
sink[valid_slicing] = smoothed[valid_slicing];
del smoothed;
if verbose:
timer.print_elapsed_time('Smoothing');
if verbose:
total_time.print_elapsed_time('Binarization')
gc.collect()
return None;
def measure_expression(source,
points,
search_radius,
method='max',
sink=None,
processes=None,
verbose=False):
"""Measures the expression around a list of points in a source.
Arguments
---------
source : array
Source for measurement.
points : array
List of indices to measure radis for.
search_radius : int or array
List of search radii to use around each point. If int use
this radius for all points. Array should be of length of points.
method : 'max' or 'min', 'mean'
Measurement type.
processes : int or None
Number of processes to use.
verbose : bool
If True, print progress info.
"""
source = io.as_source(source)
ndim = source.ndim
if verbose:
timer = tmr.Timer()
print('Measuring expression of %d points in array of shape %r.' %
(points.shape[0], source.shape))
if not hasattr(search_radius, '__len__'):
search_radius = search_radius * np.ones(points.shape[0])
if len(search_radius) != len(points):
raise ValueError('The search_radius is not valid!')
indices, radii_indices = search_indices(search_radius, ndim)
if method == 'max':
expression = mpl.measure_max(source,
points,
indices,
radii_indices,
sink=sink,
processes=processes,
verbose=verbose)
elif method == 'min':
expression = mpl.measure_min(source,
points,
indices,
radii_indices,
sink=sink,
processes=processes,
verbose=verbose)
elif method == 'mean':
expression = mpl.measure_mean(source,
points,
indices,
radii_indices,
sink=sink,
processes=processes,
verbose=verbose)
elif method == 'sum':
expression = mpl.measure_sum(source,
points,
indices,
radii_indices,
sink=sink,
processes=processes,
verbose=verbose)
else:
raise ValueError("Method %r not in 'max', 'min', 'mean'" % method)
if verbose:
timer.print_elapsed_time('Measuring expression done')
return expression
def measure_radius(source,
points,
fraction=None,
value=None,
max_radius=100,
method='sphere',
default=np.inf,
scale=None,
return_radii=True,
return_radii_as_scalar=True,
return_indices=False,
processes=None,
verbose=False):
"""Measures a radius via decay of intensity values for a list of points.
Arguments
---------
source : array
Source for measurement.
points : array
List of indices to measure radis for.
fraction : float or None
Fraction of center intensity that needs to be reached to detemrine the
radius.
If None, value needs to be given.
value : array or float or None:
The value below which the inensity has to fall to measure the radius
from the center pixel. If array, it has to be the same size as the
points. If None, fraction has to be given.
max_radius : int or tuple of ints
The maximal pixel radius to consider in each dimension. The larger the
slower the measurement.
default : number or None
Default value to use if no radius was detected.
scale: tuple or float
An optional scale in each direction to determine the distance.
return_radii : bool
If True, return the radii measured.
return_radii_as_scalar : bool
If True, returnt the radii as single floats, otherwise a radius for
each dimension.
return_indices : bool
If True, return the indices of the search which allows to idenitfy the
pixel at which the radius condition was met.
processes : int or None
Number of processes to use.
verbose : bool
If True, print progress info.
Returns
-------
radii : array
Array of measured radii if return_radii is True.
indices : array
Array of measured indices at which the radius detrection condition
is met.
"""
source = io.as_source(source).array
if verbose:
timer = tmr.Timer()
print('Measuring radii of %d points in array of shape %r.' %
(points.shape[0], source.shape))
ndim = source.ndim
if not hasattr(max_radius, '__len__'):
max_radius = [max_radius] * ndim
if len(max_radius) != ndim:
raise ValueError(' The maximal search radius %r has wronf dimension!' %
max_radius)
if method == 'sphere':
search = search_indices_sphere(max_radius)
elif method == 'rectangle':
search = search_indices_rectangle(max_radius)
else:
raise ValueError("The method is not 'sphere' or 'rectangle' but %r!" %
method)
if value is not None:
if hasattr(value, '__len__'):
measured = mpl.find_smaller_than_values(source,
points,
search,
value,
sink=None,
processes=processes,
verbose=verbose)
else:
measured = mpl.find_smaller_than_value(source,
points,
search,
value,
sink=None,
processes=processes,
verbose=verbose)
elif fraction is not None:
measured = mpl.find_smaller_than_fraction(source,
points,
search,
fraction,
sink=None,
processes=processes,
verbose=verbose)
else:
raise ValueError('fraction or value cannot both be None!')
if verbose:
timer.print_elapsed_time('Measuring radii done')
result = ()
if return_radii:
if scale is None:
scale = 1
if not hasattr(scale, '__len__'):
scale = [scale] * ndim
scale = np.asarray(scale)
radii = np.abs(search) * scale
if return_radii_as_scalar:
radii = np.sqrt(np.sum(radii * radii, axis=1))
radii = np.hstack([radii, default])
else:
radii = np.vstack([radii, [default] * ndim])
radii = radii[measured]
result += (radii, )
if return_indices:
search = np.vstack([search, [np.max(search, axis=0) + 1]])
indices = search[measured]
result += (indices, )
if len(result) == 1:
result = result[0]
return result
def clean_graph(graph, remove_self_loops = True, remove_isolated_vertices = True,
vertex_mappings = {'coordinates' : mean_vertex_coordinates,
'radii' : np.max},
verbose = False):
"""Remove all cliques to get pure branch structure of a graph.
Arguments
---------
graph : Graph
The graph to clean up.
verbose : bool
If True, prin progress information.
Returns
-------
graph : Graph
A graph removed of all cliques.
Note
----
cliques are replaced by a single vertex connecting to all non-clique neighbours
The center coordinate is used for that vertex as the coordinate.
"""
if verbose:
timer = tmr.Timer();
timer_all = tmr.Timer();
# find branch points
n_vertices = graph.n_vertices;
degrees = graph.vertex_degrees();
branches = degrees >= 3;
n_branch_points = branches.sum();
if verbose:
timer.print_elapsed_time('Graph cleaning: found %d branch points among %d vertices' % (n_branch_points, n_vertices));
timer.reset();
if n_branch_points == 0:
return graph.copy();
# detect 'cliques', i.e. connected components of branch points
gb = graph.sub_graph(vertex_filter=branches, view=True);
components, counts = gb.label_components(return_vertex_counts=True);
#note: graph_tools components of a view is a property map of the full graph
components[branches] += 1;
# group components
components = _group_labels(components);
#note: graph_tools components of a view is a property map of the full graph
#note: remove the indices of the non branch nodes
components = components[1:];
clique_ids = np.where(counts > 1)[0];
n_cliques = len(clique_ids);
if verbose:
timer.print_elapsed_time('Graph cleaning: detected %d cliques of branch points' % n_cliques);
timer.reset();
# remove cliques
g = graph.copy();
g.add_vertex(n_cliques);
#mappings
properties = {};
mappings = {};
for k in vertex_mappings.keys():
if k in graph.vertex_properties:
mappings[k] = vertex_mappings[k];
properties[k] = graph.vertex_property(k);
vertex_filter = np.ones(n_vertices + n_cliques, dtype = bool)
for i,ci in enumerate(clique_ids):
vi = n_vertices + i;
cc = components[ci];
#remove clique vertices
vertex_filter[cc] = False;
# get neighbours
neighbours = np.hstack([graph.vertex_neighbours(c) for c in cc])
neighbours = np.setdiff1d(np.unique(neighbours), cc);
# connect to new node
g.add_edge([[n, vi] for n in neighbours]);
#map properties
for k in mappings.keys():
g.set_vertex_property(k, mappings[k](properties[k][cc]), vertex=vi);
if verbose and i+1 % 10000 == 0:
timer.print_elapsed_time('Graph cleaning: reducing %d / %d' % (i+1, n_cliques));
#generate new graph
g = g.sub_graph(vertex_filter=vertex_filter);
if remove_self_loops:
g.remove_self_loops();
if verbose:
timer.print_elapsed_time('Graph cleaning: removed %d cliques of branch points from %d to %d nodes and %d to %d edges' % (n_cliques, graph.n_vertices, g.n_vertices, graph.n_edges, g.n_edges));
timer.reset();
# remove isolated vertices
if remove_isolated_vertices:
non_isolated = g.vertex_degrees() > 0;
g = g.sub_graph(vertex_filter = non_isolated)
if verbose:
timer.print_elapsed_time('Graph cleaning: Removed %d isolated nodes' % np.logical_not(non_isolated).sum());
timer.reset();
del non_isolated;
if verbose:
timer_all.print_elapsed_time('Graph cleaning: cleaned graph has %d nodes and %d edges' % (g.n_vertices, g.n_edges));
return g;
def save(filename,
data,
region=None,
blocks=None,
processes=cpu_count(),
verbose=False):
"""Save a large npy array to disk in parallel
Arguments:
filename : str
filename of array to load
data : array
array to save to disk
blocks : int or None
number of blocks to split array into for parallel processing
processes : None or int
number of processes, if None use number of cpus
verbose : bool
print info about the file to be loaded
Returns:
str
the filename of the numpy array on disk
"""
if processes is None:
processes = cpu_count()
if blocks is None:
blocks = processes * defaultBlocksPerProcess
if region is None:
#create file on disk via memmap
memmap = np.lib.format.open_memmap(filename,
mode='w+',
shape=data.shape,
dtype=data.dtype,
fortran_order=np.isfortran(data))
memmap.flush()
del (memmap)
#get specs from header specs
shape, dtype, order, offset = readNumpyHeader(filename)
if verbose:
timer = tmr.Timer()
print(
'Saving array of shape = %r, dtype = %r, order = %r, offset = %r' %
(shape, dtype, order, offset))
if (np.isfortran(data) and order != 'F') or (not np.isfortran(data)
and order != 'C'):
raise RuntimeError(
'Order of arrays do not match isfortran=%r and order=%s' %
(np.isfortran(data), order))
d = data.reshape(-1, order='A')
if dtype == bool:
d = d.view('uint8')
if region is not None:
sourceSlice = region.sourceSlice()
off = _offsetFromSlice(sourceSlice, order=order)
if order == 'F':
offset += data.strides[-1] * off
else:
offset += data.strides[1] * off
#print d.dtype, filename, offset, blocks, processes
filename = str(filename).encode('UTF-8')
code.save(data=d,
filename=filename,
offset=offset,
blocks=blocks,
processes=processes)
if verbose:
timer.print_elapsed_time(head='Saving array to %s' % filename)
return filename
def temp():
import scipy.ndimage as ndi
corr_ndi = ndi.correlate1d(data,
kernel,
axis=axis,
mode='constant',
cval=0)
assert np.allclose(corr.array, corr_ndi)
c = corr.array
c[0, :, 0]
corr_ndi[0, :, 0]
data = np.array(np.random.rand(1000, 1000, 500), order='F')
data = np.array(np.random.rand(300, 400, 1500), order='F')
kernel = np.array([1, 2, 3, 4, 5])
import ClearMap.Utils.Timer as tmr
timer = tmr.Timer()
for axis in range(3):
corr = ap.correlate1d(data,
kernel,
axis=axis,
verbose=False,
processes=None)
timer.print_elapsed_time('ap')
import ClearMap.Utils.Timer as tmr
timer = tmr.Timer()
for axis in range(3):
corr2 = ap2.correlate1d(data,
kernel,
axis=axis,
verbose=False,
processes=None)
timer.print_elapsed_time('ap')
import scipy.ndimage as ndi
timer = tmr.Timer()
for axis in range(3):
corr_ndi = ndi.correlate1d(data,
kernel,
axis=axis,
mode='constant',
cval=0)
timer.print_elapsed_time('ndi')
assert np.allclose(corr.array, corr_ndi)
assert np.allclose(corr2.array, corr_ndi)
# IO
import ClearMap.ParallelProcessing.DataProcessing.ArrayProcessing as ap
import numpy as np
reload(ap)
data = np.random.rand(10, 200, 10)
sink = ap.write('test.npy', data, verbose=True)
assert (np.all(sink.array == data))
read = ap.read('test.npy', verbose=True)
assert (np.all(read.array == data))
ap.io.delete_file('test.npy')
# where
reload(ap)
data = np.random.rand(30, 20, 40) > 0.5
where_np = np.array(np.where(data)).T
where = ap.where(data, cutoff=2**0)
check_np = np.zeros(data.shape, dtype=bool)
check = np.zeros(data.shape, dtype=bool)
check_np[tuple(where_np.T)] = True
check[tuple(where.array.T)] = True
assert (np.all(check_np == check))
def read(filename,
sink=None,
slicing=None,
as_shared=None,
blocks=None,
processes=None,
verbose=False,
**kwargs):
"""Read a large array into memory in parallel.
Arguments
---------
filename : str
The filename of array to load.
slicing : slice, tuple, or None
if not None this specifies the slice to read.
as_shared : bool
If True, read into shared memory
blocks : int or None
number of blocks to split array into for parallel processing
processes : None or int
number of processes, if None use number of cpus
verbose : bool
print info about the file to be loaded
Returns
-------
array : array
The data as an array in memory.
"""
if processes is None:
processes = default_processes
if blocks is None:
blocks = processes * default_blocks_per_process
#source info
source = mmp.Source(filename)
if slicing is not None:
source = slc.Slice(source=source, slicing=slicing)
shape, dtype, order, offset = source.shape, source.dtype, source.order, source.offset
if order not in ['C', 'F']:
raise NotImplementedError(
'Cannot read in parallel from non-contigous source!')
#TODO: implement parallel reader with strides !
if verbose:
timer = tmr.Timer()
print(
'Reading data from source of shape = %r, dtype = %r, order = %r, offset = %r'
% (shape, dtype, order, offset))
#use initialze form IO !!
#prepare outputs
if as_shared:
data = sma.empty(shape, dtype=dtype, order=order)
else:
data = np.empty(shape, dtype=dtype, order=order)
d = data.reshape(-1, order='A')
if dtype == bool:
d = d.view('uint8')
code.read(data=d,
filename=filename,
offset=offset,
blocks=blocks,
processes=processes)
if verbose:
timer.print_elapsed_time(head='Reading data from %s' % filename)
return data
def reduce_graph(graph, vertex_to_edge_mappings = {'radii' : np.max},
edge_to_edge_mappings = {'length' : np.sum},
edge_geometry = True, edge_length = None,
edge_geometry_vertex_properties = ['coordinates', 'radii'],
edge_geometry_edge_properties = None,
return_maps = False, verbose = False):
"""Reduce graph by replacing all vertices with degree two."""
if verbose:
timer = tmr.Timer();
timer_all = tmr.Timer();
print('Graph reduction: initialized.');
#ensure conditions for precessing step are fullfiled
if graph.is_view:
raise ValueError('Cannot process on graph view, prune graph before graph reduction.');
if not np.all(np.diff(graph.vertex_indices())==1) or int(graph.vertex_iterator().next()) != 0:
raise ValueError('Graph vertices not ordered!')
#copy graph
g = graph.copy();
#find non branching points, i.e. vertices with deg 2
branch_points = g.vertex_degrees() == 2;
non_branch_ids = np.where(branch_points)[0];
branch_points = np.logical_not(branch_points)
branch_ids = np.where(branch_points)[0];
n_non_branch_points = len(non_branch_ids);
n_branch_points = len(branch_ids);
del branch_points;
if verbose:
timer.print_elapsed_time('Graph reduction: Found %d branching and %d non-branching nodes' % (n_branch_points, n_non_branch_points));
timer.reset();
#mappings
if edge_to_edge_mappings is None:
edge_to_edge_mappings = {};
if vertex_to_edge_mappings is None:
vertex_to_edge_mappings = {};
if edge_length:
if 'length' not in edge_to_edge_mappings.keys():
edge_to_edge_mappings['length'] = np.sum;
if 'length' not in g.edge_properties:
g.add_edge_property('length', np.ones(g.n_edges, dtype = float));
edge_to_edge = {};
edge_properties = {};
edge_lists = {};
for k in edge_to_edge_mappings.keys():
if k in g.edge_properties:
edge_to_edge[k] = edge_to_edge_mappings[k];
edge_properties[k] = g.edge_property_map(k);
edge_lists[k] = [];
edge_to_edge_keys = edge_to_edge.keys();
vertex_to_edge = {};
vertex_properties = {};
vertex_lists = {};
for k in vertex_to_edge_mappings.keys():
if k in g.vertex_properties:
vertex_to_edge[k] = vertex_to_edge_mappings[k];
vertex_properties[k] = g.vertex_property_map(k);
vertex_lists[k] = [];
vertex_to_edge_keys = vertex_to_edge.keys();
#edge geometry
calculate_vertex_to_vertex_map = False;
calculate_edge_to_edge_map = False;
calculate_edge_to_vertex_map = False;
if edge_geometry:
calculate_edge_to_vertex_map = True;
reduced_edge_to_vertex_map = [];
if return_maps:
calculate_vertex_to_vertex_map = True;
reduced_vertex_to_vertex_map = [];
calculate_edge_to_vertex_map = True;
reduced_edge_to_vertex_map = [];
calculate_edge_to_edge_map = True;
reduced_edge_to_edge_map = [];
if edge_geometry_edge_properties is not None:
calculate_edge_to_edge_map = True;
reduced_edge_to_edge_map = [];
if edge_geometry_vertex_properties is None:
edge_geometry_vertex_properties = [];
if edge_geometry_edge_properties is None:
edge_geometry_edge_properties = [];
#direct edges between branch points
edge_list = [];
checked = np.zeros(g.n_edges, dtype=bool);
for i,v in enumerate(branch_ids):
for e in g.vertex_edges_iterator(v):
if e.target().out_degree() != 2:
eid = g.edge_index(e);
if not checked[eid]:
checked[eid] = True;
vertex_ids = [int(e.source()), int(e.target())];
edge_list.append(vertex_ids);
for k in edge_to_edge_keys:
edge_lists[k].append(edge_to_edge[k]([edge_properties[k][e]]));
for k in vertex_to_edge_keys:
vv = vertex_properties[k];
vv = [vv[e.source()], vv[e.target()]];
vertex_lists[k].append(vertex_to_edge[k](vv));
if calculate_edge_to_vertex_map:
reduced_edge_to_vertex_map.append(vertex_ids);
if calculate_edge_to_edge_map:
reduced_edge_to_edge_map.append([e]);
if verbose and (i+1) % 250000 == 0:
timer.print_elapsed_time('Graph reduction: Scanned %d/%d branching nodes, found %d branches' % (i+1, n_branch_points, len(edge_list)));
if verbose:
timer.print_elapsed_time('Graph reduction: Scanned %d/%d branching nodes, found %d branches' % (i +1, n_branch_points, len(edge_list)));
#reduce non-direct edges
checked = np.zeros(g.n_vertices, dtype=bool);
for i,v in enumerate(non_branch_ids):
if not checked[v]:
# extract branch
checked[v] = True;
vertex = g.vertex(v);
vertices, edges, endpoints, isolated_loop = _graph_branch(vertex);
#print([int(vv) for vv in vertices], [[int(ee.source()), int(ee.target())] for ee in edges], endpoints, isolated_loop)
if not isolated_loop:
vertices_ids = [int(vv) for vv in vertices];
checked[vertices_ids[1:-1]] = True;
edge_list.append([int(ep) for ep in endpoints]);
#mappings
for k in edge_to_edge.keys():
ep = edge_properties[k];
edge_lists[k].append(edge_to_edge[k]([ep[e] for e in edges]))
for k in vertex_to_edge.keys():
vp = vertex_properties[k];
vertex_lists[k].append(vertex_to_edge[k]([vp[vv] for vv in vertices]));
#if edge_geometry is not None:
# branch_start.append(index_pos);
# index_pos += len(vertices_ids);
# branch_end.append(index_pos);
if calculate_edge_to_vertex_map:
reduced_edge_to_vertex_map.append(vertices_ids);
if calculate_edge_to_edge_map:
reduced_edge_to_edge_map.append(edges);
if verbose and (i+1) % 250000 == 0:
timer.print_elapsed_time('Graph reduction: Scanned %d/%d non-branching nodes found %d branches' % (i+1, n_non_branch_points, len(edge_list)));
if verbose:
timer.print_elapsed_time('Graph reduction: Scanned %d/%d non-branching nodes found %d branches' % (i+1, n_non_branch_points, len(edge_list)));
#reduce graph
#redefine branch edges
gr = g.sub_graph(edge_filter=np.zeros(g.n_edges, dtype=bool));
gr.add_edge(edge_list);
#determine edge ordering
edge_order = gr.edge_indices();
# remove non-branching points
if calculate_vertex_to_vertex_map:
gr.add_vertex_property('_vertex_id_', graph.vertex_indices());
gr = gr.sub_graph(vertex_filter=np.logical_not(checked));
if calculate_vertex_to_vertex_map:
reduced_vertex_to_vertex_map = gr.vertex_property('_vertex_id_');
gr.remove_vertex_property('_vertex_id_');
# maps
if calculate_edge_to_vertex_map:
reduced_edge_to_vertex_map = np.array(reduced_edge_to_vertex_map, dtype=object)[edge_order];
if calculate_edge_to_edge_map:
reduced_edge_to_edge_map = np.array(reduced_edge_to_edge_map, dtype=object)[edge_order];
# add edge properties
for k in edge_to_edge_keys:
gr.define_edge_property(k, np.array(edge_lists[k])[edge_order]);
for k in vertex_to_edge_keys:
gr.define_edge_property(k, np.array(vertex_lists[k])[edge_order]);
if edge_geometry is not None:
branch_indices = np.hstack(reduced_edge_to_vertex_map);
indices = [0] + [len(m) for m in reduced_edge_to_vertex_map];
indices = np.cumsum(indices);
indices = np.array([indices[:-1], indices[1:]]).T;
#branch_start = np.array(branch_start);
#branch_end = np.array(branch_end);
#vertex properties
#indices = np.array([branch_start, branch_end]).T;
#indices = indices[edge_order];
indices_use = indices;
for p in edge_geometry_vertex_properties:
if p in g.vertex_properties:
values = g.vertex_property(p);
values = values[branch_indices];
gr.set_edge_geometry(name=p, values=values, indices=indices_use);
indices_use = None;
for p in edge_geometry_edge_properties:
if p in g.edge_properties:
values = g.edge_property_map(p);
#there is one edge less than vertices in each reduced edge !
values = [[values[e] for e in edge] + [values[edge[-1]]] for edge in reduced_edge_to_edge_map];
gr.set_edge_geometry(name='edge_' + p, values=values, indices=indices_use);
indices_use = None;
if calculate_edge_to_edge_map:
reduced_edge_to_edge_map = [[g.edge_index(e) for e in edge] for edge in reduced_edge_to_edge_map]
if verbose:
timer_all.print_elapsed_time('Graph reduction: Graph reduced from %d to %d nodes and %d to %d edges' % (graph.n_vertices, gr.n_vertices, graph.n_edges, gr.n_edges));
if return_maps:
return gr, (reduced_vertex_to_vertex_map, reduced_edge_to_vertex_map, reduced_edge_to_edge_map);
else:
return gr;
def process(function,
source,
sink=None,
axes=None,
size_max=None,
size_min=None,
overlap=None,
optimization=True,
optimization_fix='all',
neighbours=False,
function_type=None,
as_memory=False,
return_result=False,
return_blocks=False,
processes=None,
verbose=False,
**kwargs):
"""Create blocks and process a function on them in parallel.
Arguments
---------
function : function
The main data processing script.
source : str, Source, or list
The source or list of sources to apply a function to
sink : str, Source, list, or None
The sink or list of sinks to write the result to.
If None, return single array.
axes : int, list of ints, or None
Axes along which to split the source. If None, the
splitting is determined automaticlly from the order of the array.
size_max : int, list of ints or None
Maximal size of a block along the axes.
If None, :const:`default_size_max` is used.
size_min : int or list of ints
Minial size of a block along the axes.
If None, :const:`default_size_min` is used.
overlap : int, list of ints or None
Minimal overlap between blocks along the axes.
If None, :const:`default_overlap` is used.
optimization : bool or list of bools
If True, optimize block sizes to best fit number of processes.
optimization_fix : 'increase', 'decrease', 'all' or None or list
Increase, decrease or optimally change the block size when optimization
is active.
neighbours : bool
If True, also include information about the neighbourhood in the blocks.
function_type : 'array', 'source', 'block' or None
The function type passed. If None, 'array' is used.
* 'array'
Reading and writing the valid slices from the blocks is automatic
and the function gets passed numpy arrays.
* 'source'
Reading and writing the valid slices from the blocks is automatic
and the function gets passed Source classes as inputs.
* 'block'
The function is assumed to act on and update blocks itself.
as_memory : bool
If True, load full blocks into memory before applying the function.
Can be useful to reduce frequent reading and writing operations of memmaps.
return_result : bool
If True, return the results of the proceessing functions.
return_blocks : bool
If True, return the block information used to distribute the processing.
processes : int
The number of parallel processes, if 'serial', use serial processing.
verbose : bool
Print information on sub-stack generation.
Returns
-------
sink : str, Source, list or array
The results of the processing.
Note
----
This implementation only supports processing into sinks with the same shape as the source.
"""
#sources and sinks
if isinstance(source, list):
sources = source
else:
sources = [source]
sources = [io.as_source(s).as_virtual() for s in sources]
#if sink is None:
# sink = sma.Source(shape=sources[0].shape, dtype=sources[0].dtype, order=sources[0].order);
if isinstance(sink, list):
sinks = sink
elif sink is None:
sinks = []
else:
sinks = [sink]
sinks = [io.initialize(s, hint=sources[0]) for s in sinks]
sinks = [io.as_source(s).as_virtual() for s in sinks]
axes = block_axes(sources[0], axes=axes)
split = ft.partial(split_into_blocks,
processes=processes,
axes=axes,
size_max=size_max,
size_min=size_min,
overlap=overlap,
optimization=optimization,
optimization_fix=optimization_fix,
neighbours=neighbours,
verbose=False)
source_blocks = [split(s) for s in sources]
sink_blocks = [split(s) for s in sinks]
n_blocks = len(source_blocks[0])
source_blocks = [[blocks[i] for blocks in source_blocks]
for i in range(n_blocks)]
sink_blocks = [[blocks[i] for blocks in sink_blocks]
for i in range(n_blocks)]
if function_type is None:
function_type = 'array'
if function_type == 'block':
func = ft.partial(process_block_block,
function=function,
as_memory=as_memory,
return_result=return_result,
verbose=verbose,
**kwargs)
elif function_type == 'source':
func = ft.partial(process_block_source,
function=function,
as_memory=as_memory,
as_array=False,
verbose=verbose,
**kwargs)
elif function_type == 'array':
func = ft.partial(process_block_source,
function=function,
as_memory=as_memory,
as_array=True,
verbose=verbose,
**kwargs)
else:
raise ValueError(
"function type %r not 'array', 'source', 'block' or None!")
if not isinstance(processes, int) and processes != "serial":
processes = mp.cpu_count()
if verbose:
timer = tmr.Timer()
print("Processing %d blocks with function %r." %
(n_blocks, function.__name__))
if isinstance(processes, int):
#from bounded_pool_executor import BoundedProcessPoolExecutor
with cf.ProcessPoolExecutor(max_workers=processes) as executor:
#with BoundedProcessPoolExecutor(max_workers=processes) as executor:
futures = [
executor.submit(func, *args)
for args in zip(source_blocks, sink_blocks)
]
result = [f.result() for f in futures]
#executor.map(function, source_blocks, sink_blocks)
else:
result = [func(*args) for args in zip(source_blocks, sink_blocks)]
#analysis:ignore
if verbose:
timer.print_elapsed_time("Processed %d blocks with function %r" %
(n_blocks, function.__name__))
#gc.collect();
if return_result:
ret = result
else:
ret = sink
if return_blocks:
ret = (ret, [source_blocks, sink_blocks])
return ret
def _test():
import numpy as np
import ClearMap.ParallelProcessing.DataProcessing.ArrayProcessing as ap
## Lookup table processing
#apply_lut
x = np.random.randint(0, 100, size=(20, 30))
lut = np.arange(100) + 1
y = ap.apply_lut(x, lut)
assert np.all(y == x + 1)
#apply_lut_to_index
import ClearMap.ImageProcessing.Topology.Topology3d as t3d
kernel = t3d.index_kernel(dtype=int)
import ClearMap.ImageProcessing.Binary.Smoothing as sm
lut = sm.initialize_lookup_table()
data = np.array(np.random.rand(150, 30, 40) > 0.75, order='F')
result = ap.apply_lut_to_index(data, kernel, lut, sink=None, verbose=True)
import ClearMap.Visualization.Plot3d as p3d
p3d.plot([[data, result]])
### Correlation
#correlate1d
kernel = np.array(range(11), dtype='uint32')
data = np.array(np.random.randint(0,
2**27, (300, 400, 1500),
dtype='uint32'),
order='F')
#data = np.array(np.random.rand(3,4,5), order='F');
data = np.empty((300, 400, 1500), order='F')
kernel = np.array([1, 2, 3, 4, 5], dtype='uint8')
sink = 'test.npy'
import ClearMap.Utils.Timer as tmr
import scipy.ndimage as ndi
timer = tmr.Timer()
for axis in range(3):
print(axis)
corr_ndi = ndi.correlate1d(data, axis=axis, mode='constant', cval=0)
timer.print_elapsed_time('ndi')
timer = tmr.Timer()
for axis in range(3):
print(axis)
corr = ap.correlate1d(data,
sink=sink,
kernel=kernel,
axis=axis,
verbose=False,
processes=None)
timer.print_elapsed_time('ap')
assert np.allclose(corr.array, corr_ndi)
# IO
import ClearMap.ParallelProcessing.DataProcessing.ArrayProcessing as ap
import numpy as np
reload(ap)
data = np.random.rand(10, 200, 10)
sink = ap.write('test.npy', data, verbose=True)
assert (np.all(sink.array == data))
read = ap.read('test.npy', verbose=True)
assert (np.all(read.array == data))
ap.io.delete_file('test.npy')
# where
reload(ap)
data = np.random.rand(30, 20, 40) > 0.5
where_np = np.array(np.where(data)).T
where = ap.where(data, cutoff=2**0)
check_np = np.zeros(data.shape, dtype=bool)
check = np.zeros(data.shape, dtype=bool)
check_np[tuple(where_np.T)] = True
check[tuple(where.array.T)] = True
assert (np.all(check_np == check))
def graph_from_skeleton(skeleton, points = None, radii = None, vertex_coordinates = True,
check_border = True, delete_border = False, verbose = False):
"""Converts a binary skeleton image to a graph-tool graph.
Arguments
---------
skeleton : array
Source with 2d/3d binary skeleton.
points : array
List of skeleton points as 1d indices of flat skeleton array (optional to save processing time).
radii : array
List of radii associated with each vertex.
vertex_coordinates : bool
If True, store coordiantes of the vertices / edges.
check_border : bool
If True, check if the boder is empty. The algorithm reuqires this.
delete_border : bool
If True, delete the border.
verbose : bool
If True, print progress information.
Returns
-------
graph : Graph class
The graph corresponding to the skeleton.
Note
----
Edges are detected between neighbouring foreground pixels using 26-connectivty.
"""
skeleton = io.as_source(skeleton);
if delete_border:
skeleton = t3d.delete_border(skeleton);
check_border = False;
if check_border:
if not t3d.check_border(skeleton):
raise ValueError('The skeleton array needs to have no points on the border!');
if verbose:
timer = tmr.Timer();
timer_all = tmr.Timer();
print('Graph from skeleton calculation initialize.!');
if points is None:
points = ap.where(skeleton.reshape(-1, order = 'A')).array;
if verbose:
timer.print_elapsed_time('Point list generation');
timer.reset();
#create graph
n_vertices = points.shape[0];
g = ggt.Graph(n_vertices=n_vertices, directed=False);
g.shape = skeleton.shape;
if verbose:
timer.print_elapsed_time('Graph initialized with %d vertices' % n_vertices);
timer.reset();
#detect edges
edges_all = np.zeros((0,2), dtype = int);
for i,o in enumerate(t3d.orientations()):
# calculate off set
offset = np.sum((np.hstack(np.where(o))-[1,1,1]) * skeleton.strides)
edges = ap.neighbours(points, offset);
if len(edges) > 0:
edges_all = np.vstack([edges_all, edges]);
if verbose:
timer.print_elapsed_time('%d edges with orientation %d/13 found' % (edges.shape[0], i+1));
timer.reset();
if edges_all.shape[0] > 0:
g.add_edge(edges_all);
if verbose:
timer.print_elapsed_time('Added %d edges to graph' % (edges_all.shape[0]));
timer.reset();
if vertex_coordinates:
vertex_coordinates = np.array(np.unravel_index(points, skeleton.shape, order=skeleton.order)).T;
g.set_vertex_coordinates(vertex_coordinates);
if radii is not None:
g.set_vertex_radius(radii);
if verbose:
timer_all.print_elapsed_time('Skeleton to Graph');
return g;
def load(filename,
region=None,
shared=False,
blocks=None,
processes=cpu_count(),
verbose=False):
"""Load a large npy array into memory in parallel
Arguments:
filename : str
filename of array to load
region : Region or None
if not None this specifies the sub-region to read
shared : bool
if True read into shared memory
blocks : int or None
number of blocks to split array into for parallel processing
processes : None or int
number of processes, if None use number of cpus
verbose : bool
print info about the file to be loaded
Returns:
array
the data as numpy array
"""
if processes is None:
processes = cpu_count()
if blocks is None:
blocks = processes * defaultBlocksPerProcess
#get specs from header specs
shape, dtype, order, offset = readNumpyHeader(filename)
if verbose:
timer = tmr.Timer()
print(
'Loading array of shape = %r, dtype = %r, order = %r, offset = %r'
% (shape, dtype, order, offset))
if region is not None:
shape = region.shape()
sourceSlice = region.sourceSlice()
off = _offsetFromSlice(sourceSlice, order=order)
if shared:
data = shm.create(shape, dtype=dtype, order=order)
else:
data = np.empty(shape, dtype=dtype, order=order)
d = data.reshape(-1, order='A')
if dtype == bool:
d = d.view('uint8')
if region is not None:
if order == 'F':
offset += data.strides[-1] * off
else:
offset += data.strides[1] * off
filename = str(filename).encode('UTF-8')
code.load(data=d,
filename=filename,
offset=offset,
blocks=blocks,
processes=processes)
if verbose:
timer.print_elapsed_time(head='Loading array from %s' % filename)
return data
def detect_cells_block(source, parameter=default_cell_detection_parameter):
"""Detect cells in a Block."""
#initialize parameter and slicings
verbose = parameter.get('verbose', False)
if verbose:
prefix = 'Block %s: ' % (source.info(), )
total_time = tmr.Timer(prefix)
base_slicing = source.valid.base_slicing
valid_slicing = source.valid.slicing
valid_lower = source.valid.lower
valid_upper = source.valid.upper
lower = source.lower
parameter_intensity = parameter.get('intensity_detection', None)
measure_to_array = dict()
if parameter_intensity:
parameter_intensity = parameter_intensity.copy()
measure = parameter_intensity.pop('measure', [])
if measure is None:
measure = []
for m in measure:
measure_to_array[m] = None
if 'source' in measure_to_array:
measure_to_array['source'] = source
# correct illumination
parameter_illumination = parameter.get('illumination_correction', None)
if parameter_illumination:
parameter_illumination = parameter_illumination.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_illumination,
head=prefix + 'Illumination correction')
save = parameter_illumination.pop('save', None)
corrected = ic.correct_illumination(source, **parameter_illumination)
if save:
save = io.as_source(save)
save[base_slicing] = corrected[valid_slicing]
if verbose:
timer.print_elapsed_time('Illumination correction')
else:
corrected = np.array(source.array)
if 'illumination' in measure_to_array:
measure_to_array['illumination'] = corrected
#background subtraction
parameter_background = parameter.get('background_correction', None)
if parameter_background:
parameter_background = parameter_background.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_background,
head=prefix + 'Background removal')
save = parameter_background.pop('save', None)
background = remove_background(corrected, **parameter_background)
if save:
save = io.as_source(save)
save[base_slicing] = corrected[valid_slicing]
if verbose:
timer.print_elapsed_time('Illumination correction')
else:
background = corrected
del corrected
if 'background' in measure_to_array:
measure_to_array['background'] = background
# equalize
parameter_equalize = parameter.get('equalization', None)
if parameter_equalize:
parameter_equalize = parameter_equalize.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_equalize, head=prefix + 'Equalization:')
save = parameter_equalize.pop('save', None)
equalized = equalize(background, mask=None, **parameter_equalize)
if save:
save = io.as_source(save)
save[base_slicing] = equalized[valid_slicing]
if verbose:
timer.print_elapsed_time('Equalization')
else:
equalized = background
del background
if 'equalized' in measure_to_array:
measure_to_array['equalized'] = equalized
#DoG filter
parameter_dog_filter = parameter.get('dog_filter', None)
if parameter_dog_filter:
parameter_dog_filter = parameter_dog_filter.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_dog_filter, head=prefix + 'DoG filter:')
save = parameter_dog_filter.pop('save', None)
dog = dog_filter(equalized, **parameter_dog_filter)
if save:
save = io.as_source(save)
save[base_slicing] = dog[valid_slicing]
if verbose:
timer.print_elapsed_time('DoG filter')
else:
dog = equalized
del equalized
if 'dog' in measure_to_array:
measure_to_array['dog'] = dog
#Maxima detection
parameter_maxima = parameter.get('maxima_detection', None)
parameter_shape = parameter.get('shape_detection', None)
if parameter_shape or parameter_intensity:
if not parameter_maxima:
print(
prefix +
'Warning: maxima detection needed for shape and intensity detection!'
)
parameter_maxima = dict()
if parameter_maxima:
parameter_maxima = parameter_maxima.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_maxima, head=prefix + 'Maxima detection:')
save = parameter_maxima.pop('save', None)
valid = parameter_maxima.pop('valid', None)
# extended maxima
maxima = md.find_maxima(source.array,
**parameter_maxima,
verbose=verbose)
if save:
save = io.as_source(save)
save[base_slicing] = maxima[valid_slicing]
#center of maxima
if parameter_maxima['h_max']:
centers = md.find_center_of_maxima(source,
maxima=maxima,
verbose=verbose)
else:
centers = ap.where(maxima).array
if verbose:
timer.print_elapsed_time('Maxima detection')
#correct for valid region
if valid:
ids = np.ones(len(centers), dtype=bool)
for c, l, u in zip(centers.T, valid_lower, valid_upper):
ids = np.logical_and(ids, np.logical_and(l <= c, c < u))
centers = centers[ids]
del ids
del dog, maxima
results = (centers, )
#cell shape detection
if parameter_shape:
parameter_shape = parameter_shape.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_shape, head=prefix + 'Shape detection:')
save = parameter_shape.pop('save', None)
# shape detection
shape = sd.detect_shape(source,
centers,
**parameter_shape,
verbose=verbose)
if save:
save = io.as_source(save)
save[base_slicing] = shape[valid_slicing]
#size detection
max_label = centers.shape[0]
sizes = sd.find_size(shape, max_label=max_label)
valid = sizes > 0
if verbose:
timer.print_elapsed_time('Shape detection')
results += (sizes, )
else:
valid = None
shape = None
#cell intensity detection
if parameter_intensity:
parameter_intensity = parameter_intensity.copy()
if verbose:
timer = tmr.Timer(prefix)
hdict.pprint(parameter_intensity,
head=prefix + 'Intensity detection:')
if not shape is None:
r = parameter_intensity.pop('shape', 3)
if isinstance(r, tuple):
r = r[0]
for m in measure:
if shape is not None:
intensity = sd.find_intensity(measure_to_array[m],
label=shape,
max_label=max_label,
**parameter_intensity)
else:
intensity = me.measure_expression(measure_to_array[m],
centers,
search_radius=r,
**parameter_intensity,
processes=1,
verbose=False)
results += (intensity, )
if verbose:
timer.print_elapsed_time('Shape detection')
if valid is not None:
results = tuple(r[valid] for r in results)
#correct coordinate offsets of blocks
results = (results[0] + lower, ) + results[1:]
#correct shapes for merging
results = tuple(r[:, None] if r.ndim == 1 else r for r in results)
if verbose:
total_time.print_elapsed_time('Cell detection')
gc.collect()
return results
def write(filename,
data,
slicing=None,
blocks=None,
processes=None,
verbose=False):
"""Write a large array to disk in parallel.
Arguments
---------
filename : str
Filename of array to load.
data : array
Array to save to disk.
blocks : int or None
Number of blocks to split array into for parallel processing.
processes : None or int
Number of processes, if None use number of cpus.
verbose : bool
Print info about the file to be loaded.
Returns
-------
filename : str
The filename of the numpy array on disk.
"""
if processes is None:
processes = mp.cpu_count()
if blocks is None:
blocks = processes * default_blocks_per_process
#data
data = io.as_source(data)
#prepare sink
is_file = fu.is_file(filename)
if slicing is not None and not is_file:
raise ValueError('Cannot write to a slice to a non-existing file %s!' %
filename)
if slicing is None:
#create file on disk via memmap
fortran_order = 'F' == data.order
memmap = np.lib.format.open_memmap(filename,
mode='w+',
shape=data.shape,
dtype=data.dtype,
fortran_order=fortran_order)
memmap.flush()
del (memmap)
sink = mmp.Source(location=filename)
if slicing is not None:
sink = slc.Slice(source=sink, slicing=slicing)
shape, dtype, order, offset = sink.shape, sink.dtype, sink.order, sink.offset
if (data.order != order):
raise RuntimeError('Order of arrays do not match %r!=%r' %
(data.order, order))
if order not in ['C', 'F']:
raise NotImplementedError(
'Cannot read in parallel from non-contigous source!')
#TODO: implement parallel reader with strides !
if verbose:
timer = tmr.Timer()
print(
'Writing data to sink of shape = %r, dtype = %r, order = %r, offset = %r'
% (shape, dtype, order, offset))
d = data.reshape(-1, order='A')
if d.dtype == bool:
d = d.view('uint8')
code.write(data=d,
filename=filename,
offset=offset,
blocks=blocks,
processes=processes)
if verbose:
timer.print_elapsed_time(head='Writing data to %s' % filename)
return filename
