def _initialize_processing(processes = None, verbose = False, function = None): """Initialize parallel array processing. Arguments --------- processes : int, 'seial' or None The number of processes to use. If None use number of cpus. verbose : bool If True, print progress information. function : str or None The nae of the function. Returns ------- processes : int The number of processes. timer : Timer A timer for the processing. """ if processes is None: processes = default_processes; if processes == 'serial': processes = 1; if verbose: if function: print('%s: initialized!' % function); timer = tmr.Timer(); else: timer = None; return processes, timer
def find_size(label, max_label=None, verbose=False): """Find size given object shapes as a labled image Arguments --------- label : array, str or Source Labeled image in which each object has its own label. max_label : int or None Maximal label to include, if None use all label. verbose : bool Print progress info. Returns ------- sizes : array Measured intensities """ if verbose: timer = tmr.Timer() hdict.pprint(head='Size detection:', max_label=max_label) label = io.as_source(label) if max_label is None: max_label = int(label.max()) sizes = scipy.ndimage.measurements.sum(np.ones(label.shape, dtype=bool), labels=label, index=np.arange(1, max_label + 1)) if verbose: timer.print_elapsed_time(head='Size detection') return sizes
def grey_reconstruct(source, mask=None, sink=None, method=None, shape=3, verbose=False): """Calculates the grey reconstruction of the image Arguments --------- source : array The source image data. method : 'dilation' or 'erosion' or None The mehtjod to use, if None return original image. shape : in or tuple Shape of the strucuturing element for the grey reconstruction. verbose : boo; If True, print progress info. Returns ------- reconstructed: array Grey reconstructed image. Note ---- The reconstruction is done slice by slice along the z-axis. """ if verbose: timer = tmr.Timer() hdict.pprint(head='Grey reconstruction', method=method, shape=shape) if method is None: return source if sink is None: sink = np.empty(source.shape, dtype=source.dtype) # background subtraction in each slice selem = se.structure_element(form='Disk', shape=shape, ndim=2).astype('uint8') for z in range(source.shape[2]): #img[:,:,z] = img[:,:,z] - grey_opening(img[:,:,z], structure = structureElement('Disk', (30,30))); #img[:,:,z] = img[:,:,z] - morph.grey_opening(img[:,:,z], structure = self.structureELement('Disk', (150,150))); sink[:, :, z] = source[:, :, z] - reconstruct( source[:, :, z], mask=mask[:, :, z], method=method, selem=selem) if verbose: timer.print_elapsed_time('Grey reconstruction') return sink
def _test(): import numpy as np import ClearMap.Utils.Timer as tmr import ClearMap.ImageProcessing.Clipping.Clipping as clp data = np.random.rand(1000,1000,2000); for p in [1, 10, None]: print('Clipping: processes = %r' % p); timer = tmr.Timer(); clipped = clp.clip(data, clip_max = 0.5, processes = p); timer.print_elapsed_time('Clipping');
def find_intensity(source, label, max_label=None, method='sum', verbose=False): """Find integrated intensity given object shapes as labled image. Arguments --------- source : array, str, or Source Source to measure intensities from. label : array, str, or Source Labeled image with a separate label for each object. max_label : int or None Maximal label to include. If None use all. method : {'sum', 'mean', 'max', 'min'} Method to use to measure the intensities in each object's area. verbose : bool If True, print progress information. Returns ------- intensities : array Measured intensities. """ if verbose: timer = tmr.Timer() hdict.pprint(head='Intensity detection:', max_label=max_label, method=method) source = io.as_source(source).array label = io.as_source(label) if max_label is None: max_label = label.max() if method.lower() == 'sum': measure = scipy.ndimage.measurements.sum elif method.lower() == 'mean': measure = scipy.ndimage.measurements.mean elif method.lower() == 'max': measure = scipy.ndimage.measurements.maximum elif method.lower() == 'min': measure = scipy.ndimage.measurements.minimum else: raise RuntimeError('Unkown method %r!' % (method, )) intensities = measure(label, labels=label, index=np.arange(1, max_label + 1)) if verbose: timer.print_elapsed_time(head='Intensity detection') return intensities
def find_maxima(source, h_max=None, shape=5, threshold=None, verbose=None): """Find local and extended maxima in an image. Arguments --------- source : array The source data. h_max : float or None H parameter for the initial h-Max transform. If None, do not perform a h-max transform. shape : int or tuple Shape for the structure element for the local maxima filter. threshold : float or None If float, include only maxima larger than this threshold. verbose : bool Print progress info. Returns ------- maxima : array Binary image with True pixel at extended maxima. Notes ----- This routine performs a h-max transfrom, followed by a local maxima search and thresholding of the maxima. See also -------- :func:`h_max_transform`, :func:`local_max` """ if verbose: timer = tmr.Timer() hdict.pprint(head='Find Maxima:', h_max=h_max, shape=shape, threshold=threshold) # extended maxima maxima = h_max_transform(source, h_max=h_max) #local maxima maxima = local_max(maxima, shape=shape) #thresholding if not threshold is None: maxima = np.logical_and(maxima, source >= threshold) if verbose: timer.print_elapsed_time(head='Find Maxima') return maxima
def convert_files(filenames, extension = None, path = None, processes = None, verbose = False): """Transforms list of files to their sink format in parallel. Arguments --------- filenames : list of str The filenames to convert extension : str The new file format extension. path : str or None Optional path speicfication. processes : int, 'serial' or None The number of processes to use for parallel conversion. verbose : bool If True, print progress information. Returns ------- filenames : list of str The new file names. """ if not isinstance(filenames, (tuple, list)): filenames = [filenames]; if len(filenames) == 0: return []; n_files = len(filenames); if path is not None: filenames = [fu.join(path, fu.split(f)[1]) for f in filenames]; sinks = ['.'.join(f.split('.')[:-1] + [extension]) for f in filenames]; if verbose: timer = tmr.Timer() print('Converting %d files to %s!' % (n_files, extension)); if not isinstance(processes, int) and processes != 'serial': processes = mp.cpu_count(); #print(n_files, extension, filenames, sinks) _convert = functools.partial(_convert_files, n_files=n_files, extension=extension, verbose=verbose); if processes == 'serial': [_convert(source,sink,i) for i,source,sink in zip(range(n_files), filenames, sinks)]; else: with concurrent.futures.ProcessPoolExecutor(processes) as executor: executor.map(_convert, filenames, sinks, range(n_files)); if verbose: timer.print_elapsed_time('Converting %d files to %s' % (n_files, extension)); return sinks;
def rank(source, sink=None, function=rnk.median, resample=None, verbose=False, out=sys.stdout, **kwargs): """Rank filter inbetween reshaping.""" timer = tmr.Timer() sink, sink_buffer = ap.initialize_sink(sink=sink, source=source, order='F') if resample: interpolation = cv2.INTER_NEAREST new_shape = np.round(np.array(sink.shape, dtype=float) * resample).astype(int) new_shape[2] = sink.shape[2] data = np.zeros(tuple(new_shape), order='F', dtype=source.dtype) new_shape = tuple(new_shape[1::-1]) for z in range(source.shape[2]): data[:, :, z] = cv2.resize(src=source[:, :, z], dsize=new_shape, interpolation=interpolation) #print data.shape, data.dtype out.write(timer.elapsed_time(head='Rank filter: Resampling') + '\n') else: data = source #keys = inspect.getargspec(function).args; #kwargs = { k : v for k,v in kwargs.iteritems() if k in keys}; data = function(data, **kwargs) out.write( timer.elapsed_time(head='Rank filter: %s' % function.__name__) + '\n') if resample: #interpolation = cv2.INTER_LINEAR; interpolation = cv2.INTER_AREA for z in range(sink.shape[2]): sink_buffer[:, :, z] = cv2.resize(src=data[:, :, z], dsize=sink.shape[1::-1], interpolation=interpolation) out.write(timer.elapsed_time(head='Rank filter: Upsampling') + '\n') else: sink_buffer[:] = data return sink
def process_block_source(sources, sinks, function, as_memory=False, as_array=False, verbose=False, **kwargs): """Process a block with full traceback. Arguments --------- sources : source specifications Sources passed to the function. sinks : sourcespecifications Sinks where data is written to. function func : function The function to call. """ if verbose: timer = tmr.Timer() print('Processing block %s' % (sources[0].info(), )) #sources = [s.as_real() for s in sources]; sources_input = sources if as_memory: sources = [s.as_memory() for s in sources] if as_array: sources = [s.array for s in sources] results = function(*sources, **kwargs) if not isinstance(results, (list, tuple)): results = [results] if len(sources_input) != len(sinks): sources_input = sources_input + [sources_input[0] ] * (len(sinks) - len(sources)) for sink, source, result in zip(sinks, sources_input, results): #sink = sink.as_real(); sink.valid[:] = result[source.valid.slicing] if verbose: timer.print_elapsed_time('Processing block %s' % (sources_input[0].info(), )) gc.collect() return None
def detect_shape(source, seeds, threshold=None, verbose=False): """Detect object shapes by generatng a labeled image from seeds. Arguments --------- source : array, str or Source Source image. seeds : array, str or Source Cell centers as point coordinates. threshold : float or None Threshold to determine mask for watershed, pixel below this are treated as background. If None, the seeds are expanded indefinately. verbose :bool If True, print progress info. Returns ------- shapes : array Labeled image, where each label indicates a object. """ if verbose: timer = tmr.Timer() hdict.pprint(head='Shape detection', threshold=threshold) source = io.as_source(source).array seeds = io.as_source(seeds) if threshold is None: mask = None else: mask = source > threshold peaks = vox.voxelize(seeds, shape=source.shape, weights=np.arange(1, seeds.shape[0] + 1)).array shapes = skimage.morphology.watershed(-source, peaks, mask=mask) #shapes = watershed_ift(-source.astype('uint16'), peaks); #shapes[numpy.logical_not(mask)] = 0; if verbose: timer.print_elapsed_time('Shape detection') return shapes
def initialize_processing(processes=None, verbose=False, function=None, blocks=None, return_blocks=False): """Initialize parallel array processing. Arguments --------- processes : int, 'seial' or None The number of processes to use. If None use number of cpus. verbose : bool If True, print progress information. function : str or None The nae of the function. Returns ------- processes : int The number of processes. timer : Timer A timer for the processing. """ if processes is None: processes = default_processes if processes == 'serial': processes = 1 if verbose: if function: print('%s: initialized!' % function) timer = tmr.Timer() else: timer = None results = (processes, timer) if return_blocks: if blocks is None: blocks = processes * default_blocks_per_process results += (blocks, ) return results
def process_block_block(sources, sinks, function, as_memory=False, return_result=False, verbose=False, **kwargs): """Process a block with full traceback. Arguments --------- sources : source specifications Sources passed to the function. sinks : sourcespecifications Sinks where data is written to. function func : function The function to call. """ if verbose: timer = tmr.Timer() print('Processing block %s' % (sources[0].info(), )) if as_memory: sinks = sinks sinks_memory = [s.as_memory_block() for s in sinks] sources_and_sinks = [s.as_memory_block() for s in sources] + sinks_memory else: sources_and_sinks = sources + sinks result = function(*sources_and_sinks, **kwargs) if as_memory: for sink, sink_memory in zip(sinks, sinks_memory): sink.valid[:] = sink_memory.valid[:] if verbose: timer.print_elapsed_time('Processing block %s' % (sources[0].info(), )) gc.collect() if return_result: return result else: return None
def find_center_of_maxima(source, maxima=None, label=None, verbose=False): """Find center of detected maxima weighted by intensity Arguments --------- source : array Intensity image data. maxima : array or None Binary array indicating the maxima. I label is not None this can be None. label : array or None Labeled image of the shapes of the maxima. If None, determined from maxima. verbose : bool Print progress info. Returns ------- coordinates : array Coordinates of the n centers of maxima as (n,d)-array. """ if verbose: timer = tmr.Timer() print('Center of Maxima initialized!') #center of maxima if label is None: label, n_label = ndm.label(maxima) else: n_label = label.max() if n_label > 0: centers = np.array( ndm.center_of_mass(source, label, index=np.arange(1, n_label))) else: centers = np.zeros((0, source.ndim)) if verbose: timer.print_elapsed_time('Center of Maxima: %d maxima detected' % centers.shape[0]) return centers
def resample(source, sink = None, orientation = None, sink_shape = None, source_resolution = None, sink_resolution = None, interpolation = 'linear', axes_order = None, method = 'shared', processes = None, verbose = True): """Resample data of source in new shape/resolution and orientation. Arguments --------- source : str or array The source to be resampled. sink : str or None The sink for the resampled image. orientation : tuple or None: The orientation specified by permuation and change in sign of (1,2,3). sink_shape : tuple or None The target shape of the resampled sink. source_resolution : tuple or None The resolution of the source (in length per pixel). sink_resolution : tuple or None The resolution of the resampled source (in length per pixel). interpolation : str The method to use for interpolating to the resmapled array. axis_order : str, list of tuples of int or None The axes pairs along which to resample the data at each step. If None, this is detertmined automatically. For a FileList source, setting the first tuple should point to axis not indicating files. If 'size' the axis order is determined automatically to maximally reduce the size of the array in each resmapling step. If 'order' the axis order is chosed automatically to optimize io speed. method : 'shared' or 'memmap' Method to handle intermediate resampling results. If 'shared' use shared memory, otherwise use a memory map on disk. processes : int, None or 'serial' Number of processes to use for parallel resampling, if None use maximal processes avaialable, if 'serial' process in serial. verbose : bool If True, display progress information. Returns ------- sink : array or str The data or filename of resampled sink. Notes ----- * Resolutions are assumed to be given for the axes of the intrinsic orientation of the data and reference (as when viewed by ImageJ). * Orientation: permuation of 1,2,3 with potential sign, indicating which axes map onto the reference axes, a negative sign indicates reversal of that particular axes. * Only a minimal set of information to determine the resampling parameter has to be given, e.g. source_shape and sink_shape. * The resampling is done by iterating two dimensional resampling steps. """ #TODO: write full nd resampling routine extending cv2 lib. if verbose: timer = tmr.Timer(); source = io.as_source(source); source_shape = source.shape; ndim = len(source_shape); dtype = source.dtype; order = source.order; orientation = format_orientation(orientation); source_shape, sink_shape, source_resolution, sink_resolution = \ resample_shape(source_shape=source_shape, sink_shape=sink_shape, source_resolution=source_resolution, sink_resolution=sink_resolution, orientation=orientation); sink_shape_in_source_orientation = orient_shape(sink_shape, orientation, inverse=True); interpolation = _interpolation_to_cv2(interpolation); if not isinstance(processes, int) and processes != 'serial': processes = io.mp.cpu_count(); #detemine order of resampling axes_order, shape_order = _axes_order(axes_order, source, sink_shape_in_source_orientation, order=order); #print(axes_order, shape_order) if len(axes_order) == 0: if verbose: print('resampling: no resampling necessary, source has same size as sink!'); if sink != source: return io.write(sink, source); else: return source; #resample n_steps = len(axes_order); last_source = source; delete_files = []; for step, axes, shape in zip(range(n_steps), axes_order, shape_order): if step == n_steps-1 and orientation is None: resampled = io.initialize(source=sink, shape=sink_shape, dtype=dtype, as_source=True); else: if method == 'shared': resampled = io.sma.create(shape, dtype=dtype, order=order, as_source=True); else: location = tempfile.mktemp() + '.npy'; resampled = io.mmp.create(location, shape=shape, dtype=dtype, order=order, as_source=True); delete_files.append(location); #print(resampled) #indices for non-resampled axes indices = tuple([range(s) for d,s in enumerate(shape) if d not in axes]); indices = [i for i in itertools.product(*indices)]; n_indices = len(indices); #resample step last_source_virtual = last_source.as_virtual(); resampled_virtual = resampled.as_virtual(); _resample = ft.partial(_resample_2d, source=last_source_virtual, sink=resampled_virtual, axes=axes, shape=shape, interpolation=interpolation, n_indices=n_indices, verbose=verbose) if processes == 'serial': for index in indices: _resample(index=index); else: #print(processes); with concurrent.futures.ProcessPoolExecutor(processes) as executor: executor.map(_resample, indices); last_source = resampled; #fix orientation if not orientation is None: #permute per = orientation_to_permuation(orientation); resampled = resampled.transpose(per); #reverse axes reslice = False; slicing = [slice(None)] * ndim; for d,o in enumerate(orientation): if o < 0: slicing[d] = slice(None, None, -1); reslice = True; if reslice: resampled = resampled[slicing]; if verbose: print("resample: re-oriented shape %r!" % (resampled.shape,)) sink = io.write(sink, resampled); else: sink = resampled; for f in delete_files: io.delete_file(f); if verbose: timer.print_elapsed_time('Resampling') return sink;
def smooth_by_configuration(source, sink = None, iterations = 1, processing_parameter = None, processes = None, verbose = False): """Smooth a binary source using the local configuration around each pixel. Arguments --------- source : array or Source The binary source to smooth. sink : array, Source or None The sink to write result of smoothing. If None, return array. iterations : int Number of smoothing iterations. processing_parameter : None or dict The parameter passed to :func:`ClearMap.ParallelProcessing.BlockProcessing.process`. processes : int or None number of processes to use. verbose : bool If True, print progress information. Returns ------- smoothed : array or Source Thre smoothed binary array. Note ---- The algorithm is based on a topological smoothing operation defined by adding or removing forground pixels based on the local topology of the binary array. """ if verbose: print('Binary smoothing: initialized!'); timer = tmr.Timer(); #smoothing function smooth = functools.partial(smooth_by_configuration_block, iterations=iterations, verbose=False); smooth.__name__ = 'smooth_by_configuration' #initialize sources and sinks source = io.as_source(source); sink = io.initialize(sink, shape=source.shape, dtype=bool, order=source.order); #block processing parameter block_processing_parameter = dict(axes = bp.block_axes(source), as_memory=True, overlap=None, function_type='source', processes=processes, verbose=verbose); if processing_parameter is not None: block_processing_parameter.update(processing_parameter); if not 'overlap' in block_processing_parameter or block_processing_parameter['overlap'] is None: block_processing_parameter['overlap'] = 2 + 2 * iterations; if not 'size_min' in block_processing_parameter or block_processing_parameter['size_min'] is None: block_processing_parameter['size_min'] = 2 + 2 * iterations + 1; if not 'axes' in block_processing_parameter or block_processing_parameter['axes'] is None: block_processing_parameter['axes'] = bp.block_axes(source); #print(block_processing_parameter) #block process bp.process(smooth, source, sink, **block_processing_parameter); if verbose: timer.print_elapsed_time('Binary smoothing: done'); return sink;
def postprocess(source, sink = None, postprocessing_parameter = default_postprocessing_parameter, processing_parameter = default_postprocessing_processing_parameter, processes = None, verbose = True): """Postprocess a binarized image. Arguments --------- source : source specification The binary source. sink : sink specification or None The sink to write the postprocesses result to. If None, an array is returned. postprocessing_parameter : dict Parameter for the postprocessing. processing_parameter : dict Parameter for the parallel processing. verbose : bool If True, print progress output. Returns ------- sink : Source The result of the binarization. Notes ----- * The postporcessing pipeline is composed of several steps. The parameters for each step are passed as sub-dictionaries to the postprocessing_parameter dictionary. * If None is passed for one of the steps the step is skipped. Smoothing --------- smooth : dict or None Smoothing step parameter. See :func:`ClearMap.ImageProcessing.Binary.Smoothing.smooth_by_configuration` iterations : int Number of smoothing iterations. For the vasculature a typical value is 6. Filling ------- fill : bool or None If True, fill holes in the binary data. """ source = io.as_source(source); sink = ap.initialize_sink(sink, shape=source.shape, dtype=source.dtype, order=source.order, return_buffer=False); if verbose: timer = tmr.Timer(); print('Binary post processing: initialized.'); postprocessing_parameter = postprocessing_parameter.copy(); parameter_smooth = postprocessing_parameter.pop('smooth', None); parameter_fill = postprocessing_parameter.pop('fill', None); #print(parameter_smooth, parameter_fill) #smoothing save = None; if parameter_smooth: #intialize temporary files if needed if parameter_fill: save = parameter_smooth.pop('save', None); temporary_filename = save; if temporary_filename is None: temporary_filename = postprocessing_parameter['temporary_filename']; if temporary_filename is None: temporary_filename = tmpf.mktemp(prefix='TubeMap_Vasculature_postprocessing', suffix='.npy'); sink_smooth = ap.initialize_sink(temporary_filename, shape=source.shape, dtype=source.dtype, order=source.order, return_buffer=False); else: sink_smooth = sink; #run smoothing source_fill = bs.smooth_by_configuration(source, sink=sink_smooth, processing_parameter=processing_parameter, processes=processes, verbose=verbose, **parameter_smooth); else: source_fill = source; if parameter_fill: sink = bf.fill(source_fill, sink=sink, processes=processes, verbose=verbose); if parameter_smooth and save is None: io.delete_file(temporary_filename); else: sink = source_fill; if verbose: timer.print_elapsed_time('Binary post processing'); gc.collect() return None;
def binarize_block(source, sink, parameter = default_binarization_parameter): """Binarize a Block.""" #initialize parameter and slicings verbose = parameter.get('verbose', False); if verbose: prefix = 'Block %s: ' % (source.info(),); total_time = tmr.Timer(prefix); max_bin = parameter.get('max_bin', MAX_BIN); base_slicing = sink.valid.base_slicing; valid_slicing = source.valid.slicing; #initialize binary status for inspection binary_status = parameter.get('binary_status', None); if binary_status: binary_status = io.as_source(binary_status); binary_status = binary_status[base_slicing]; #clipping parameter_clip = parameter.get('clip', None); if parameter_clip: parameter_clip = parameter_clip.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_clip, head = prefix + 'Clipping:') parameter_clip.update(norm=max_bin, dtype=DTYPE); save = parameter_clip.pop('save', None); clipped, mask, high, low = clip(source, **parameter_clip); not_low = np.logical_not(low); if save: save = io.as_source(save); save[base_slicing] = clipped[valid_slicing]; if binary_status is not None: binary_status[high[valid_slicing]] += BINARY_STATUS['High'] else: sink[valid_slicing] = high[valid_slicing]; del high, low if verbose: timer.print_elapsed_time('Clipping'); else: clipped = source mask = not_low = np.ones(source.shape, dtype=bool); #high = low = np.zeros(source.shape, dtype=bool); #low = np.zeros(source.shape, dtype=bool); #not_low = np.logical_not(low); #active arrays: clipped, mask, not_low #lightsheet correction parameter_lightsheet = parameter.get('lightsheet', None); if parameter_lightsheet: parameter_lightsheet = parameter_lightsheet.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_lightsheet, head = prefix + 'Lightsheet:') #parameter_lightsheet.update(max_bin=max_bin); save = parameter_lightsheet.pop('save', None); corrected = lc.correct_lightsheet(clipped, mask=mask, max_bin=max_bin, **parameter_lightsheet); if save: save = io.as_source(save); save[base_slicing] = corrected[valid_slicing]; if verbose: timer.print_elapsed_time('Lightsheet'); else: corrected = clipped; del clipped #active arrays: corrected, mask, not_low #median filter parameter_median = parameter.get('median', None); if parameter_median: parameter_median = parameter_median.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_median, head = prefix + 'Median:') save = parameter_median.pop('save', None); median = rnk.median(corrected, max_bin=max_bin, mask=not_low, **parameter_median); if save: save = io.as_source(save); save[base_slicing] = median[valid_slicing]; if verbose: timer.print_elapsed_time('Median'); else: median = corrected; del corrected, not_low; #active arrays: median, mask #pseudo deconvolution parameter_deconvolution = parameter.get('deconvolve', None); if parameter_deconvolution: parameter_deconvolution = parameter_deconvolution.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_deconvolution, head = prefix + 'Deconvolution:') save = parameter_deconvolution.pop('save', None); threshold = parameter_deconvolution.pop('threshold', None); if binary_status is not None: binarized = binary_status > 0; else: binarized = sink[:]; deconvolved = deconvolve(median, binarized[:], **parameter_deconvolution) del binarized if save: save = io.as_source(save); save[base_slicing] = deconvolved[valid_slicing]; if verbose: timer.print_elapsed_time('Deconvolution'); if threshold: binary_deconvolved = deconvolved > threshold; if binary_status is not None: binary_status[binary_deconvolved[valid_slicing]] += BINARY_STATUS['Deconvolved']; else: sink[valid_slicing] += binary_deconvolved[valid_slicing]; del binary_deconvolved if verbose: timer.print_elapsed_time('Deconvolution: binarization'); else: deconvolved = median; #active arrays: median, mask, deconvolved #adaptive parameter_adaptive = parameter.get('adaptive', None); if parameter_adaptive: parameter_adaptive = parameter_adaptive.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_adaptive, head = prefix + 'Adaptive:') save = parameter_adaptive.pop('save', None); adaptive = threshold_adaptive(deconvolved, **parameter_adaptive) if save: save = io.as_source(save); save[base_slicing] = adaptive[valid_slicing]; binary_adaptive = deconvolved > adaptive; if binary_status is not None: binary_status[binary_adaptive[valid_slicing]] += BINARY_STATUS['Adaptive']; else: sink[valid_slicing] += binary_adaptive[valid_slicing]; del binary_adaptive, adaptive; if verbose: timer.print_elapsed_time('Adaptive'); del deconvolved #active arrays: median, mask # equalize parameter_equalize = parameter.get('equalize', None); if parameter_equalize: parameter_equalize = parameter_equalize.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_equalize, head = prefix + 'Equalization:') save = parameter_equalize.pop('save', None); threshold = parameter_equalize.pop('threshold', None); equalized = equalize(median, mask=mask, **parameter_equalize); if save: save = io.as_source(save); save[base_slicing] = equalized[valid_slicing]; if verbose: timer.print_elapsed_time('Equalization'); if threshold: binary_equalized = equalized > threshold if binary_status is not None: binary_status[binary_equalized[valid_slicing]] += BINARY_STATUS['Equalized']; else: sink[valid_slicing] += binary_equalized[valid_slicing]; #prepare equalized for use in vesselization parameter_vesselization = parameter.get('vesselize', None); if parameter_vesselization and parameter_vesselization.get('background', None): equalized[binary_equalized] = threshold; equalized = float(max_bin-1) / threshold * equalized; del binary_equalized if verbose: timer.print_elapsed_time('Equalization: binarization'); else: equalized = median; del median #active arrays: mask, equalized # smaller vessels /capilarries parameter_vesselization = parameter.get('vesselize', None); if parameter_vesselization: parameter_vesselization = parameter_vesselization.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_vesselization, head = prefix + 'Vesselization:') parameter_background = parameter_vesselization.get('background', None) parameter_background = parameter_background.copy(); if parameter_background: save = parameter_background.pop('save', None); equalized = np.array(equalized, dtype = 'uint16'); background = rnk.percentile(equalized, max_bin=max_bin, mask=mask, **parameter_background); tubeness = equalized - np.minimum(equalized, background); del background if save: save = io.as_source(save); save[base_slicing] = tubeness[valid_slicing]; else: tubeness = equalized; parameter_tubeness = parameter_vesselization.get('tubeness', {}) tubeness = tubify(tubeness, **parameter_tubeness); save = parameter_vesselization.get('save', None); if save: save = io.as_source(save); save[base_slicing] = tubeness[valid_slicing]; if verbose: timer.print_elapsed_time('Vesselization'); threshold = parameter_vesselization.get('threshold', None); if threshold: binary_vesselized = tubeness > threshold; if binary_status is not None: binary_status[binary_vesselized[valid_slicing]] += BINARY_STATUS['Tube']; else: sink[valid_slicing] += binary_vesselized[valid_slicing]; del binary_vesselized if verbose: timer.print_elapsed_time('Vesselization: binarization'); del tubeness del equalized, mask #active arrays: None #fill holes parameter_fill = parameter.get('fill', None); if parameter_fill: parameter_fill = parameter_fill.copy(); if verbose: timer = tmr.Timer(prefix); #hdict.pprint(parameter_fill, head = 'Filling:') if binary_status is not None: foreground = binary_status > 0; filled = ndi.morphology.binary_fill_holes(foreground); binary_status[np.logical_and(filled, np.logical_not(foreground))] += BINARY_STATUS['Fill']; del foreground, filled else: filled = ndi.morphology.binary_fill_holes(sink[:]); sink[valid_slicing] += filled[valid_slicing]; del filled if verbose: timer.print_elapsed_time('Filling'); if binary_status is not None: sink[valid_slicing] = binary_status[valid_slicing] > 0; #smooth binary parameter_smooth = parameter.get('smooth', None); if parameter_smooth: parameter_smooth = parameter_smooth.copy(); if verbose: timer = tmr.Timer(prefix); hdict.pprint(parameter_smooth, head = prefix + 'Smoothing:') smoothed = bs.smooth_by_configuration(sink, sink=None, processes=1, **parameter_smooth); sink[valid_slicing] = smoothed[valid_slicing]; del smoothed; if verbose: timer.print_elapsed_time('Smoothing'); if verbose: total_time.print_elapsed_time('Binarization') gc.collect() return None;
def measure_expression(source, points, search_radius, method='max', sink=None, processes=None, verbose=False): """Measures the expression around a list of points in a source. Arguments --------- source : array Source for measurement. points : array List of indices to measure radis for. search_radius : int or array List of search radii to use around each point. If int use this radius for all points. Array should be of length of points. method : 'max' or 'min', 'mean' Measurement type. processes : int or None Number of processes to use. verbose : bool If True, print progress info. """ source = io.as_source(source) ndim = source.ndim if verbose: timer = tmr.Timer() print('Measuring expression of %d points in array of shape %r.' % (points.shape[0], source.shape)) if not hasattr(search_radius, '__len__'): search_radius = search_radius * np.ones(points.shape[0]) if len(search_radius) != len(points): raise ValueError('The search_radius is not valid!') indices, radii_indices = search_indices(search_radius, ndim) if method == 'max': expression = mpl.measure_max(source, points, indices, radii_indices, sink=sink, processes=processes, verbose=verbose) elif method == 'min': expression = mpl.measure_min(source, points, indices, radii_indices, sink=sink, processes=processes, verbose=verbose) elif method == 'mean': expression = mpl.measure_mean(source, points, indices, radii_indices, sink=sink, processes=processes, verbose=verbose) elif method == 'sum': expression = mpl.measure_sum(source, points, indices, radii_indices, sink=sink, processes=processes, verbose=verbose) else: raise ValueError("Method %r not in 'max', 'min', 'mean'" % method) if verbose: timer.print_elapsed_time('Measuring expression done') return expression
def measure_radius(source, points, fraction=None, value=None, max_radius=100, method='sphere', default=np.inf, scale=None, return_radii=True, return_radii_as_scalar=True, return_indices=False, processes=None, verbose=False): """Measures a radius via decay of intensity values for a list of points. Arguments --------- source : array Source for measurement. points : array List of indices to measure radis for. fraction : float or None Fraction of center intensity that needs to be reached to detemrine the radius. If None, value needs to be given. value : array or float or None: The value below which the inensity has to fall to measure the radius from the center pixel. If array, it has to be the same size as the points. If None, fraction has to be given. max_radius : int or tuple of ints The maximal pixel radius to consider in each dimension. The larger the slower the measurement. default : number or None Default value to use if no radius was detected. scale: tuple or float An optional scale in each direction to determine the distance. return_radii : bool If True, return the radii measured. return_radii_as_scalar : bool If True, returnt the radii as single floats, otherwise a radius for each dimension. return_indices : bool If True, return the indices of the search which allows to idenitfy the pixel at which the radius condition was met. processes : int or None Number of processes to use. verbose : bool If True, print progress info. Returns ------- radii : array Array of measured radii if return_radii is True. indices : array Array of measured indices at which the radius detrection condition is met. """ source = io.as_source(source).array if verbose: timer = tmr.Timer() print('Measuring radii of %d points in array of shape %r.' % (points.shape[0], source.shape)) ndim = source.ndim if not hasattr(max_radius, '__len__'): max_radius = [max_radius] * ndim if len(max_radius) != ndim: raise ValueError(' The maximal search radius %r has wronf dimension!' % max_radius) if method == 'sphere': search = search_indices_sphere(max_radius) elif method == 'rectangle': search = search_indices_rectangle(max_radius) else: raise ValueError("The method is not 'sphere' or 'rectangle' but %r!" % method) if value is not None: if hasattr(value, '__len__'): measured = mpl.find_smaller_than_values(source, points, search, value, sink=None, processes=processes, verbose=verbose) else: measured = mpl.find_smaller_than_value(source, points, search, value, sink=None, processes=processes, verbose=verbose) elif fraction is not None: measured = mpl.find_smaller_than_fraction(source, points, search, fraction, sink=None, processes=processes, verbose=verbose) else: raise ValueError('fraction or value cannot both be None!') if verbose: timer.print_elapsed_time('Measuring radii done') result = () if return_radii: if scale is None: scale = 1 if not hasattr(scale, '__len__'): scale = [scale] * ndim scale = np.asarray(scale) radii = np.abs(search) * scale if return_radii_as_scalar: radii = np.sqrt(np.sum(radii * radii, axis=1)) radii = np.hstack([radii, default]) else: radii = np.vstack([radii, [default] * ndim]) radii = radii[measured] result += (radii, ) if return_indices: search = np.vstack([search, [np.max(search, axis=0) + 1]]) indices = search[measured] result += (indices, ) if len(result) == 1: result = result[0] return result
def clean_graph(graph, remove_self_loops = True, remove_isolated_vertices = True, vertex_mappings = {'coordinates' : mean_vertex_coordinates, 'radii' : np.max}, verbose = False): """Remove all cliques to get pure branch structure of a graph. Arguments --------- graph : Graph The graph to clean up. verbose : bool If True, prin progress information. Returns ------- graph : Graph A graph removed of all cliques. Note ---- cliques are replaced by a single vertex connecting to all non-clique neighbours The center coordinate is used for that vertex as the coordinate. """ if verbose: timer = tmr.Timer(); timer_all = tmr.Timer(); # find branch points n_vertices = graph.n_vertices; degrees = graph.vertex_degrees(); branches = degrees >= 3; n_branch_points = branches.sum(); if verbose: timer.print_elapsed_time('Graph cleaning: found %d branch points among %d vertices' % (n_branch_points, n_vertices)); timer.reset(); if n_branch_points == 0: return graph.copy(); # detect 'cliques', i.e. connected components of branch points gb = graph.sub_graph(vertex_filter=branches, view=True); components, counts = gb.label_components(return_vertex_counts=True); #note: graph_tools components of a view is a property map of the full graph components[branches] += 1; # group components components = _group_labels(components); #note: graph_tools components of a view is a property map of the full graph #note: remove the indices of the non branch nodes components = components[1:]; clique_ids = np.where(counts > 1)[0]; n_cliques = len(clique_ids); if verbose: timer.print_elapsed_time('Graph cleaning: detected %d cliques of branch points' % n_cliques); timer.reset(); # remove cliques g = graph.copy(); g.add_vertex(n_cliques); #mappings properties = {}; mappings = {}; for k in vertex_mappings.keys(): if k in graph.vertex_properties: mappings[k] = vertex_mappings[k]; properties[k] = graph.vertex_property(k); vertex_filter = np.ones(n_vertices + n_cliques, dtype = bool) for i,ci in enumerate(clique_ids): vi = n_vertices + i; cc = components[ci]; #remove clique vertices vertex_filter[cc] = False; # get neighbours neighbours = np.hstack([graph.vertex_neighbours(c) for c in cc]) neighbours = np.setdiff1d(np.unique(neighbours), cc); # connect to new node g.add_edge([[n, vi] for n in neighbours]); #map properties for k in mappings.keys(): g.set_vertex_property(k, mappings[k](properties[k][cc]), vertex=vi); if verbose and i+1 % 10000 == 0: timer.print_elapsed_time('Graph cleaning: reducing %d / %d' % (i+1, n_cliques)); #generate new graph g = g.sub_graph(vertex_filter=vertex_filter); if remove_self_loops: g.remove_self_loops(); if verbose: timer.print_elapsed_time('Graph cleaning: removed %d cliques of branch points from %d to %d nodes and %d to %d edges' % (n_cliques, graph.n_vertices, g.n_vertices, graph.n_edges, g.n_edges)); timer.reset(); # remove isolated vertices if remove_isolated_vertices: non_isolated = g.vertex_degrees() > 0; g = g.sub_graph(vertex_filter = non_isolated) if verbose: timer.print_elapsed_time('Graph cleaning: Removed %d isolated nodes' % np.logical_not(non_isolated).sum()); timer.reset(); del non_isolated; if verbose: timer_all.print_elapsed_time('Graph cleaning: cleaned graph has %d nodes and %d edges' % (g.n_vertices, g.n_edges)); return g;
def save(filename, data, region=None, blocks=None, processes=cpu_count(), verbose=False): """Save a large npy array to disk in parallel Arguments: filename : str filename of array to load data : array array to save to disk blocks : int or None number of blocks to split array into for parallel processing processes : None or int number of processes, if None use number of cpus verbose : bool print info about the file to be loaded Returns: str the filename of the numpy array on disk """ if processes is None: processes = cpu_count() if blocks is None: blocks = processes * defaultBlocksPerProcess if region is None: #create file on disk via memmap memmap = np.lib.format.open_memmap(filename, mode='w+', shape=data.shape, dtype=data.dtype, fortran_order=np.isfortran(data)) memmap.flush() del (memmap) #get specs from header specs shape, dtype, order, offset = readNumpyHeader(filename) if verbose: timer = tmr.Timer() print( 'Saving array of shape = %r, dtype = %r, order = %r, offset = %r' % (shape, dtype, order, offset)) if (np.isfortran(data) and order != 'F') or (not np.isfortran(data) and order != 'C'): raise RuntimeError( 'Order of arrays do not match isfortran=%r and order=%s' % (np.isfortran(data), order)) d = data.reshape(-1, order='A') if dtype == bool: d = d.view('uint8') if region is not None: sourceSlice = region.sourceSlice() off = _offsetFromSlice(sourceSlice, order=order) if order == 'F': offset += data.strides[-1] * off else: offset += data.strides[1] * off #print d.dtype, filename, offset, blocks, processes filename = str(filename).encode('UTF-8') code.save(data=d, filename=filename, offset=offset, blocks=blocks, processes=processes) if verbose: timer.print_elapsed_time(head='Saving array to %s' % filename) return filename
def temp(): import scipy.ndimage as ndi corr_ndi = ndi.correlate1d(data, kernel, axis=axis, mode='constant', cval=0) assert np.allclose(corr.array, corr_ndi) c = corr.array c[0, :, 0] corr_ndi[0, :, 0] data = np.array(np.random.rand(1000, 1000, 500), order='F') data = np.array(np.random.rand(300, 400, 1500), order='F') kernel = np.array([1, 2, 3, 4, 5]) import ClearMap.Utils.Timer as tmr timer = tmr.Timer() for axis in range(3): corr = ap.correlate1d(data, kernel, axis=axis, verbose=False, processes=None) timer.print_elapsed_time('ap') import ClearMap.Utils.Timer as tmr timer = tmr.Timer() for axis in range(3): corr2 = ap2.correlate1d(data, kernel, axis=axis, verbose=False, processes=None) timer.print_elapsed_time('ap') import scipy.ndimage as ndi timer = tmr.Timer() for axis in range(3): corr_ndi = ndi.correlate1d(data, kernel, axis=axis, mode='constant', cval=0) timer.print_elapsed_time('ndi') assert np.allclose(corr.array, corr_ndi) assert np.allclose(corr2.array, corr_ndi) # IO import ClearMap.ParallelProcessing.DataProcessing.ArrayProcessing as ap import numpy as np reload(ap) data = np.random.rand(10, 200, 10) sink = ap.write('test.npy', data, verbose=True) assert (np.all(sink.array == data)) read = ap.read('test.npy', verbose=True) assert (np.all(read.array == data)) ap.io.delete_file('test.npy') # where reload(ap) data = np.random.rand(30, 20, 40) > 0.5 where_np = np.array(np.where(data)).T where = ap.where(data, cutoff=2**0) check_np = np.zeros(data.shape, dtype=bool) check = np.zeros(data.shape, dtype=bool) check_np[tuple(where_np.T)] = True check[tuple(where.array.T)] = True assert (np.all(check_np == check))
def read(filename, sink=None, slicing=None, as_shared=None, blocks=None, processes=None, verbose=False, **kwargs): """Read a large array into memory in parallel. Arguments --------- filename : str The filename of array to load. slicing : slice, tuple, or None if not None this specifies the slice to read. as_shared : bool If True, read into shared memory blocks : int or None number of blocks to split array into for parallel processing processes : None or int number of processes, if None use number of cpus verbose : bool print info about the file to be loaded Returns ------- array : array The data as an array in memory. """ if processes is None: processes = default_processes if blocks is None: blocks = processes * default_blocks_per_process #source info source = mmp.Source(filename) if slicing is not None: source = slc.Slice(source=source, slicing=slicing) shape, dtype, order, offset = source.shape, source.dtype, source.order, source.offset if order not in ['C', 'F']: raise NotImplementedError( 'Cannot read in parallel from non-contigous source!') #TODO: implement parallel reader with strides ! if verbose: timer = tmr.Timer() print( 'Reading data from source of shape = %r, dtype = %r, order = %r, offset = %r' % (shape, dtype, order, offset)) #use initialze form IO !! #prepare outputs if as_shared: data = sma.empty(shape, dtype=dtype, order=order) else: data = np.empty(shape, dtype=dtype, order=order) d = data.reshape(-1, order='A') if dtype == bool: d = d.view('uint8') code.read(data=d, filename=filename, offset=offset, blocks=blocks, processes=processes) if verbose: timer.print_elapsed_time(head='Reading data from %s' % filename) return data
def reduce_graph(graph, vertex_to_edge_mappings = {'radii' : np.max}, edge_to_edge_mappings = {'length' : np.sum}, edge_geometry = True, edge_length = None, edge_geometry_vertex_properties = ['coordinates', 'radii'], edge_geometry_edge_properties = None, return_maps = False, verbose = False): """Reduce graph by replacing all vertices with degree two.""" if verbose: timer = tmr.Timer(); timer_all = tmr.Timer(); print('Graph reduction: initialized.'); #ensure conditions for precessing step are fullfiled if graph.is_view: raise ValueError('Cannot process on graph view, prune graph before graph reduction.'); if not np.all(np.diff(graph.vertex_indices())==1) or int(graph.vertex_iterator().next()) != 0: raise ValueError('Graph vertices not ordered!') #copy graph g = graph.copy(); #find non branching points, i.e. vertices with deg 2 branch_points = g.vertex_degrees() == 2; non_branch_ids = np.where(branch_points)[0]; branch_points = np.logical_not(branch_points) branch_ids = np.where(branch_points)[0]; n_non_branch_points = len(non_branch_ids); n_branch_points = len(branch_ids); del branch_points; if verbose: timer.print_elapsed_time('Graph reduction: Found %d branching and %d non-branching nodes' % (n_branch_points, n_non_branch_points)); timer.reset(); #mappings if edge_to_edge_mappings is None: edge_to_edge_mappings = {}; if vertex_to_edge_mappings is None: vertex_to_edge_mappings = {}; if edge_length: if 'length' not in edge_to_edge_mappings.keys(): edge_to_edge_mappings['length'] = np.sum; if 'length' not in g.edge_properties: g.add_edge_property('length', np.ones(g.n_edges, dtype = float)); edge_to_edge = {}; edge_properties = {}; edge_lists = {}; for k in edge_to_edge_mappings.keys(): if k in g.edge_properties: edge_to_edge[k] = edge_to_edge_mappings[k]; edge_properties[k] = g.edge_property_map(k); edge_lists[k] = []; edge_to_edge_keys = edge_to_edge.keys(); vertex_to_edge = {}; vertex_properties = {}; vertex_lists = {}; for k in vertex_to_edge_mappings.keys(): if k in g.vertex_properties: vertex_to_edge[k] = vertex_to_edge_mappings[k]; vertex_properties[k] = g.vertex_property_map(k); vertex_lists[k] = []; vertex_to_edge_keys = vertex_to_edge.keys(); #edge geometry calculate_vertex_to_vertex_map = False; calculate_edge_to_edge_map = False; calculate_edge_to_vertex_map = False; if edge_geometry: calculate_edge_to_vertex_map = True; reduced_edge_to_vertex_map = []; if return_maps: calculate_vertex_to_vertex_map = True; reduced_vertex_to_vertex_map = []; calculate_edge_to_vertex_map = True; reduced_edge_to_vertex_map = []; calculate_edge_to_edge_map = True; reduced_edge_to_edge_map = []; if edge_geometry_edge_properties is not None: calculate_edge_to_edge_map = True; reduced_edge_to_edge_map = []; if edge_geometry_vertex_properties is None: edge_geometry_vertex_properties = []; if edge_geometry_edge_properties is None: edge_geometry_edge_properties = []; #direct edges between branch points edge_list = []; checked = np.zeros(g.n_edges, dtype=bool); for i,v in enumerate(branch_ids): for e in g.vertex_edges_iterator(v): if e.target().out_degree() != 2: eid = g.edge_index(e); if not checked[eid]: checked[eid] = True; vertex_ids = [int(e.source()), int(e.target())]; edge_list.append(vertex_ids); for k in edge_to_edge_keys: edge_lists[k].append(edge_to_edge[k]([edge_properties[k][e]])); for k in vertex_to_edge_keys: vv = vertex_properties[k]; vv = [vv[e.source()], vv[e.target()]]; vertex_lists[k].append(vertex_to_edge[k](vv)); if calculate_edge_to_vertex_map: reduced_edge_to_vertex_map.append(vertex_ids); if calculate_edge_to_edge_map: reduced_edge_to_edge_map.append([e]); if verbose and (i+1) % 250000 == 0: timer.print_elapsed_time('Graph reduction: Scanned %d/%d branching nodes, found %d branches' % (i+1, n_branch_points, len(edge_list))); if verbose: timer.print_elapsed_time('Graph reduction: Scanned %d/%d branching nodes, found %d branches' % (i +1, n_branch_points, len(edge_list))); #reduce non-direct edges checked = np.zeros(g.n_vertices, dtype=bool); for i,v in enumerate(non_branch_ids): if not checked[v]: # extract branch checked[v] = True; vertex = g.vertex(v); vertices, edges, endpoints, isolated_loop = _graph_branch(vertex); #print([int(vv) for vv in vertices], [[int(ee.source()), int(ee.target())] for ee in edges], endpoints, isolated_loop) if not isolated_loop: vertices_ids = [int(vv) for vv in vertices]; checked[vertices_ids[1:-1]] = True; edge_list.append([int(ep) for ep in endpoints]); #mappings for k in edge_to_edge.keys(): ep = edge_properties[k]; edge_lists[k].append(edge_to_edge[k]([ep[e] for e in edges])) for k in vertex_to_edge.keys(): vp = vertex_properties[k]; vertex_lists[k].append(vertex_to_edge[k]([vp[vv] for vv in vertices])); #if edge_geometry is not None: # branch_start.append(index_pos); # index_pos += len(vertices_ids); # branch_end.append(index_pos); if calculate_edge_to_vertex_map: reduced_edge_to_vertex_map.append(vertices_ids); if calculate_edge_to_edge_map: reduced_edge_to_edge_map.append(edges); if verbose and (i+1) % 250000 == 0: timer.print_elapsed_time('Graph reduction: Scanned %d/%d non-branching nodes found %d branches' % (i+1, n_non_branch_points, len(edge_list))); if verbose: timer.print_elapsed_time('Graph reduction: Scanned %d/%d non-branching nodes found %d branches' % (i+1, n_non_branch_points, len(edge_list))); #reduce graph #redefine branch edges gr = g.sub_graph(edge_filter=np.zeros(g.n_edges, dtype=bool)); gr.add_edge(edge_list); #determine edge ordering edge_order = gr.edge_indices(); # remove non-branching points if calculate_vertex_to_vertex_map: gr.add_vertex_property('_vertex_id_', graph.vertex_indices()); gr = gr.sub_graph(vertex_filter=np.logical_not(checked)); if calculate_vertex_to_vertex_map: reduced_vertex_to_vertex_map = gr.vertex_property('_vertex_id_'); gr.remove_vertex_property('_vertex_id_'); # maps if calculate_edge_to_vertex_map: reduced_edge_to_vertex_map = np.array(reduced_edge_to_vertex_map, dtype=object)[edge_order]; if calculate_edge_to_edge_map: reduced_edge_to_edge_map = np.array(reduced_edge_to_edge_map, dtype=object)[edge_order]; # add edge properties for k in edge_to_edge_keys: gr.define_edge_property(k, np.array(edge_lists[k])[edge_order]); for k in vertex_to_edge_keys: gr.define_edge_property(k, np.array(vertex_lists[k])[edge_order]); if edge_geometry is not None: branch_indices = np.hstack(reduced_edge_to_vertex_map); indices = [0] + [len(m) for m in reduced_edge_to_vertex_map]; indices = np.cumsum(indices); indices = np.array([indices[:-1], indices[1:]]).T; #branch_start = np.array(branch_start); #branch_end = np.array(branch_end); #vertex properties #indices = np.array([branch_start, branch_end]).T; #indices = indices[edge_order]; indices_use = indices; for p in edge_geometry_vertex_properties: if p in g.vertex_properties: values = g.vertex_property(p); values = values[branch_indices]; gr.set_edge_geometry(name=p, values=values, indices=indices_use); indices_use = None; for p in edge_geometry_edge_properties: if p in g.edge_properties: values = g.edge_property_map(p); #there is one edge less than vertices in each reduced edge ! values = [[values[e] for e in edge] + [values[edge[-1]]] for edge in reduced_edge_to_edge_map]; gr.set_edge_geometry(name='edge_' + p, values=values, indices=indices_use); indices_use = None; if calculate_edge_to_edge_map: reduced_edge_to_edge_map = [[g.edge_index(e) for e in edge] for edge in reduced_edge_to_edge_map] if verbose: timer_all.print_elapsed_time('Graph reduction: Graph reduced from %d to %d nodes and %d to %d edges' % (graph.n_vertices, gr.n_vertices, graph.n_edges, gr.n_edges)); if return_maps: return gr, (reduced_vertex_to_vertex_map, reduced_edge_to_vertex_map, reduced_edge_to_edge_map); else: return gr;
def process(function, source, sink=None, axes=None, size_max=None, size_min=None, overlap=None, optimization=True, optimization_fix='all', neighbours=False, function_type=None, as_memory=False, return_result=False, return_blocks=False, processes=None, verbose=False, **kwargs): """Create blocks and process a function on them in parallel. Arguments --------- function : function The main data processing script. source : str, Source, or list The source or list of sources to apply a function to sink : str, Source, list, or None The sink or list of sinks to write the result to. If None, return single array. axes : int, list of ints, or None Axes along which to split the source. If None, the splitting is determined automaticlly from the order of the array. size_max : int, list of ints or None Maximal size of a block along the axes. If None, :const:`default_size_max` is used. size_min : int or list of ints Minial size of a block along the axes. If None, :const:`default_size_min` is used. overlap : int, list of ints or None Minimal overlap between blocks along the axes. If None, :const:`default_overlap` is used. optimization : bool or list of bools If True, optimize block sizes to best fit number of processes. optimization_fix : 'increase', 'decrease', 'all' or None or list Increase, decrease or optimally change the block size when optimization is active. neighbours : bool If True, also include information about the neighbourhood in the blocks. function_type : 'array', 'source', 'block' or None The function type passed. If None, 'array' is used. * 'array' Reading and writing the valid slices from the blocks is automatic and the function gets passed numpy arrays. * 'source' Reading and writing the valid slices from the blocks is automatic and the function gets passed Source classes as inputs. * 'block' The function is assumed to act on and update blocks itself. as_memory : bool If True, load full blocks into memory before applying the function. Can be useful to reduce frequent reading and writing operations of memmaps. return_result : bool If True, return the results of the proceessing functions. return_blocks : bool If True, return the block information used to distribute the processing. processes : int The number of parallel processes, if 'serial', use serial processing. verbose : bool Print information on sub-stack generation. Returns ------- sink : str, Source, list or array The results of the processing. Note ---- This implementation only supports processing into sinks with the same shape as the source. """ #sources and sinks if isinstance(source, list): sources = source else: sources = [source] sources = [io.as_source(s).as_virtual() for s in sources] #if sink is None: # sink = sma.Source(shape=sources[0].shape, dtype=sources[0].dtype, order=sources[0].order); if isinstance(sink, list): sinks = sink elif sink is None: sinks = [] else: sinks = [sink] sinks = [io.initialize(s, hint=sources[0]) for s in sinks] sinks = [io.as_source(s).as_virtual() for s in sinks] axes = block_axes(sources[0], axes=axes) split = ft.partial(split_into_blocks, processes=processes, axes=axes, size_max=size_max, size_min=size_min, overlap=overlap, optimization=optimization, optimization_fix=optimization_fix, neighbours=neighbours, verbose=False) source_blocks = [split(s) for s in sources] sink_blocks = [split(s) for s in sinks] n_blocks = len(source_blocks[0]) source_blocks = [[blocks[i] for blocks in source_blocks] for i in range(n_blocks)] sink_blocks = [[blocks[i] for blocks in sink_blocks] for i in range(n_blocks)] if function_type is None: function_type = 'array' if function_type == 'block': func = ft.partial(process_block_block, function=function, as_memory=as_memory, return_result=return_result, verbose=verbose, **kwargs) elif function_type == 'source': func = ft.partial(process_block_source, function=function, as_memory=as_memory, as_array=False, verbose=verbose, **kwargs) elif function_type == 'array': func = ft.partial(process_block_source, function=function, as_memory=as_memory, as_array=True, verbose=verbose, **kwargs) else: raise ValueError( "function type %r not 'array', 'source', 'block' or None!") if not isinstance(processes, int) and processes != "serial": processes = mp.cpu_count() if verbose: timer = tmr.Timer() print("Processing %d blocks with function %r." % (n_blocks, function.__name__)) if isinstance(processes, int): #from bounded_pool_executor import BoundedProcessPoolExecutor with cf.ProcessPoolExecutor(max_workers=processes) as executor: #with BoundedProcessPoolExecutor(max_workers=processes) as executor: futures = [ executor.submit(func, *args) for args in zip(source_blocks, sink_blocks) ] result = [f.result() for f in futures] #executor.map(function, source_blocks, sink_blocks) else: result = [func(*args) for args in zip(source_blocks, sink_blocks)] #analysis:ignore if verbose: timer.print_elapsed_time("Processed %d blocks with function %r" % (n_blocks, function.__name__)) #gc.collect(); if return_result: ret = result else: ret = sink if return_blocks: ret = (ret, [source_blocks, sink_blocks]) return ret
def _test(): import numpy as np import ClearMap.ParallelProcessing.DataProcessing.ArrayProcessing as ap ## Lookup table processing #apply_lut x = np.random.randint(0, 100, size=(20, 30)) lut = np.arange(100) + 1 y = ap.apply_lut(x, lut) assert np.all(y == x + 1) #apply_lut_to_index import ClearMap.ImageProcessing.Topology.Topology3d as t3d kernel = t3d.index_kernel(dtype=int) import ClearMap.ImageProcessing.Binary.Smoothing as sm lut = sm.initialize_lookup_table() data = np.array(np.random.rand(150, 30, 40) > 0.75, order='F') result = ap.apply_lut_to_index(data, kernel, lut, sink=None, verbose=True) import ClearMap.Visualization.Plot3d as p3d p3d.plot([[data, result]]) ### Correlation #correlate1d kernel = np.array(range(11), dtype='uint32') data = np.array(np.random.randint(0, 2**27, (300, 400, 1500), dtype='uint32'), order='F') #data = np.array(np.random.rand(3,4,5), order='F'); data = np.empty((300, 400, 1500), order='F') kernel = np.array([1, 2, 3, 4, 5], dtype='uint8') sink = 'test.npy' import ClearMap.Utils.Timer as tmr import scipy.ndimage as ndi timer = tmr.Timer() for axis in range(3): print(axis) corr_ndi = ndi.correlate1d(data, axis=axis, mode='constant', cval=0) timer.print_elapsed_time('ndi') timer = tmr.Timer() for axis in range(3): print(axis) corr = ap.correlate1d(data, sink=sink, kernel=kernel, axis=axis, verbose=False, processes=None) timer.print_elapsed_time('ap') assert np.allclose(corr.array, corr_ndi) # IO import ClearMap.ParallelProcessing.DataProcessing.ArrayProcessing as ap import numpy as np reload(ap) data = np.random.rand(10, 200, 10) sink = ap.write('test.npy', data, verbose=True) assert (np.all(sink.array == data)) read = ap.read('test.npy', verbose=True) assert (np.all(read.array == data)) ap.io.delete_file('test.npy') # where reload(ap) data = np.random.rand(30, 20, 40) > 0.5 where_np = np.array(np.where(data)).T where = ap.where(data, cutoff=2**0) check_np = np.zeros(data.shape, dtype=bool) check = np.zeros(data.shape, dtype=bool) check_np[tuple(where_np.T)] = True check[tuple(where.array.T)] = True assert (np.all(check_np == check))
def graph_from_skeleton(skeleton, points = None, radii = None, vertex_coordinates = True, check_border = True, delete_border = False, verbose = False): """Converts a binary skeleton image to a graph-tool graph. Arguments --------- skeleton : array Source with 2d/3d binary skeleton. points : array List of skeleton points as 1d indices of flat skeleton array (optional to save processing time). radii : array List of radii associated with each vertex. vertex_coordinates : bool If True, store coordiantes of the vertices / edges. check_border : bool If True, check if the boder is empty. The algorithm reuqires this. delete_border : bool If True, delete the border. verbose : bool If True, print progress information. Returns ------- graph : Graph class The graph corresponding to the skeleton. Note ---- Edges are detected between neighbouring foreground pixels using 26-connectivty. """ skeleton = io.as_source(skeleton); if delete_border: skeleton = t3d.delete_border(skeleton); check_border = False; if check_border: if not t3d.check_border(skeleton): raise ValueError('The skeleton array needs to have no points on the border!'); if verbose: timer = tmr.Timer(); timer_all = tmr.Timer(); print('Graph from skeleton calculation initialize.!'); if points is None: points = ap.where(skeleton.reshape(-1, order = 'A')).array; if verbose: timer.print_elapsed_time('Point list generation'); timer.reset(); #create graph n_vertices = points.shape[0]; g = ggt.Graph(n_vertices=n_vertices, directed=False); g.shape = skeleton.shape; if verbose: timer.print_elapsed_time('Graph initialized with %d vertices' % n_vertices); timer.reset(); #detect edges edges_all = np.zeros((0,2), dtype = int); for i,o in enumerate(t3d.orientations()): # calculate off set offset = np.sum((np.hstack(np.where(o))-[1,1,1]) * skeleton.strides) edges = ap.neighbours(points, offset); if len(edges) > 0: edges_all = np.vstack([edges_all, edges]); if verbose: timer.print_elapsed_time('%d edges with orientation %d/13 found' % (edges.shape[0], i+1)); timer.reset(); if edges_all.shape[0] > 0: g.add_edge(edges_all); if verbose: timer.print_elapsed_time('Added %d edges to graph' % (edges_all.shape[0])); timer.reset(); if vertex_coordinates: vertex_coordinates = np.array(np.unravel_index(points, skeleton.shape, order=skeleton.order)).T; g.set_vertex_coordinates(vertex_coordinates); if radii is not None: g.set_vertex_radius(radii); if verbose: timer_all.print_elapsed_time('Skeleton to Graph'); return g;
def load(filename, region=None, shared=False, blocks=None, processes=cpu_count(), verbose=False): """Load a large npy array into memory in parallel Arguments: filename : str filename of array to load region : Region or None if not None this specifies the sub-region to read shared : bool if True read into shared memory blocks : int or None number of blocks to split array into for parallel processing processes : None or int number of processes, if None use number of cpus verbose : bool print info about the file to be loaded Returns: array the data as numpy array """ if processes is None: processes = cpu_count() if blocks is None: blocks = processes * defaultBlocksPerProcess #get specs from header specs shape, dtype, order, offset = readNumpyHeader(filename) if verbose: timer = tmr.Timer() print( 'Loading array of shape = %r, dtype = %r, order = %r, offset = %r' % (shape, dtype, order, offset)) if region is not None: shape = region.shape() sourceSlice = region.sourceSlice() off = _offsetFromSlice(sourceSlice, order=order) if shared: data = shm.create(shape, dtype=dtype, order=order) else: data = np.empty(shape, dtype=dtype, order=order) d = data.reshape(-1, order='A') if dtype == bool: d = d.view('uint8') if region is not None: if order == 'F': offset += data.strides[-1] * off else: offset += data.strides[1] * off filename = str(filename).encode('UTF-8') code.load(data=d, filename=filename, offset=offset, blocks=blocks, processes=processes) if verbose: timer.print_elapsed_time(head='Loading array from %s' % filename) return data
def detect_cells_block(source, parameter=default_cell_detection_parameter): """Detect cells in a Block.""" #initialize parameter and slicings verbose = parameter.get('verbose', False) if verbose: prefix = 'Block %s: ' % (source.info(), ) total_time = tmr.Timer(prefix) base_slicing = source.valid.base_slicing valid_slicing = source.valid.slicing valid_lower = source.valid.lower valid_upper = source.valid.upper lower = source.lower parameter_intensity = parameter.get('intensity_detection', None) measure_to_array = dict() if parameter_intensity: parameter_intensity = parameter_intensity.copy() measure = parameter_intensity.pop('measure', []) if measure is None: measure = [] for m in measure: measure_to_array[m] = None if 'source' in measure_to_array: measure_to_array['source'] = source # correct illumination parameter_illumination = parameter.get('illumination_correction', None) if parameter_illumination: parameter_illumination = parameter_illumination.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_illumination, head=prefix + 'Illumination correction') save = parameter_illumination.pop('save', None) corrected = ic.correct_illumination(source, **parameter_illumination) if save: save = io.as_source(save) save[base_slicing] = corrected[valid_slicing] if verbose: timer.print_elapsed_time('Illumination correction') else: corrected = np.array(source.array) if 'illumination' in measure_to_array: measure_to_array['illumination'] = corrected #background subtraction parameter_background = parameter.get('background_correction', None) if parameter_background: parameter_background = parameter_background.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_background, head=prefix + 'Background removal') save = parameter_background.pop('save', None) background = remove_background(corrected, **parameter_background) if save: save = io.as_source(save) save[base_slicing] = corrected[valid_slicing] if verbose: timer.print_elapsed_time('Illumination correction') else: background = corrected del corrected if 'background' in measure_to_array: measure_to_array['background'] = background # equalize parameter_equalize = parameter.get('equalization', None) if parameter_equalize: parameter_equalize = parameter_equalize.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_equalize, head=prefix + 'Equalization:') save = parameter_equalize.pop('save', None) equalized = equalize(background, mask=None, **parameter_equalize) if save: save = io.as_source(save) save[base_slicing] = equalized[valid_slicing] if verbose: timer.print_elapsed_time('Equalization') else: equalized = background del background if 'equalized' in measure_to_array: measure_to_array['equalized'] = equalized #DoG filter parameter_dog_filter = parameter.get('dog_filter', None) if parameter_dog_filter: parameter_dog_filter = parameter_dog_filter.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_dog_filter, head=prefix + 'DoG filter:') save = parameter_dog_filter.pop('save', None) dog = dog_filter(equalized, **parameter_dog_filter) if save: save = io.as_source(save) save[base_slicing] = dog[valid_slicing] if verbose: timer.print_elapsed_time('DoG filter') else: dog = equalized del equalized if 'dog' in measure_to_array: measure_to_array['dog'] = dog #Maxima detection parameter_maxima = parameter.get('maxima_detection', None) parameter_shape = parameter.get('shape_detection', None) if parameter_shape or parameter_intensity: if not parameter_maxima: print( prefix + 'Warning: maxima detection needed for shape and intensity detection!' ) parameter_maxima = dict() if parameter_maxima: parameter_maxima = parameter_maxima.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_maxima, head=prefix + 'Maxima detection:') save = parameter_maxima.pop('save', None) valid = parameter_maxima.pop('valid', None) # extended maxima maxima = md.find_maxima(source.array, **parameter_maxima, verbose=verbose) if save: save = io.as_source(save) save[base_slicing] = maxima[valid_slicing] #center of maxima if parameter_maxima['h_max']: centers = md.find_center_of_maxima(source, maxima=maxima, verbose=verbose) else: centers = ap.where(maxima).array if verbose: timer.print_elapsed_time('Maxima detection') #correct for valid region if valid: ids = np.ones(len(centers), dtype=bool) for c, l, u in zip(centers.T, valid_lower, valid_upper): ids = np.logical_and(ids, np.logical_and(l <= c, c < u)) centers = centers[ids] del ids del dog, maxima results = (centers, ) #cell shape detection if parameter_shape: parameter_shape = parameter_shape.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_shape, head=prefix + 'Shape detection:') save = parameter_shape.pop('save', None) # shape detection shape = sd.detect_shape(source, centers, **parameter_shape, verbose=verbose) if save: save = io.as_source(save) save[base_slicing] = shape[valid_slicing] #size detection max_label = centers.shape[0] sizes = sd.find_size(shape, max_label=max_label) valid = sizes > 0 if verbose: timer.print_elapsed_time('Shape detection') results += (sizes, ) else: valid = None shape = None #cell intensity detection if parameter_intensity: parameter_intensity = parameter_intensity.copy() if verbose: timer = tmr.Timer(prefix) hdict.pprint(parameter_intensity, head=prefix + 'Intensity detection:') if not shape is None: r = parameter_intensity.pop('shape', 3) if isinstance(r, tuple): r = r[0] for m in measure: if shape is not None: intensity = sd.find_intensity(measure_to_array[m], label=shape, max_label=max_label, **parameter_intensity) else: intensity = me.measure_expression(measure_to_array[m], centers, search_radius=r, **parameter_intensity, processes=1, verbose=False) results += (intensity, ) if verbose: timer.print_elapsed_time('Shape detection') if valid is not None: results = tuple(r[valid] for r in results) #correct coordinate offsets of blocks results = (results[0] + lower, ) + results[1:] #correct shapes for merging results = tuple(r[:, None] if r.ndim == 1 else r for r in results) if verbose: total_time.print_elapsed_time('Cell detection') gc.collect() return results
def write(filename, data, slicing=None, blocks=None, processes=None, verbose=False): """Write a large array to disk in parallel. Arguments --------- filename : str Filename of array to load. data : array Array to save to disk. blocks : int or None Number of blocks to split array into for parallel processing. processes : None or int Number of processes, if None use number of cpus. verbose : bool Print info about the file to be loaded. Returns ------- filename : str The filename of the numpy array on disk. """ if processes is None: processes = mp.cpu_count() if blocks is None: blocks = processes * default_blocks_per_process #data data = io.as_source(data) #prepare sink is_file = fu.is_file(filename) if slicing is not None and not is_file: raise ValueError('Cannot write to a slice to a non-existing file %s!' % filename) if slicing is None: #create file on disk via memmap fortran_order = 'F' == data.order memmap = np.lib.format.open_memmap(filename, mode='w+', shape=data.shape, dtype=data.dtype, fortran_order=fortran_order) memmap.flush() del (memmap) sink = mmp.Source(location=filename) if slicing is not None: sink = slc.Slice(source=sink, slicing=slicing) shape, dtype, order, offset = sink.shape, sink.dtype, sink.order, sink.offset if (data.order != order): raise RuntimeError('Order of arrays do not match %r!=%r' % (data.order, order)) if order not in ['C', 'F']: raise NotImplementedError( 'Cannot read in parallel from non-contigous source!') #TODO: implement parallel reader with strides ! if verbose: timer = tmr.Timer() print( 'Writing data to sink of shape = %r, dtype = %r, order = %r, offset = %r' % (shape, dtype, order, offset)) d = data.reshape(-1, order='A') if d.dtype == bool: d = d.view('uint8') code.write(data=d, filename=filename, offset=offset, blocks=blocks, processes=processes) if verbose: timer.print_elapsed_time(head='Writing data to %s' % filename) return filename