def _processSubStack(dsr):
"""Helper to process stack in parallel"""
sf = dsr[0];
pp = dsr[1];
sub = dsr[2];
verbose = dsr[3];
timer = Timer();
pw = ProcessWriter(sub["stackId"]);
if verbose:
pw.write("processing substack " + str(sub["stackId"]) + "/" + str(sub["nStacks"]));
pw.write("file = " + sub["source"]);
pw.write("segmentation = " + str(sf));
pw.write("ranges: x,y,z = " + str(sub["x"]) + "," + str(sub["y"]) + "," + str(sub["z"]));
img = io.readData(sub["source"], x = sub["x"], y = sub["y"], z = sub["z"]);
if verbose:
pw.write(timer.elapsedTime(head = 'Reading data of size ' + str(img.shape)));
timer.reset();
seg = sf(img, subStack = sub, out = pw, **pp);
if verbose:
pw.write(timer.elapsedTime(head = 'Processing substack of size ' + str(img.shape)));
return seg;
def findPixelCoordinates(imgmax,
subStack=None,
verbose=False,
out=sys.stdout,
**parameter):
"""Find coordinates of all pixel in an image with positive or True value
Arguments:
img (array): image data
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: coordinates of centers of True pixels, shape is (n,d) where n is number of maxima and d the dimension of the image
"""
timer = Timer()
centers = numpy.nonzero(imgmax)
centers = numpy.vstack(centers).T
if verbose:
out.write(timer.elapsedTime(head='Cell Centers') + '\n')
return centers
def filterDoG(img, filterDoGParameter = None, size = None, sigma = None, sigma2 = None, save = None, verbose = None,
subStack = None, out = sys.stdout, **parameter):
"""Difference of Gaussians (DoG) filter step
Arguments:
img (array): image data
filterDoGParameter (dict):
========= ==================== ================================================================
Name Type Descritption
========= ==================== ================================================================
*size* (tuple or None) size for the DoG filter
if None, do not correct for any background
*sigma* (tuple or None) std of outer Guassian, if None autmatically determined from size
*sigma2* (tuple or None) std of inner Guassian, if None autmatically determined from size
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print progress information
========= ==================== ================================================================
subStack (dict or None): sub-stack information
out (object): object to write progress info to
Returns:
array: DoG filtered image
"""
timer = Timer();
dogSize = getParameter(filterDoGParameter, "size", size);
dogSigma = getParameter(filterDoGParameter, "sigma", sigma);
dogSigma2= getParameter(filterDoGParameter, "sigma2",sigma2);
dogSave = getParameter(filterDoGParameter, "save", save);
verbose = getParameter(filterDoGParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'DoG:', size = dogSize, sigma = dogSigma, sigma2 = dogSigma2, save = dogSave);
#DoG filter
img = img.astype('float32'); # always convert to float for downstream processing
if not dogSize is None:
fdog = filterKernel(ftype = 'DoG', size = dogSize, sigma = dogSigma, sigma2 = dogSigma2);
fdog = fdog.astype('float32');
#img = correlate(img, fdog);
#img = scipy.signal.correlate(img, fdog);
img = correlate(img, fdog);
#img = convolve(img, fdog, mode = 'same');
img[img < 0] = 0;
if verbose > 1:
plotTiling(img);
if not dogSave is None:
writeSubStack(dogSave, img, subStack = subStack);
if verbose:
out.write(timer.elapsedTime(head = 'DoG') + '\n');
return img
def detectCellShape(img, peaks, detectCellShapeParameter = None, threshold = None, save = None, verbose = False,
subStack = None, out = sys.stdout, **parameter):
"""Find cell shapes as labeled image
Arguments:
img (array): image data
peaks (array): point data of cell centers / seeds
detectCellShape (dict):
============ =================== ===========================================================
Name Type Descritption
============ =================== ===========================================================
*threshold* (float or None) threshold to determine mask, pixel below this are background
if None no mask is generated
*save* (tuple) size of the box on which to perform the *method*
*verbose* (bool or int) print / plot information about this step
============ =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: labeled image where each label indicates a cell
"""
threshold = getParameter(detectCellShapeParameter, "threshold", threshold);
save = getParameter(detectCellShapeParameter, "save", save);
verbose = getParameter(detectCellShapeParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Cell shape detection:', threshold = threshold, save = save);
# extended maxima
timer = Timer();
if threshold is None:
imgmask = None;
else:
imgmask = img > threshold;
imgpeaks = voxelizePixel(peaks, dataSize = img.shape, weights = numpy.arange(1, peaks.shape[0]+1));
imgws = watershed(-img, imgpeaks, mask = imgmask);
#imgws = watershed_ift(-img.astype('uint16'), imgpeaks);
#imgws[numpy.logical_not(imgmask)] = 0;
if not save is None:
writeSubStack(save, imgws.astype('int32'), subStack = subStack);
if verbose > 1:
#plotTiling(img)
plotOverlayLabel(img * 0.01, imgws, alpha = False);
#plotOverlayLabel(img, imgmax.astype('int64'), alpha = True)
if verbose:
out.write(timer.elapsedTime(head = 'Cell Shape:') + '\n');
return imgws
def removeBackground(img, removeBackgroundParameter = None, size = None, save = None, verbose = False,
subStack = None, out = sys.stdout, **parameter):
"""Remove background via subtracting a morphological opening from the original image
Background removal is done z-slice by z-slice.
Arguments:
img (array): image data
removeBackGroundParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*size* (tuple or None) size for the structure element of the morphological opening
if None, do not correct for any background
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: background corrected image
"""
size = getParameter(removeBackgroundParameter, "size", size);
save = getParameter(removeBackgroundParameter, "save", save);
verbose = getParameter(removeBackgroundParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Background Removal:', size = size, save = save);
if size is None:
return img;
img = io.readData(img);
timer = Timer();
# background subtraction in each slice
se = structureElement('Disk', size).astype('uint8');
for z in range(img.shape[2]):
#img[:,:,z] = img[:,:,z] - grey_opening(img[:,:,z], structure = structureElement('Disk', (30,30)));
#img[:,:,z] = img[:,:,z] - morph.grey_opening(img[:,:,z], structure = self.structureELement('Disk', (150,150)));
img[:,:,z] = img[:,:,z] - cv2.morphologyEx(img[:,:,z], cv2.MORPH_OPEN, se)
if not save is None:
writeSubStack(save, img, subStack = subStack)
if verbose > 1:
plotTiling(10*img);
if verbose:
out.write(timer.elapsedTime(head = 'Background') + '\n');
return img
def greyReconstruction(img, mask, greyReconstructionParameter = None, method = None, size = 3, save = None, verbose = False,
subStack = None, out = sys.stdout, **parameter):
"""Calculates the grey reconstruction of the image
Reconstruction is done z-slice by z-slice.
Arguments:
img (array): image data
removeBackGroundParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*method* (tuple or None) 'dilation' or 'erosion', if None return original image
*size* (int or tuple) size of structuring element
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: grey reconstructed image
"""
method = getParameter(greyReconstructionParameter, "method", method);
size = getParameter(greyReconstructionParameter, "size", size);
save = getParameter(greyReconstructionParameter, "save", save);
verbose= getParameter(greyReconstructionParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Grey reconstruction:', method = method, size = size, save = save);
if method is None:
return img;
timer = Timer();
# background subtraction in each slice
se = structureElement('Disk', size).astype('uint8');
for z in range(img.shape[2]):
#img[:,:,z] = img[:,:,z] - grey_opening(img[:,:,z], structure = structureElement('Disk', (30,30)));
#img[:,:,z] = img[:,:,z] - morph.grey_opening(img[:,:,z], structure = self.structureELement('Disk', (150,150)));
img[:,:,z] = img[:,:,z] - reconstruct(img[:,:,z], method = method, selem = se)
if not save is None:
writeSubStack(save, img, subStack = subStack)
if verbose > 1:
plotTiling(img);
if verbose:
out.write(timer.elapsedTime(head = 'Grey reconstruction:') + '\n');
return img
def classifyPixel(img, classifyPixelParameter = None, subStack = None, verbose = False, out = sys.stdout, **parameter):
"""Detect Cells Using a trained classifier in Ilastik
Arguments:from ClearMap.ImageProcessing.CellSizeDetection import detectCellShape, findCellSize, findCellIntensity
img (array): image data
classifyPixelParameter (dict):
============ ==================== ===========================================================
Name Type Descritption
============ ==================== ===========================================================
*classifier* (str or None) Ilastik project file with trained pixel classifier
*save* (str or None) save the classification propabilities to a file
*verbose* (bool or int) print / plot information about this step
============ ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: probabilities for each pixel to belong to a class in the classifier, shape is (img.shape, number of classes)
"""
ilastik.checkInitialized();
classifier = getParameter(classifyPixelParameter, "classifier", None);
save = getParameter(classifyPixelParameter, "save", None);
verbose = getParameter(classifyPixelParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Ilastik classification:', classifier = classifier, save = save);
timer = Timer();
#remove background
#img2 = removeBackground(img, verbose = verbose, out = out, **parameter);
#classify image
if classifier is None:
return img;
imgclass = ilastik.classifyPixel(classifier, img);
if not save is None:
for i in range(imgclass.shape[4]):
fn = save[:-4] + '_class_' + str(i) + save[-4:];
writeSubStack(fn, imgclass[:,:,:,i], subStack = subStack)
if verbose > 1:
for i in range(imgclass.shape[4]):
plotTiling(imgclass[:,:,:,i]);
if verbose:
out.write(timer.elapsedTime(head = 'Ilastik classification') + '\n');
return imgclass;
def classifyPixel(img, classifyPixelParameter = None, subStack = None, verbose = False, out = sys.stdout, **parameter):
"""Detect Cells Using a trained classifier in Ilastik
Arguments:from ClearMap.ImageProcessing.CellSizeDetection import detectCellShape, findCellSize, findCellIntensity
img (array): image data
classifyPixelParameter (dict):
============ ==================== ===========================================================
Name Type Descritption
============ ==================== ===========================================================
*classifier* (str or None) Ilastik project file with trained pixel classifier
*save* (str or None) save the classification propabilities to a file
*verbose* (bool or int) print / plot information about this step
============ ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: probabilities for each pixel to belong to a class in the classifier, shape is (img.shape, number of classes)
"""
ilastik.checkInitialized();
classifier = getParameter(classifyPixelParameter, "classifier", None);
save = getParameter(classifyPixelParameter, "save", None);
verbose = getParameter(classifyPixelParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Ilastik classification:', classifier = classifier, save = save);
timer = Timer();
#remove background
#img2 = removeBackground(img, verbose = verbose, out = out, **parameter);
#classify image
if classifier is None:
return img;
imgclass = ilastik.classifyPixel(classifier, img);
if not save is None:
for i in range(imgclass.shape[4]):
fn = save[:-4] + '_class_' + str(i) + save[-4:];
writeSubStack(fn, imgclass[:,:,:,i], subStack = subStack)
if verbose > 1:
for i in range(imgclass.shape[4]):
plotTiling(imgclass[:,:,:,i]);
if verbose:
out.write(timer.elapsedTime(head = 'Ilastik classification') + '\n');
return imgclass;
def findCellIntensity(img, imglabel, findCellIntensityParameter = None, maxLabel = None, method = 'Sum', verbose = False,
out = sys.stdout, **parameter):
"""Find integrated cell intensity given cell shapes as labled image
Arguments:
img (array or str): image data
imglabel (array or str): labeled image, where each cell has its own label
findCellIntensityParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*maxLabel* (int or None) maximal label to include, if None determine automatically
*method* (str) method to use for measurment: 'Sum', 'Mean', 'Max', 'Min'
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: measured intensities
"""
maxLabel = getParameter(findCellIntensityParameter, "maxLabel", maxLabel);
method = getParameter(findCellIntensityParameter, "method", method);
verbose = getParameter(findCellIntensityParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Cell intensity detection:', method = method, maxLabel = maxLabel);
timer = Timer();
if maxLabel is None:
maxLabel = imglabel.max();
if method.lower() == 'sum':
i = scipy.ndimage.measurements.sum(img, labels = imglabel, index = numpy.arange(1, maxLabel + 1));
elif method.lower() == 'mean':
i = scipy.ndimage.measurements.mean(img, labels = imglabel, index = numpy.arange(1, maxLabel + 1));
elif method.lower() == 'max':
i = scipy.ndimage.measurements.maximum(img, labels = imglabel, index = numpy.arange(1, maxLabel + 1));
elif method.lower() == 'min':
i = scipy.ndimage.measurements.minimum(img, labels = imglabel, index = numpy.arange(1, maxLabel + 1));
else:
raise RuntimeError('cellIntensity: unkown method %s!' % method);
if verbose:
out.write(timer.elapsedTime(head = 'Cell intensity detection:') + '\n');
return i
def detectCells(source, sink = None, method ="SpotDetection", processMethod = all, verbose = False, **parameter):
"""Detect cells in data
This is a main script to start running the cell detection.
Arguments:
source (str or array): Image source
sink (str or None): destination for the results
method (str or function):
================ ============================================================
Method Description
================ ============================================================
"SpotDetection" uses predefined spot detection pipline
"Ilastik" uses predefined pipline with cell classification via Ilastik
function a user defined function
================ ============================================================
processMethod (str or all): 'sequential' or 'parallel'. if all its choosen
automatically
verbose (bool): print info
**parameter (dict): parameter for the image procesing sub-routines
Returns:
"""
timer = Timer();
# run segmentation
if method == "SpotDetection":
detectCells = ClearMap.ImageProcessing.SpotDetection.detectSpots;
elif method == 'Ilastik':
if ClearMap.ImageProcessing.Ilastik.Initialized:
detectCells = ClearMap.ImageProcessing.IlastikClassification.classifyCells;
processMethod = 'sequential'; #ilastik does parallel processing so force sequential processing here
else:
raise RuntimeError("detectCells: Ilastik not initialized, fix in Settings.py or use SpotDectection method instead!");
else:
raise RuntimeError("detectCells: invalid method %s" % str(method));
if processMethod == 'sequential':
result = sequentiallyProcessStack(source, sink = sink, function = detectCells, verbose = verbose, **parameter);
elif processMethod is all or processMethod == 'parallel':
result = parallelProcessStack(source, sink = sink, function = detectCells, verbose = verbose, **parameter);
else:
raise RuntimeError("detectCells: invalid processMethod %s" % str(processMethod));
if verbose:
timer.printElapsedTime("Total Cell Detection");
return result;
def detectCells(source, sink = None, method ="SpotDetection", processMethod = all, verbose = False, **parameter):
"""Detect cells in data
This is a main script to start running the cell detection.
Arguments:
source (str or array): Image source
sink (str or None): destination for the results
method (str or function):
================ ============================================================
Method Description
================ ============================================================
"SpotDetection" uses predefined spot detection pipline
"Ilastik" uses predefined pipline with cell classification via Ilastik
function a user defined function
================ ============================================================
processMethod (str or all): 'sequential' or 'parallel'. if all its choosen
automatically
verbose (bool): print info
**parameter (dict): parameter for the image procesing sub-routines
Returns:
"""
timer = Timer();
# run segmentation
if method == "SpotDetection":
detectCells = ClearMap.ImageProcessing.SpotDetection.detectSpots;
elif method == 'Ilastik':
if ClearMap.ImageProcessing.Ilastik.Initialized:
detectCells = ClearMap.ImageProcessing.IlastikClassification.classifyCells;
processMethod = 'sequential'; #ilastik does parallel processing so force sequential processing here
else:
raise RuntimeError("detectCells: Ilastik not initialized, fix in Settings.py or use SpotDectection method instead!");
else:
raise RuntimeError("detectCells: invalid method %s" % str(method));
if processMethod == 'sequential':
result = sequentiallyProcessStack(source, sink = sink, function = detectCells, verbose = verbose, **parameter);
elif processMethod is all or processMethod == 'parallel':
result = parallelProcessStack(source, sink = sink, function = detectCells, verbose = verbose, **parameter);
else:
raise RuntimeError("detectCells: invalid processMethod %s" % str(processMethod));
if verbose:
timer.printElapsedTime("Total Cell Detection");
return result;
def findCellSize(imglabel,
findCelSizeParameter=None,
maxLabel=None,
verbose=False,
out=sys.stdout,
**parameter):
"""Find cell size given cell shapes as labled image
Arguments:
imglabel (array or str): labeled image, where each cell has its own label
findCelSizeParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*maxLabel* (int or None) maximal label to include, if None determine automatically
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: measured intensities
"""
maxLabel = getParameter(findCelSizeParameter, "maxLabel", maxLabel)
verbose = getParameter(findCelSizeParameter, "verbose", verbose)
if verbose:
writeParameter(out=out, head='Cell size detection:', maxLabel=maxLabel)
timer = Timer()
if maxLabel is None:
maxLabel = int(imglabel.max())
size = scipy.ndimage.measurements.sum(numpy.ones(imglabel.shape,
dtype=bool),
labels=imglabel,
index=numpy.arange(1, maxLabel + 1))
if verbose:
out.write(timer.elapsedTime(head='Cell size detection:') + '\n')
return size
def findPixelCoordinates(imgmax, subStack = None, verbose = False, out = sys.stdout, **parameter):
"""Find coordinates of all pixel in an image with positive or True value
Arguments:
img (array): image data
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: coordinates of centers of True pixels, shape is (n,d) where n is number of maxima and d the dimension of the image
"""
timer = Timer();
centers = numpy.nonzero(imgmax);
centers = numpy.vstack(centers).T;
if verbose:
out.write(timer.elapsedTime(head = 'Cell Centers') + '\n');
return centers;
def _processSubStack(dsr):
"""Helper to process stack in parallel"""
sf = dsr[0];
pp = dsr[1];
sub = dsr[2];
verbose = dsr[3];
timer = Timer();
pw = ProcessWriter(sub["stackId"]);
if verbose:
pw.write("processing substack " + str(sub["stackId"]) + "/" + str(sub["nStacks"]));
pw.write("file = " + sub["source"]);
pw.write("segmentation = " + str(sf));
pw.write("ranges: x,y,z = " + str(sub["x"]) + "," + str(sub["y"]) + "," + str(sub["z"]));
img = io.readData(sub["source"], x = sub["x"], y = sub["y"], z = sub["z"]);
if verbose:
pw.write(timer.elapsedTime(head = 'Reading data of size ' + str(img.shape)));
timer.reset();
seg = sf(img, subStack = sub, out = pw, **pp);
if verbose:
pw.write(timer.elapsedTime(head = 'Processing substack of size ' + str(img.shape)));
return seg;
def findCellSize(imglabel, findCelSizeParameter = None, maxLabel = None, verbose = False,
out = sys.stdout, **parameter):
"""Find cell size given cell shapes as labled image
Arguments:
imglabel (array or str): labeled image, where each cell has its own label
findCelSizeParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*maxLabel* (int or None) maximal label to include, if None determine automatically
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: measured intensities
"""
maxLabel = getParameter(findCelSizeParameter, "maxLabel", maxLabel);
verbose = getParameter(findCelSizeParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Cell size detection:', maxLabel = maxLabel);
timer = Timer();
if maxLabel is None:
maxLabel = int(imglabel.max());
size = scipy.ndimage.measurements.sum(numpy.ones(imglabel.shape, dtype = bool), labels = imglabel, index = numpy.arange(1, maxLabel + 1));
if verbose:
out.write(timer.elapsedTime(head = 'Cell size detection:') + '\n');
return size
def filterDoG(img,
filterDoGParameter=None,
size=None,
sigma=None,
sigma2=None,
save=None,
verbose=None,
subStack=None,
out=sys.stdout,
**parameter):
"""Difference of Gaussians (DoG) filter step
Arguments:
img (array): image data
filterDoGParameter (dict):
========= ==================== ================================================================
Name Type Descritption
========= ==================== ================================================================
*size* (tuple or None) size for the DoG filter
if None, do not correct for any background
*sigma* (tuple or None) std of outer Guassian, if None autmatically determined from size
*sigma2* (tuple or None) std of inner Guassian, if None autmatically determined from size
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print progress information
========= ==================== ================================================================
subStack (dict or None): sub-stack information
out (object): object to write progress info to
Returns:
array: DoG filtered image
"""
timer = Timer()
dogSize = getParameter(filterDoGParameter, "size", size)
dogSigma = getParameter(filterDoGParameter, "sigma", sigma)
dogSigma2 = getParameter(filterDoGParameter, "sigma2", sigma2)
dogSave = getParameter(filterDoGParameter, "save", save)
verbose = getParameter(filterDoGParameter, "verbose", verbose)
if verbose:
writeParameter(out=out,
head='DoG:',
size=dogSize,
sigma=dogSigma,
sigma2=dogSigma2,
save=dogSave)
#DoG filter
img = img.astype('float32')
# always convert to float for downstream processing
if not dogSize is None:
fdog = filterKernel(ftype='DoG',
size=dogSize,
sigma=dogSigma,
sigma2=dogSigma2)
fdog = fdog.astype('float32')
#img = correlate(img, fdog);
#img = scipy.signal.correlate(img, fdog);
img = correlate(img, fdog)
#img = convolve(img, fdog, mode = 'same');
img[img < 0] = 0
if verbose > 1:
plotTiling(img)
if not dogSave is None:
writeSubStack(dogSave, img, subStack=subStack)
if verbose:
out.write(timer.elapsedTime(head='DoG') + '\n')
return img
def _test():
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.ParallelProcessing.ParallelIO as pio
reload(pio)
#dat = np.random.rand(2000,2000,1000) > 0.5;
#dat = np.random.rand(1000,1000,500) > 0.5;
#dat = np.random.rand(1000,1000,500);
dat = np.random.rand(200, 300, 400)
filename = 'test.npy'
timer = Timer()
np.save(filename, dat)
timer.print_elapsed_time('Numpy saving data of size %d' % dat.size)
filename2 = 'test2.npy'
timer = Timer()
pio.write(filename2, dat, processes=4, blocks=4, verbose=False)
timer.print_elapsed_time('ParallelIO writing data of size %d' % dat.size)
timer = Timer()
dat2 = np.load(filename)
timer.print_elapsed_time('Numpy loading data of size %d' % dat.size)
print(np.all(dat == dat2))
timer = Timer()
dat3 = pio.read(filename, verbose=True)
timer.print_elapsed_time('ParallelIO reading data of size %d' % dat2.size)
print(np.all(dat3 == dat))
pio.io.fu.delete_file(filename)
pio.io.fu.delete_file(filename2)
def findIntensity(img, centers, findIntensityParameter = None, method = None, size = (3,3,3), verbose = False,
out = sys.stdout, **parameter):
"""Find instensity value around centers in the image
Arguments:
img (array): image data
findIntensityParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*method* (str, func, None) method to use to determine intensity (e.g. "Max" or "Mean")
if None take intensities at the given pixels
*size* (tuple) size of the box on which to perform the *method*
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: measured intensities
"""
method = getParameter(findIntensityParameter, "method", "Max");
size = getParameter(findIntensityParameter, "size", (3,3,3));
verbose = getParameter(findIntensityParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Cell Intensities:', method = method, size = size);
timer = Timer();
if centers.shape[0] == 0:
return numpy.zeros(0);
if method is None:
return numpy.array([img[centers[i,0], centers[i,1], centers[i,2]] for i in range(centers.shape[0])]);
isize = img.shape;
#print isize
offs = structureElementOffsets(size);
if isinstance(method, str):
method = eval('numpy.' + method.lower());
intensities = numpy.zeros(centers.shape[0], dtype = img.dtype);
for c in range(centers.shape[0]):
xmin = int(-offs[0,0] + centers[c,0]);
if xmin < 0:
xmin = 0;
xmax = int(offs[0,1] + centers[c,0]);
if xmax > isize[0]:
xmax = isize[0];
ymin = int(-offs[1,0] + centers[c,1]);
if ymin < 0:
ymin = 0;
ymax = int(offs[1,1] + centers[c,1]);
if ymax > isize[1]:
ymax = isize[1];
zmin = int(-offs[2,0] + centers[c,2]);
if zmin < 0:
zmin = 0;
zmax = int(offs[1,1] + centers[c,2]);
if zmax > isize[2]:
zmax = isize[2];
#print xmin, xmax, ymin, ymax, zmin, zmax
data = img[xmin:xmax, ymin:ymax, zmin:zmax];
intensities[c] = method(data);
if verbose:
out.write(timer.elapsedTime(head = 'Cell Intensities'));
return intensities;
def greyReconstruction(img,
mask,
greyReconstructionParameter=None,
method=None,
size=3,
save=None,
verbose=False,
subStack=None,
out=sys.stdout,
**parameter):
"""Calculates the grey reconstruction of the image
Reconstruction is done z-slice by z-slice.
Arguments:
img (array): image data
removeBackGroundParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*method* (tuple or None) 'dilation' or 'erosion', if None return original image
*size* (int or tuple) size of structuring element
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: grey reconstructed image
"""
method = getParameter(greyReconstructionParameter, "method", method)
size = getParameter(greyReconstructionParameter, "size", size)
save = getParameter(greyReconstructionParameter, "save", save)
verbose = getParameter(greyReconstructionParameter, "verbose", verbose)
if verbose:
writeParameter(out=out,
head='Grey reconstruction:',
method=method,
size=size,
save=save)
if method is None:
return img
timer = Timer()
# background subtraction in each slice
se = structureElement('Disk', size).astype('uint8')
for z in range(img.shape[2]):
#img[:,:,z] = img[:,:,z] - grey_opening(img[:,:,z], structure = structureElement('Disk', (30,30)));
#img[:,:,z] = img[:,:,z] - morph.grey_opening(img[:,:,z], structure = self.structureELement('Disk', (150,150)));
img[:, :, z] = img[:, :, z] - reconstruct(
img[:, :, z], method=method, selem=se)
if not save is None:
writeSubStack(save, img, subStack=subStack)
if verbose > 1:
plotTiling(img)
if verbose:
out.write(timer.elapsedTime(head='Grey reconstruction:') + '\n')
return img
def calculateStatistics(source,
sink=None,
calculateStatisticsParameter=None,
method="Max",
remove=True,
processMethod=all,
verbose=False,
**parameter):
"""Calculate statisticsfrom image data
This is a main script to start extracting statistics of volumetric image data.
Arguments:
source (str or array): Image source
sink (str or None): destination for the results
calculateStatisticsParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*method* (str or function) function to extract statistic, must be trivially distributable
if None, do not extract information
*remove* (bool) remove redundant overlap
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
method (str or function):
processMethod (str or all): 'sequential' or 'parallel'. if all its choosen automatically
verbose (bool): print info
**parameter (dict): parameter for the image processing sub-routines
Returns:
list of statistics
"""
timer = Timer()
method = getParameter(calculateStatisticsParameter, "method", method)
remove = getParameter(calculateStatisticsParameter, "remove", remove)
verbose = getParameter(calculateStatisticsParameter, "verbose", verbose)
# run segmentation
if remove:
parameter = joinParameter({"chunkOverlap": 0}, parameter)
if processMethod == 'sequential':
result = sequentiallyProcessStack(source,
sink=sink,
function=calculateStatisticsOnStack,
join=joinStatistics,
method=method,
remove=remove,
verbose=verbose,
**parameter)
elif processMethod is all or processMethod == 'parallel':
result = parallelProcessStack(source,
sink=sink,
function=calculateStatisticsOnStack,
join=joinStatistics,
method=method,
remove=remove,
verbose=verbose,
**parameter)
else:
raise RuntimeError("calculateStatistics: invalid processMethod %s" %
str(processMethod))
if verbose:
timer.printElapsedTime("Total Time Image Statistics")
return result
def detectCellShape(img,
peaks,
detectCellShapeParameter=None,
threshold=None,
save=None,
verbose=False,
subStack=None,
out=sys.stdout,
**parameter):
"""Find cell shapes as labeled image
Arguments:
img (array): image data
peaks (array): point data of cell centers / seeds
detectCellShape (dict):
============ =================== ===========================================================
Name Type Descritption
============ =================== ===========================================================
*threshold* (float or None) threshold to determine mask, pixel below this are background
if None no mask is generated
*save* (tuple) size of the box on which to perform the *method*
*verbose* (bool or int) print / plot information about this step
============ =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: labeled image where each label indicates a cell
"""
threshold = getParameter(detectCellShapeParameter, "threshold", threshold)
save = getParameter(detectCellShapeParameter, "save", save)
verbose = getParameter(detectCellShapeParameter, "verbose", verbose)
if verbose:
writeParameter(out=out,
head='Cell shape detection:',
threshold=threshold,
save=save)
# extended maxima
timer = Timer()
if threshold is None:
imgmask = None
else:
imgmask = img > threshold
imgpeaks = voxelizePixel(peaks,
dataSize=img.shape,
weights=numpy.arange(1, peaks.shape[0] + 1))
imgws = watershed(-img, imgpeaks, mask=imgmask)
#imgws = watershed_ift(-img.astype('uint16'), imgpeaks);
#imgws[numpy.logical_not(imgmask)] = 0;
if not save is None:
writeSubStack(save, imgws.astype('int32'), subStack=subStack)
if verbose > 1:
#plotTiling(img)
plotOverlayLabel(img * 0.01, imgws, alpha=False)
#plotOverlayLabel(img, imgmax.astype('int64'), alpha = True)
if verbose:
out.write(timer.elapsedTime(head='Cell Shape:') + '\n')
return imgws
def classifyCells(img, classifyCellsParameter = None, classifier = None, classindex = 0, save = None, verbose = False,
detectCellShapeParameter = None,
subStack = None, out = sys.stdout, **parameter):
"""Detect Cells Using a trained classifier in Ilastik
The routine assumes that the first class is identifying the cells.
Arguments:
img (array): image data
classifyPixelParameter (dict):
============ ==================== ===========================================================
Name Type Descritption
============ ==================== ===========================================================
*classifier* (str or None) Ilastik project file with trained pixel classifier
*classindex* (int) class index considered to be cells
*save* (str or None) save the detected cell pixel to a file
*verbose* (bool or int) print / plot information about this step
============ ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
tuple: centers of the cells, intensity measurments
Note:
The routine could be potentially refined to make use of background
detection in ilastik
"""
classifier = getParameter(classifyCellsParameter, "classifier", classifier);
classindex = getParameter(classifyCellsParameter, "classindex", classindex);
save = getParameter(classifyCellsParameter, "save", save);
verbose = getParameter(classifyCellsParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Ilastik cell detection:', classifier = classifier, classindex = classindex, save = save);
timer = Timer();
ilastik.isInitialized();
#remove background
#img = removeBackground(img, verbose = verbose, out = out, **parameter);
#classify image / assume class 1 are the cells !
timer = Timer();
imgmax = ilastik.classifyPixel(classifier, img);
#print imgmax.shape
#max probability gives final class, last axis is class axis
imgmax = numpy.argmax(imgmax, axis = -1);
if save:
writeSubStack(save, numpy.asarray(imgmax, dtype = 'float32'), subStack = subStack)
# class 0 is used as cells
imgmax = imgmax == classindex; # class 1 is used as cells
imgshape, nlab = sm.label(imgmax);
if verbose > 1:
plotTiling(imgmax);
#center of maxima
centers = findCenterOfMaxima(img, imgmax, imgshape, verbose = verbose, out = out, **parameter);
#intensity of cells
#cintensity = findIntensity(img, centers, verbose = verbose, out = out, **parameter);
#intensity of cells in filtered image
#cintensity2 = findIntensity(img, centers, verbose = verbose, out = out, **parameter);
#if verbose:
# out.write(timer.elapsedTime(head = 'Ilastik cell detection') + '\n');
#return ( centers, numpy.vstack((cintensity, cintensity2)).transpose() );
#return ( centers, cintensity );
#cell size detection
#detectCellShapeParameter = getParameter(classifyCellsParameter, "detectCellShapeParameter", detectCellShapeParameter);
#cellShapeThreshold = getParameter(detectCellShapeParameter, "threshold", None);
#if not cellShapeThreshold is None:
# cell shape via watershed
#imgshape = detectCellShape(img, centers, detectCellShapeParameter = detectCellShapeParameter, verbose = verbose, out = out, **parameter);
#size of cells
csize = findCellSize(imgshape, maxLabel = centers.shape[0], out = out, **parameter);
#intensity of cells
cintensity = findCellIntensity(img, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
#intensity of cells in background image
#cintensity2 = findCellIntensity(img2, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
#intensity of cells in dog filtered image
#if dogSize is None:
# cintensity3 = cintensity2;
#else:
# cintensity3 = findCellIntensity(img3, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
if verbose:
out.write(timer.elapsedTime(head = 'Ilastik Cell Detection') + '\n');
#remove cell;s of size 0
idz = csize > 0;
#return ( centers[idz], numpy.vstack((cintensity[idz], cintensity3[idz], cintensity2[idz], csize[idz])).transpose());
return ( centers[idz], numpy.vstack((cintensity[idz], csize[idz])).transpose() );
def calculateStatisticsOnStack(img,
calculateStatisticsParameter=None,
method='Max',
remove=True,
verbose=False,
subStack=None,
out=sys.stdout,
**parameter):
"""Calculate a statistics from a large volumetric image
The statistics is assumed to be trivially distributable, i.e. max or mean.
Arguments:
img (array): image data
calculateStatisticsParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*method* (str or function) function to extract statistic, must be trivially distributable
if None, do not extract information
*remove* (bool) remove redundant overlap
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array or number: extracted statistics
Note:
One might need to choose zero overlap in the stacks to function properly!
"""
method = getParameter(calculateStatisticsParameter, "method", method)
remove = getParameter(calculateStatisticsParameter, "remove", remove)
verbose = getParameter(calculateStatisticsParameter, "verbose", verbose)
if verbose:
writeParameter(out=out, head='Image Statistics:', method=method)
if method is None:
return None
timer = Timer()
if not isinstance(method, list):
method = [method]
s = []
for m in range(len(method)):
f = _methodToFunction(method[m])
if remove and not subStack is None:
img = writeSubStack(None, img, subStack=subStack)
#print img.shape
s.append(f(img))
if verbose:
out.write(timer.elapsedTime(head='Image Statistics:') + '\n')
return s
def detectSpots(img,
detectSpotsParameter=None,
correctIlluminationParameter=None,
removeBackgroundParameter=None,
filterDoGParameter=None,
findExtendedMaximaParameter=None,
detectCellShapeParameter=None,
verbose=False,
out=sys.stdout,
**parameter):
"""Detect Cells in 3d grayscale image using DoG filtering and maxima detection
Effectively this function performs the following steps:
* illumination correction via :func:`~ClearMap.ImageProcessing.IlluminationCorrection.correctIllumination`
* background removal via :func:`~ClearMap.ImageProcessing.BackgroundRemoval.removeBackground`
* difference of Gaussians (DoG) filter via :func:`~ClearMap.ImageProcessing.Filter.filterDoG`
* maxima detection via :func:`~ClearMap.ImageProcessing.MaximaDetection.findExtendedMaxima`
* cell shape detection via :func:`~ClearMap.ImageProcessing.CellSizeDetection.detectCellShape`
* cell intensity and size measurements via: :func:`~ClearMap.ImageProcessing.CellSizeDetection.findCellIntensity`,
:func:`~ClearMap.ImageProcessing.CellSizeDetection.findCellSize`.
Note:
Processing steps are done in place to save memory.
Arguments:
img (array): image data
detectSpotParameter: image processing parameter as described in the individual sub-routines
verbose (bool): print progress information
out (object): object to print progress information to
Returns:
tuple: tuple of arrays (cell coordinates, raw intensity, fully filtered intensty, illumination and background corrected intensity [, cell size])
"""
timer = Timer()
# correct illumination
correctIlluminationParameter = getParameter(
detectSpotsParameter, "correctIlluminationParameter",
correctIlluminationParameter)
img1 = img.copy()
img1 = correctIllumination(
img1,
correctIlluminationParameter=correctIlluminationParameter,
verbose=verbose,
out=out,
**parameter)
# background subtraction in each slice
#img2 = img.copy();
removeBackgroundParameter = getParameter(detectSpotsParameter,
"removeBackgroundParameter",
removeBackgroundParameter)
img2 = removeBackground(
img1,
removeBackgroundParameter=removeBackgroundParameter,
verbose=verbose,
out=out,
**parameter)
#DoG filter
filterDoGParameter = getParameter(detectSpotsParameter,
"filterDoGParameter", filterDoGParameter)
dogSize = getParameter(filterDoGParameter, "size", None)
#img3 = img2.copy();
img3 = filterDoG(img2,
filterDoGParameter=filterDoGParameter,
verbose=verbose,
out=out,
**parameter)
# extended maxima
findExtendedMaximaParameter = getParameter(detectSpotsParameter,
"findExtendedMaximaParameter",
findExtendedMaximaParameter)
hMax = getParameter(findExtendedMaximaParameter, "hMax", None)
imgmax = findExtendedMaxima(
img3,
findExtendedMaximaParameter=findExtendedMaximaParameter,
verbose=verbose,
out=out,
**parameter)
#center of maxima
if not hMax is None:
centers = findCenterOfMaxima(img,
imgmax,
verbose=verbose,
out=out,
**parameter)
else:
centers = findPixelCoordinates(imgmax,
verbose=verbose,
out=out,
**parameter)
#cell size detection
detectCellShapeParameter = getParameter(detectSpotsParameter,
"detectCellShapeParameter",
detectCellShapeParameter)
cellShapeThreshold = getParameter(detectCellShapeParameter, "threshold",
None)
if not cellShapeThreshold is None:
# cell shape via watershed
imgshape = detectCellShape(
img2,
centers,
detectCellShapeParameter=detectCellShapeParameter,
verbose=verbose,
out=out,
**parameter)
#size of cells
csize = findCellSize(imgshape,
maxLabel=centers.shape[0],
out=out,
**parameter)
#intensity of cells
cintensity = findCellIntensity(img,
imgshape,
maxLabel=centers.shape[0],
verbose=verbose,
out=out,
**parameter)
#intensity of cells in background image
cintensity2 = findCellIntensity(img2,
imgshape,
maxLabel=centers.shape[0],
verbose=verbose,
out=out,
**parameter)
#intensity of cells in dog filtered image
if dogSize is None:
cintensity3 = cintensity2
else:
cintensity3 = findCellIntensity(img3,
imgshape,
maxLabel=centers.shape[0],
verbose=verbose,
out=out,
**parameter)
if verbose:
out.write(timer.elapsedTime(head="Spot Detection") + "\n")
#remove cell;s of size 0
idz = csize > 0
return (centers[idz],
numpy.vstack((cintensity[idz], cintensity3[idz],
cintensity2[idz], csize[idz])).transpose())
else:
#intensity of cells
cintensity = findIntensity(img,
centers,
verbose=verbose,
out=out,
**parameter)
#intensity of cells in background image
cintensity2 = findIntensity(img2,
centers,
verbose=verbose,
out=out,
**parameter)
#intensity of cells in dog filtered image
if dogSize is None:
cintensity3 = cintensity2
else:
cintensity3 = findIntensity(img3,
centers,
verbose=verbose,
out=out,
**parameter)
if verbose:
out.write(timer.elapsedTime(head="Spot Detection") + "\n")
return (centers, numpy.vstack(
(cintensity, cintensity3, cintensity2)).transpose())
def correctIllumination(img, correctIlluminationParameter = None, flatfield = None, background = None, scaling = None, save = None, verbose = False,
subStack = None, out = sys.stdout, **parameter):
"""Correct illumination variations
The intensity image :math:`I(x)` given a flat field :math:`F(x)` and
a background :math:`B(x)` the image is corrected to :math:`C(x)` as:
.. math:
C(x) = \\frac{I(x) - B(x)}{F(x) - B(x)}
If the background is not given :math:`B(x) = 0`.
The correction is done slice by slice assuming the data was collected with
a light sheet microscope.
The image is finally optionally scaled.
Arguments:
img (array): image data
findCenterOfMaximaParameter (dict):
============ ==================== ===========================================================
Name Type Descritption
============ ==================== ===========================================================
*flatfield* (str, None or array) flat field intensities, if None d onot correct image for
illumination, if True the
*background* (str, None or array) background image as file name or array
if None background is assumed to be zero
*scaling* (str or None) scale the corrected result by this factor
if 'max'/'mean' scale to keep max/mean invariant
*save* (str or None) save the corrected image to file
*verbose* (bool or int) print / plot information about this step
============ ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: illumination corrected image
References:
Fundamentals of Light Microscopy and Electronic Imaging, p 421
See Also:
:const:`DefaultFlatFieldLineFile`
"""
flatfield = getParameter(correctIlluminationParameter, "flatfield", flatfield);
background = getParameter(correctIlluminationParameter, "background", background);
scaling = getParameter(correctIlluminationParameter, "scaling", scaling);
save = getParameter(correctIlluminationParameter, "save", save);
verbose = getParameter(correctIlluminationParameter, "verbose", verbose);
if verbose:
if flatfield is None or isinstance(flatfield, str) or flatfield is True:
fld = flatfield;
else:
fld = "image of size %s" % str(flatfield.shape);
if background is None or isinstance(background, str):
bkg = background;
else:
bkg = "image of size %s" % str(background.shape);
writeParameter(out = out, head = 'Illumination correction:', flatfield = fld, background = bkg, scaling = scaling, save = save);
print subStack;
if not subStack is None:
x = subStack["x"];
y = subStack["y"];
else:
x = all;
y = all;
#print "sizes", x, y, img.shape
#read data
timer = Timer();
if flatfield is None:
return img;
elif flatfield is True:
# default flatfield correction
if subStack is None:
flatfield = flatfieldFromLine(DefaultFlatFieldLineFile, img.shape[0]);
else:
dataSize = io.dataSize(subStack["source"]);
flatfield = flatfieldFromLine(DefaultFlatFieldLineFile, dataSize[0]);
elif isinstance(flatfield, str):
# point or image file
if io.isPointFile(flatfield):
if subStack is None:
flatfield = flatfieldFromLine(flatfield, img.shape[0]);
else:
dataSize = io.dataSize(subStack["source"]);
flatfield = flatfieldFromLine(flatfield, dataSize[0]);
else:
flatfield = io.readData(flatfield);
ffmean = flatfield.mean();
ffmax = flatfield.max();
#correct for subset
flatfield = io.readData(flatfield, x = x, y = y);
background = io.readData(background, x = x, y = y);
if flatfield.shape != img[:,:,0].shape:
raise RuntimeError("correctIllumination: flatfield does not match image size: %s vs %s" % (flatfield.shape, img[:,:,0].shape));
#convert to float for scaling
dtype = img.dtype;
img = img.astype('float32');
flatfield = flatfield.astype('float32');
# illumination correction in each slice
if background is None:
for z in range(img.shape[2]):
img[:,:,z] = img[:,:,z] / flatfield;
else:
if background.shape != flatfield.shape:
raise RuntimeError("correctIllumination: background does not match image size: %s vs %s" % (background.shape, img[:,:,0].shape));
background = background.astype('float32');
flatfield = (flatfield - background);
for z in range(img.shape[2]):
img[:,:,z] = (img[:,:,z] - background) / flatfield;
# rescale
if scaling is True:
scaling = "mean";
if isinstance(scaling, str):
if scaling.lower() == "mean":
# scale back by average flat field correction:
sf = ffmean;
elif scaling.lower() == "max":
sf = ffmax;
else:
raise RuntimeError('Scaling not "Max" or "Mean" but %s' % scaling);
else:
sf = scaling;
if verbose:
writeParameter(out = out, head = 'Illumination correction:', scaling = sf);
if not sf is None:
img = img * sf;
img = img.astype(dtype);
#write result for inspection
if not save is None:
writeSubStack(save, img, subStack = subStack);
#plot result for inspection
if verbose > 1:
plotTiling(img);
if verbose:
out.write(timer.elapsedTime(head = 'Illumination correction') + '\n');
return img
def calculateStatistics(
source,
sink=None,
calculateStatisticsParameter=None,
method="Max",
remove=True,
processMethod=all,
verbose=False,
**parameter
):
"""Calculate statisticsfrom image data
This is a main script to start extracting statistics of volumetric image data.
Arguments:
source (str or array): Image source
sink (str or None): destination for the results
calculateStatisticsParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*method* (str or function) function to extract statistic, must be trivially distributable
if None, do not extract information
*remove* (bool) remove redundant overlap
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
method (str or function):
processMethod (str or all): 'sequential' or 'parallel'. if all its choosen automatically
verbose (bool): print info
**parameter (dict): parameter for the image processing sub-routines
Returns:
list of statistics
"""
timer = Timer()
method = getParameter(calculateStatisticsParameter, "method", method)
remove = getParameter(calculateStatisticsParameter, "remove", remove)
verbose = getParameter(calculateStatisticsParameter, "verbose", verbose)
# run segmentation
if remove:
parameter = joinParameter({"chunkOverlap": 0}, parameter)
if processMethod == "sequential":
result = sequentiallyProcessStack(
source,
sink=sink,
function=calculateStatisticsOnStack,
join=joinStatistics,
method=method,
remove=remove,
verbose=verbose,
**parameter
)
elif processMethod is all or processMethod == "parallel":
result = parallelProcessStack(
source,
sink=sink,
function=calculateStatisticsOnStack,
join=joinStatistics,
method=method,
remove=remove,
verbose=verbose,
**parameter
)
else:
raise RuntimeError("calculateStatistics: invalid processMethod %s" % str(processMethod))
if verbose:
timer.printElapsedTime("Total Time Image Statistics")
return result
def calculateStatisticsOnStack(
img,
calculateStatisticsParameter=None,
method="Max",
remove=True,
verbose=False,
subStack=None,
out=sys.stdout,
**parameter
):
"""Calculate a statistics from a large volumetric image
The statistics is assumed to be trivially distributable, i.e. max or mean.
Arguments:
img (array): image data
calculateStatisticsParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*method* (str or function) function to extract statistic, must be trivially distributable
if None, do not extract information
*remove* (bool) remove redundant overlap
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array or number: extracted statistics
Note:
One might need to choose zero overlap in the stacks to function properly!
"""
method = getParameter(calculateStatisticsParameter, "method", method)
remove = getParameter(calculateStatisticsParameter, "remove", remove)
verbose = getParameter(calculateStatisticsParameter, "verbose", verbose)
if verbose:
writeParameter(out=out, head="Image Statistics:", method=method)
if method is None:
return None
timer = Timer()
if not isinstance(method, list):
method = [method]
s = []
for m in range(len(method)):
f = _methodToFunction(method[m])
if remove and not subStack is None:
img = writeSubStack(None, img, subStack=subStack)
# print img.shape
s.append(f(img))
if verbose:
out.write(timer.elapsedTime(head="Image Statistics:") + "\n")
return s
def filterLinear(img,
filterLinearParameter=None,
ftype=None,
size=None,
sigma=None,
sigma2=None,
save=None,
subStack=None,
verbose=False,
out=sys.stdout,
**parameter):
"""Applies a linear filter to the image
Arguments:
img (array): image data
filterLinearParameter (dict):
========= ==================== ================================================================
Name Type Descritption
========= ==================== ================================================================
*ftype* (str or None) the type of the filter, see :ref:`FilterTypes`
if None do ot perform any fitlering
*size* (tuple or None) size for the filter
if None, do not perform filtering
*sigma* (tuple or None) std of outer Guassian, if None autmatically determined from size
*sigma2* (tuple or None) std of inner Guassian, if None autmatically determined from size
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print progress information
========= ==================== ================================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: filtered image
Note:
Converts image to float32 type if filter is active!
"""
timer = Timer()
ftype = getParameter(filterLinearParameter, "ftype", ftype)
size = getParameter(filterLinearParameter, "size", size)
sigma = getParameter(filterLinearParameter, "sigma", sigma)
sigma2 = getParameter(filterLinearParameter, "sigma2", sigma2)
save = getParameter(filterLinearParameter, "save", save)
verbose = getParameter(filterLinearParameter, "verbose", verbose)
if verbose:
writeParameter(out=out,
head='Linear Filter:',
ftype=ftype,
size=size,
sigma=sigma,
sigma2=sigma2,
save=save)
if ftype is None:
return img
#DoG filter
img = img.astype('float32')
# always convert to float for downstream processing
if not size is None:
fil = filterKernel(ftype=ftype, size=size, sigma=sigma, sigma2=sigma2)
fil = fil.astype('float32')
#img = correlate(img, fdog);
#img = scipy.signal.correlate(img, fdog);
img = correlate(img, fil)
#img = convolve(img, fdog, mode = 'same');
img[img < 0] = 0
if verbose > 1:
plotTiling(img)
if not save is None:
writeSubStack(save, img, subStack=subStack)
if verbose:
out.write(timer.elapsedTime(head='Linear Filter') + '\n')
return img
def findExtendedMaxima(img, findExtendedMaximaParameter = None, hMax = None, size = 5, threshold = None, save = None, verbose = None,
subStack = None, out = sys.stdout, **parameter):
"""Find extended maxima in an image
Effectively this routine performs a h-max transfrom, followed by a local maxima search and
thresholding of the maxima.
Arguments:
img (array): image data
findExtendedMaximaParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*hMax* (float or None) h parameter for the initial h-Max transform
if None, do not perform a h-max transform
*size* (tuple) size for the structure element for the local maxima filter
*threshold* (float or None) include only maxima larger than a threshold
if None keep all localmaxima
*save* (str or None) file name to save result of this operation
if None do not save result to file
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: binary image with True pixel at extended maxima
See Also:
:func:`hMaxTransform`, :func:`localMax`
"""
hMax = getParameter(findExtendedMaximaParameter, "hMax", hMax);
size = getParameter(findExtendedMaximaParameter, "size", size);
threshold = getParameter(findExtendedMaximaParameter, "threshold", threshold);
save = getParameter(findExtendedMaximaParameter, "save", save);
verbose = getParameter(findExtendedMaximaParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Extended Max:', hMax = hMax, size = size, threshold = threshold, save = save);
timer = Timer();
## extended maxima
imgmax = hMaxTransform(img, hMax);
#imgmax = regionalMax(imgmax, regionalMaxStructureElement);
imgmax = localMax(imgmax, size);
#thresholding
if not threshold is None:
imgmax = numpy.logical_and(imgmax, img >= threshold);
if verbose > 1:
#plotTiling(img)
plotOverlayLabel(img * 0.01, imgmax.astype('int64'), alpha = False);
#plotOverlayLabel(img, imgmax.astype('int64'), alpha = True)
if not save is None:#
writeSubStack(save, imgmax.astype('int8'), subStack = subStack)
if verbose:
out.write(timer.elapsedTime(head = 'Extended Max') + '\n');
return imgmax
def findCellIntensity(img,
imglabel,
findCellIntensityParameter=None,
maxLabel=None,
method='Sum',
verbose=False,
out=sys.stdout,
**parameter):
"""Find integrated cell intensity given cell shapes as labled image
Arguments:
img (array or str): image data
imglabel (array or str): labeled image, where each cell has its own label
findCellIntensityParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*maxLabel* (int or None) maximal label to include, if None determine automatically
*method* (str) method to use for measurment: 'Sum', 'Mean', 'Max', 'Min'
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: measured intensities
"""
maxLabel = getParameter(findCellIntensityParameter, "maxLabel", maxLabel)
method = getParameter(findCellIntensityParameter, "method", method)
verbose = getParameter(findCellIntensityParameter, "verbose", verbose)
if verbose:
writeParameter(out=out,
head='Cell intensity detection:',
method=method,
maxLabel=maxLabel)
timer = Timer()
if maxLabel is None:
maxLabel = imglabel.max()
if method.lower() == 'sum':
i = scipy.ndimage.measurements.sum(img,
labels=imglabel,
index=numpy.arange(1, maxLabel + 1))
elif method.lower() == 'mean':
i = scipy.ndimage.measurements.mean(img,
labels=imglabel,
index=numpy.arange(
1, maxLabel + 1))
elif method.lower() == 'max':
i = scipy.ndimage.measurements.maximum(img,
labels=imglabel,
index=numpy.arange(
1, maxLabel + 1))
elif method.lower() == 'min':
i = scipy.ndimage.measurements.minimum(img,
labels=imglabel,
index=numpy.arange(
1, maxLabel + 1))
else:
raise RuntimeError('cellIntensity: unkown method %s!' % method)
if verbose:
out.write(timer.elapsedTime(head='Cell intensity detection:') + '\n')
return i
def classifyCells(img, classifyCellsParameter = None, classifier = None, classindex = 0, save = None, verbose = False,
detectCellShapeParameter = None,
subStack = None, out = sys.stdout, **parameter):
"""Detect Cells Using a trained classifier in Ilastik
The routine assumes that the first class is identifying the cells.
Arguments:
img (array): image data
classifyPixelParameter (dict):
============ ==================== ===========================================================
Name Type Descritption
============ ==================== ===========================================================
*classifier* (str or None) Ilastik project file with trained pixel classifier
*classindex* (int) class index considered to be cells
*save* (str or None) save the detected cell pixel to a file
*verbose* (bool or int) print / plot information about this step
============ ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
tuple: centers of the cells, intensity measurments
Note:
The routine could be potentially refined to make use of background
detection in ilastik
"""
classifier = getParameter(classifyCellsParameter, "classifier", classifier);
classindex = getParameter(classifyCellsParameter, "classindex", classindex);
save = getParameter(classifyCellsParameter, "save", save);
verbose = getParameter(classifyCellsParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Ilastik cell detection:', classifier = classifier, classindex = classindex, save = save);
timer = Timer();
ilastik.isInitialized();
#remove background
#img = removeBackground(img, verbose = verbose, out = out, **parameter);
#classify image / assume class 1 are the cells !
timer = Timer();
imgmax = ilastik.classifyPixel(classifier, img);
#print imgmax.shape
#max probability gives final class, last axis is class axis
imgmax = numpy.argmax(imgmax, axis = -1);
if save:
writeSubStack(save, numpy.asarray(imgmax, dtype = 'float32'), subStack = subStack)
# class 0 is used as cells
imgmax = imgmax == classindex; # class 1 is used as cells
imgshape, nlab = sm.label(imgmax);
if verbose > 1:
plotTiling(imgmax);
#center of maxima
centers = findCenterOfMaxima(img, imgmax, imgshape, verbose = verbose, out = out, **parameter);
#intensity of cells
#cintensity = findIntensity(img, centers, verbose = verbose, out = out, **parameter);
#intensity of cells in filtered image
#cintensity2 = findIntensity(img, centers, verbose = verbose, out = out, **parameter);
#if verbose:
# out.write(timer.elapsedTime(head = 'Ilastik cell detection') + '\n');
#return ( centers, numpy.vstack((cintensity, cintensity2)).transpose() );
#return ( centers, cintensity );
#cell size detection
#detectCellShapeParameter = getParameter(classifyCellsParameter, "detectCellShapeParameter", detectCellShapeParameter);
#cellShapeThreshold = getParameter(detectCellShapeParameter, "threshold", None);
#if not cellShapeThreshold is None:
# cell shape via watershed
#imgshape = detectCellShape(img, centers, detectCellShapeParameter = detectCellShapeParameter, verbose = verbose, out = out, **parameter);
#size of cells
csize = findCellSize(imgshape, maxLabel = centers.shape[0], out = out, **parameter);
#intensity of cells
cintensity = findCellIntensity(img, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
#intensity of cells in background image
#cintensity2 = findCellIntensity(img2, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
#intensity of cells in dog filtered image
#if dogSize is None:
# cintensity3 = cintensity2;
#else:
# cintensity3 = findCellIntensity(img3, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
if verbose:
out.write(timer.elapsedTime(head = 'Ilastik Cell Detection') + '\n');
#remove cell;s of size 0
idz = csize > 0;
#return ( centers[idz], numpy.vstack((cintensity[idz], cintensity3[idz], cintensity2[idz], csize[idz])).transpose());
return ( centers[idz], numpy.vstack((cintensity[idz], csize[idz])).transpose() );
def findCenterOfMaxima(img, imgmax = None, label = None, findCenterOfMaximaParameter = None, save = None, verbose = False,
subStack = None, out = sys.stdout, **parameter):
"""Find center of detected maxima weighted by intensity
Arguments:
img (array): image data
findCenterOfMaximaParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*save* (str or None) saves result of labeling the differnet maxima
if None, do the lableling is not saved
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: coordinates of centers of maxima, shape is (n,d) where n is number of maxima and d the dimension of the image
"""
save = getParameter(findCenterOfMaximaParameter, "save", save);
verbose = getParameter(findCenterOfMaximaParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Center of Maxima:', save = save);
timer = Timer();
#center of maxima
if label is None:
imglab, nlab = sm.label(imgmax);
else:
imglab = label;
nlab = imglab.max();
#print 'max', imglab.shape, img.shape
#print imglab.dtype, img.dtype
if not save is None:
writeSubStack(save, imglab, subStack = subStack);
if nlab > 0:
centers = numpy.array(sm.center_of_mass(img, imglab, index = numpy.arange(1, nlab)));
if verbose > 1:
#plotOverlayLabel(img * 0.01, imglab, alpha = False);
#plotTiling(img)
imgc = numpy.zeros(img.shape);
for i in range(centers.shape[0]):
imgc[centers[i,0], centers[i,1], centers[i,2]] = 1;
plotOverlayLabel(img, imgc, alpha = False);
#plotOverlayLabel(img, imgmax.astype('int64'), alpha = True)
#return centers, imglab, mask
#cintensity = numpy.array([img[centers[i,0], centers[i,1], centers[i,2]] for i in range(centers.shape[0])]);
if verbose:
out.write(timer.elapsedTime(head = 'Cell Centers'));
#return ( centers, cintensity );
return centers;
else:
if verbose:
out.write('Cell Centers: No Cells found !');
#return ( numpy.zeros((0,3)), numpy.zeros(0) );
#return empty set o coordinates
return numpy.zeros((0,3));
def findIntensity(img, centers, findIntensityParameter = None, method = None, size = (3,3,3), verbose = False,
out = sys.stdout, **parameter):
"""Find instensity value around centers in the image
Arguments:
img (array): image data
findIntensityParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*method* (str, func, None) method to use to determine intensity (e.g. "Max" or "Mean")
if None take intensities at the given pixels
*size* (tuple) size of the box on which to perform the *method*
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: measured intensities
"""
method = getParameter(findIntensityParameter, "method", "Max");
size = getParameter(findIntensityParameter, "size", (3,3,3));
verbose = getParameter(findIntensityParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Cell Intensities:', method = method, size = size);
timer = Timer();
if centers.shape[0] == 0:
return numpy.zeros(0);
if method is None:
return numpy.array([img[centers[i,0], centers[i,1], centers[i,2]] for i in range(centers.shape[0])]);
isize = img.shape;
#print isize
offs = structureElementOffsets(size);
if isinstance(method, basestring):
method = eval('numpy.' + method.lower());
intensities = numpy.zeros(centers.shape[0], dtype = img.dtype);
for c in range(centers.shape[0]):
xmin = int(-offs[0,0] + centers[c,0]);
if xmin < 0:
xmin = 0;
xmax = int(offs[0,1] + centers[c,0]);
if xmax > isize[0]:
xmax = isize[0];
ymin = int(-offs[1,0] + centers[c,1]);
if ymin < 0:
ymin = 0;
ymax = int(offs[1,1] + centers[c,1]);
if ymax > isize[1]:
ymax = isize[1];
zmin = int(-offs[2,0] + centers[c,2]);
if zmin < 0:
zmin = 0;
zmax = int(offs[1,1] + centers[c,2]);
if zmax > isize[2]:
zmax = isize[2];
#print xmin, xmax, ymin, ymax, zmin, zmax
data = img[xmin:xmax, ymin:ymax, zmin:zmax];
intensities[c] = method(data);
if verbose:
out.write(timer.elapsedTime(head = 'Cell Intensities'));
return intensities;
def findExtendedMaxima(img, findExtendedMaximaParameter = None, hMax = None, size = 5, threshold = None, save = None, verbose = None,
subStack = None, out = sys.stdout, **parameter):
"""Find extended maxima in an image
Effectively this routine performs a h-max transfrom, followed by a local maxima search and
thresholding of the maxima.
Arguments:
img (array): image data
findExtendedMaximaParameter (dict):
=========== =================== ===========================================================
Name Type Descritption
=========== =================== ===========================================================
*hMax* (float or None) h parameter for the initial h-Max transform
if None, do not perform a h-max transform
*size* (tuple) size for the structure element for the local maxima filter
*threshold* (float or None) include only maxima larger than a threshold
if None keep all localmaxima
*save* (str or None) file name to save result of this operation
if None do not save result to file
*verbose* (bool or int) print / plot information about this step
=========== =================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: binary image with True pixel at extended maxima
See Also:
:func:`hMaxTransform`, :func:`localMax`
"""
hMax = getParameter(findExtendedMaximaParameter, "hMax", hMax);
size = getParameter(findExtendedMaximaParameter, "size", size);
threshold = getParameter(findExtendedMaximaParameter, "threshold", threshold);
save = getParameter(findExtendedMaximaParameter, "save", save);
verbose = getParameter(findExtendedMaximaParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Extended Max:', hMax = hMax, size = size, threshold = threshold, save = save);
timer = Timer();
## extended maxima
imgmax = hMaxTransform(img, hMax);
#imgmax = regionalMax(imgmax, regionalMaxStructureElement);
imgmax = localMax(imgmax, size);
#thresholding
if not threshold is None:
imgmax = numpy.logical_and(imgmax, img >= threshold);
if verbose > 1:
#plotTiling(img)
plotOverlayLabel(img * 0.01, imgmax.astype('int64'), alpha = False);
#plotOverlayLabel(img, imgmax.astype('int64'), alpha = True)
if not save is None:#
writeSubStack(save, imgmax.astype('int8'), subStack = subStack)
if verbose:
out.write(timer.elapsedTime(head = 'Extended Max') + '\n');
return imgmax
def findCenterOfMaxima(img, imgmax = None, label = None, findCenterOfMaximaParameter = None, save = None, verbose = False,
subStack = None, out = sys.stdout, **parameter):
"""Find center of detected maxima weighted by intensity
Arguments:
img (array): image data
findCenterOfMaximaParameter (dict):
========= ==================== ===========================================================
Name Type Descritption
========= ==================== ===========================================================
*save* (str or None) saves result of labeling the differnet maxima
if None, do the lableling is not saved
*verbose* (bool or int) print / plot information about this step
========= ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: coordinates of centers of maxima, shape is (n,d) where n is number of maxima and d the dimension of the image
"""
save = getParameter(findCenterOfMaximaParameter, "save", save);
verbose = getParameter(findCenterOfMaximaParameter, "verbose", verbose);
if verbose:
writeParameter(out = out, head = 'Center of Maxima:', save = save);
timer = Timer();
#center of maxima
if label is None:
imglab, nlab = sm.label(imgmax);
else:
imglab = label;
nlab = imglab.max();
#print 'max', imglab.shape, img.shape
#print imglab.dtype, img.dtype
if not save is None:
writeSubStack(save, imglab, subStack = subStack);
if nlab > 0:
centers = numpy.array(sm.center_of_mass(img, imglab, index = numpy.arange(1, nlab)));
if verbose > 1:
#plotOverlayLabel(img * 0.01, imglab, alpha = False);
#plotTiling(img)
imgc = numpy.zeros(img.shape);
for i in range(centers.shape[0]):
imgc[centers[i,0], centers[i,1], centers[i,2]] = 1;
plotOverlayLabel(img, imgc, alpha = False);
#plotOverlayLabel(img, imgmax.astype('int64'), alpha = True)
#return centers, imglab, mask
#cintensity = numpy.array([img[centers[i,0], centers[i,1], centers[i,2]] for i in range(centers.shape[0])]);
if verbose:
out.write(timer.elapsedTime(head = 'Cell Centers'));
#return ( centers, cintensity );
return centers;
else:
if verbose:
out.write('Cell Centers: No Cells found !');
#return ( numpy.zeros((0,3)), numpy.zeros(0) );
#return empty set o coordinates
return numpy.zeros((0,3));
def detectSpots(img,
detectSpotsParameter=None,
correctIlluminationParameter=None,
removeBackgroundParameter=None,
filterDoGParameter=None,
findExtendedMaximaParameter=None,
detectCellShapeParameter=None,
verbose=False,
out=sys.stdout,
**parameter):
"""Detect Cells in 3d grayscale image using DoG filtering and maxima detection
Effectively this function performs the following steps:
* illumination correction via :func:`~ClearMap.ImageProcessing.IlluminationCorrection.correctIllumination`
* background removal via :func:`~ClearMap.ImageProcessing.BackgroundRemoval.removeBackground`
* difference of Gaussians (DoG) filter via :func:`~ClearMap.ImageProcessing.Filter.filterDoG`
* maxima detection via :func:`~ClearMap.ImageProcessing.MaximaDetection.findExtendedMaxima`
* cell shape detection via :func:`~ClearMap.ImageProcessing.CellSizeDetection.detectCellShape`
* cell intensity and size measurements via: :func:`~ClearMap.ImageProcessing.CellSizeDetection.findCellIntensity`,
:func:`~ClearMap.ImageProcessing.CellSizeDetection.findCellSize`.
detectCells
Note:
Processing steps are done in place to save memory.
Arguments:
img (array): image data
detectSpotParameter: image processing parameter as described in the individual sub-routines
verbose (bool): print progress information
out (object): object to print progress information to
Returns:
tuple: tuple of arrays (cell coordinates, raw intensity, fully filtered intensty, illumination and background corrected intensity [, cell size])
"""
timer = Timer()
# normalize data -> to check
#img = img.astype('float');
#dmax = 0.075 * 65535;
#ids = img > dmax;
#img[ids] = dmax;
#img /= dmax;
#out.write(timer.elapsedTime(head = 'Normalization'));
#img = dataset[600:1000,1600:1800,800:830];
#img = dataset[600:1000,:,800:830];
# correct illumination
correctIlluminationParameter = getParameter(
detectSpotsParameter, "correctIlluminationParameter",
correctIlluminationParameter)
img1 = img.copy()
img1 = correctIllumination(
img1,
correctIlluminationParameter=correctIlluminationParameter,
verbose=verbose,
out=out,
**parameter)
# background subtraction in each slice
#img2 = img.copy();
removeBackgroundParameter = getParameter(detectSpotsParameter,
"removeBackgroundParameter",
removeBackgroundParameter)
img2 = removeBackground(
img1,
removeBackgroundParameter=removeBackgroundParameter,
verbose=verbose,
out=out,
**parameter)
# mask
#timer.reset();
#if mask == None: #explicit mask
# mask = img > 0.01;
# mask = binary_opening(mask, self.structureELement('Disk', (3,3,3)));
#img[img < 0.01] = 0; # masking in place # extended maxima
#out.write(timer.elapsedTime(head = 'Mask'));
#DoG filter
filterDoGParameter = getParameter(detectSpotsParameter,
"filterDoGParameter", filterDoGParameter)
dogSize = getParameter(filterDoGParameter, "size", None)
#img3 = img2.copy();
img3 = filterDoG(img2,
filterDoGParameter=filterDoGParameter,
verbose=verbose,
out=out,
**parameter)
# normalize
# imax = img.max();
# if imax == 0:
# imax = 1;
# img /= imax;
# extended maxima
findExtendedMaximaParameter = getParameter(detectSpotsParameter,
"findExtendedMaximaParameter",
findExtendedMaximaParameter)
hMax = getParameter(findExtendedMaximaParameter, "hMax", None)
imgmax = findExtendedMaxima(
img3,
findExtendedMaximaParameter=findExtendedMaximaParameter,
verbose=verbose,
out=out,
**parameter)
#center of maxima
if not hMax is None:
centers = findCenterOfMaxima(img,
imgmax,
verbose=verbose,
out=out,
**parameter)
else:
centers = findPixelCoordinates(imgmax,
verbose=verbose,
out=out,
**parameter)
#cell size detection
detectCellShapeParameter = getParameter(detectSpotsParameter,
"detectCellShapeParameter",
detectCellShapeParameter)
cellShapeThreshold = getParameter(detectCellShapeParameter, "threshold",
None)
if not cellShapeThreshold is None:
# cell shape via watershed
imgshape = detectCellShape(
img2,
centers,
detectCellShapeParameter=detectCellShapeParameter,
verbose=verbose,
out=out,
**parameter)
#size of cells
csize = findCellSize(imgshape,
maxLabel=centers.shape[0],
out=out,
**parameter)
#intensity of cells
cintensity = findCellIntensity(img,
imgshape,
maxLabel=centers.shape[0],
verbose=verbose,
out=out,
**parameter)
#intensity of cells in background image
cintensity2 = findCellIntensity(img2,
imgshape,
maxLabel=centers.shape[0],
verbose=verbose,
out=out,
**parameter)
#intensity of cells in dog filtered image
if dogSize is None:
cintensity3 = cintensity2
else:
cintensity3 = findCellIntensity(img3,
imgshape,
maxLabel=centers.shape[0],
verbose=verbose,
out=out,
**parameter)
if verbose:
out.write(timer.elapsedTime(head='Spot Detection') + '\n')
#remove cell;s of size 0
idz = csize > 0
return (centers[idz],
numpy.vstack((cintensity[idz], cintensity3[idz],
cintensity2[idz], csize[idz])).transpose())
else:
#intensity of cells
cintensity = findIntensity(img,
centers,
verbose=verbose,
out=out,
**parameter)
#intensity of cells in background image
cintensity2 = findIntensity(img2,
centers,
verbose=verbose,
out=out,
**parameter)
#intensity of cells in dog filtered image
if dogSize is None:
cintensity3 = cintensity2
else:
cintensity3 = findIntensity(img3,
centers,
verbose=verbose,
out=out,
**parameter)
if verbose:
out.write(timer.elapsedTime(head='Spot Detection') + '\n')
return (centers, numpy.vstack(
(cintensity, cintensity3, cintensity2)).transpose())
def correctIllumination(img,
correctIlluminationParameter=None,
flatfield=None,
background=None,
scaling=None,
save=None,
verbose=False,
subStack=None,
out=sys.stdout,
**parameter):
"""Correct illumination variations
The intensity image :math:`I(x)` given a flat field :math:`F(x)` and
a background :math:`B(x)` the image is corrected to :math:`C(x)` as:
.. math:
C(x) = \\frac{I(x) - B(x)}{F(x) - B(x)}
If the background is not given :math:`B(x) = 0`.
The correction is done slice by slice assuming the data was collected with
a light sheet microscope.
The image is finally optionally scaled.
Arguments:
img (array): image data
findCenterOfMaximaParameter (dict):
============ ==================== ===========================================================
Name Type Descritption
============ ==================== ===========================================================
*flatfield* (str, None or array) flat field intensities, if None d onot correct image for
illumination, if True the
*background* (str, None or array) background image as file name or array
if None background is assumed to be zero
*scaling* (str or None) scale the corrected result by this factor
if 'max'/'mean' scale to keep max/mean invariant
*save* (str or None) save the corrected image to file
*verbose* (bool or int) print / plot information about this step
============ ==================== ===========================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: illumination corrected image
References:
Fundamentals of Light Microscopy and Electronic Imaging, p 421
See Also:
:const:`DefaultFlatFieldLineFile`
"""
flatfield = getParameter(correctIlluminationParameter, "flatfield",
flatfield)
background = getParameter(correctIlluminationParameter, "background",
background)
scaling = getParameter(correctIlluminationParameter, "scaling", scaling)
save = getParameter(correctIlluminationParameter, "save", save)
verbose = getParameter(correctIlluminationParameter, "verbose", verbose)
if verbose:
if flatfield is None or isinstance(flatfield,
str) or flatfield is True:
fld = flatfield
else:
fld = "image of size %s" % str(flatfield.shape)
if background is None or isinstance(background, str):
bkg = background
else:
bkg = "image of size %s" % str(background.shape)
writeParameter(out=out,
head='Illumination correction:',
flatfield=fld,
background=bkg,
scaling=scaling,
save=save)
print subStack
if not subStack is None:
x = subStack["x"]
y = subStack["y"]
else:
x = all
y = all
#print "sizes", x, y, img.shape
#read data
timer = Timer()
if flatfield is None:
return img
elif flatfield is True:
# default flatfield correction
if subStack is None:
flatfield = flatfieldFromLine(DefaultFlatFieldLineFile,
img.shape[0])
else:
dataSize = io.dataSize(subStack["source"])
flatfield = flatfieldFromLine(DefaultFlatFieldLineFile,
dataSize[0])
elif isinstance(flatfield, str):
# point or image file
if io.isPointFile(flatfield):
if subStack is None:
flatfield = flatfieldFromLine(flatfield, img.shape[0])
else:
dataSize = io.dataSize(subStack["source"])
flatfield = flatfieldFromLine(flatfield, dataSize[0])
else:
flatfield = io.readData(flatfield)
ffmean = flatfield.mean()
ffmax = flatfield.max()
#correct for subset
flatfield = io.readData(flatfield, x=x, y=y)
background = io.readData(background, x=x, y=y)
if flatfield.shape != img[:, :, 0].shape:
raise RuntimeError(
"correctIllumination: flatfield does not match image size: %s vs %s"
% (flatfield.shape, img[:, :, 0].shape))
#convert to float for scaling
dtype = img.dtype
img = img.astype('float32')
flatfield = flatfield.astype('float32')
# illumination correction in each slice
if background is None:
for z in range(img.shape[2]):
img[:, :, z] = img[:, :, z] / flatfield
else:
if background.shape != flatfield.shape:
raise RuntimeError(
"correctIllumination: background does not match image size: %s vs %s"
% (background.shape, img[:, :, 0].shape))
background = background.astype('float32')
flatfield = (flatfield - background)
for z in range(img.shape[2]):
img[:, :, z] = (img[:, :, z] - background) / flatfield
# rescale
if scaling is True:
scaling = "mean"
if isinstance(scaling, str):
if scaling.lower() == "mean":
# scale back by average flat field correction:
sf = ffmean
elif scaling.lower() == "max":
sf = ffmax
else:
raise RuntimeError('Scaling not "Max" or "Mean" but %s' % scaling)
else:
sf = scaling
if verbose:
writeParameter(out=out, head='Illumination correction:', scaling=sf)
if not sf is None:
img = img * sf
img = img.astype(dtype)
#write result for inspection
if not save is None:
writeSubStack(save, img, subStack=subStack)
#plot result for inspection
if verbose > 1:
plotTiling(img)
if verbose:
out.write(timer.elapsedTime(head='Illumination correction') + '\n')
return img
def _test():
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.ParallelProcessing.DataProcessing.LargeData as ld
from importlib import reload
reload(ld)
#dat = np.random.rand(2000,2000,1000) > 0.5;
#dat = np.random.rand(1000,1000,500) > 0.5;
dat = np.random.rand(200, 300, 400) > 0.5
#datan = io.MMP.writeData('test.npy', dat);
dat = np.load('data.npy')
xyz1 = np.load('points.npy')
s = ld.sum(dat)
print(s == np.sum(s))
timer = Timer()
xyz = ld.where(dat)
timer.print_elapsed_time('parallel')
#parallel: elapsed time: 0:00:25.807
timer = Timer()
xyz1 = np.vstack(np.where(dat)).T
timer.print_elapsed_time('numpy')
#numpy: elapsed time: 0:05:45.590
d0 = np.zeros(dat.shape, dtype=bool)
d1 = np.zeros(dat.shape, dtype=bool)
d0[xyz[:, 0], xyz[:, 1], xyz[:, 2]] = True
d1[xyz1[:, 0], xyz1[:, 1], xyz1[:, 2]] = True
np.all(d0 == d1)
dat2 = np.array(np.random.rand(1000, 1000, 1000) > 0, dtype='bool')
filename = 'test.npy'
np.save(filename, dat2)
filename = '/disque/raid/vasculature/4X-test2/170824_IgG_2/170824_IgG_16-23-46/rank_threshold.npy'
timer = Timer()
ldat = ld.load(filename, verbose=True)
timer.print_elapsed_time('load')
#load: elapsed time: 0:00:04.867
timer = Timer()
ldat2 = np.load(filename)
timer.print_elapsed_time('numpy')
#numpy: elapsed time: 0:00:27.982
np.all(ldat == ldat2)
timer = Timer()
xyz = ld.where(ldat)
timer.printElapsedTime('parallel')
#parallel: elapsed time: 0:07:25.698
lldat = ldat.reshape(-1, order='A')
timer = Timer()
xyz = ld.where(lldat)
timer.printElapsedTime('parallel 1d')
#parallel 1d: elapsed time: 0:00:49.034
timer = Timer()
xyz = np.where(ldat)
timer.printElapsedTime('numpy')
import os
#os.remove(filename)
filename = './ClearMap/Test/Skeletonization/test_bin.npy'
timer = Timer()
ldat = ld.load(filename, shared=True, verbose=True)
timer.printElapsedTime('load')
ld.shm.isShared(ldat)
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.DataProcessing.LargeData as ld
reload(ld)
filename = 'test_save.npy'
dat = np.random.rand(100, 200, 100)
ld.save(filename, dat)
dat2 = ld.load(filename)
np.all(dat == dat2)
os.remove(filename)
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.DataProcessing.LargeData as ld
reload(ld)
dat = np.zeros(100, dtype=bool)
dat2 = dat.copy()
indices = np.array([5, 6, 7, 8, 13, 42])
ld.setValue(dat, indices, True, cutoff=0)
dat2[indices] = True
np.all(dat2 == dat)
d = ld.take(dat, indices, cutoff=0)
np.all(d)
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.DataProcessing.LargeData as ld
reload(ld)
pts = np.array([0, 1, 5, 6, 10, 11], dtype=int)
ld.neighbours(pts, -10)
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.DataProcessing.LargeData as ld
import ClearMap.ImageProcessing.Filter.StructureElement as sel
reload(ld)
dat = np.random.rand(30, 40, 50) > 0.5
mask = sel.structureElement('Disk', (5, 5, 5))
indices = np.where(dat.reshape(-1))[0]
c_id = len(indices) / 2
c = indices[c_id]
xyz = np.unravel_index(c, dat.shape)
l = np.array(mask.shape) / 2
r = np.array(mask.shape) - l
dlo = [max(0, xx - ll) for xx, ll in zip(xyz, l)]
dhi = [min(xx + rr, ss) for xx, rr, ss in zip(xyz, r, dat.shape)]
mlo = [-min(0, xx - ll) for xx, ll in zip(xyz, l)]
mhi = [
mm + min(0, ss - xx - rr)
for xx, rr, ss, mm in zip(xyz, r, dat.shape, mask.shape)
]
nbh = dat[dlo[0]:dhi[0], dlo[1]:dhi[1], dlo[2]:dhi[2]]
nbhm = np.logical_and(
nbh, mask[mlo[0]:mhi[0], mlo[1]:mhi[1], mlo[2]:mhi[2]] > 0)
nxyz = np.where(nbhm)
nxyz = [nn + dl for nn, dl in zip(nxyz, dlo)]
nbi = np.ravel_multi_index(nxyz, dat.shape)
nbs = ld.findNeighbours(indices, c_id, dat.shape, dat.strides, mask)
nbs.sort()
print(np.all(nbs == nbi))
dat = np.random.rand(30, 40, 50) > 0.5
indices = np.where(dat.reshape(-1))[0]
c_id = len(indices) / 2
c = indices[c_id]
xyz = np.unravel_index(c, dat.shape)
l = np.array([2, 2, 2])
r = l + 1
dlo = [max(0, xx - ll) for xx, ll in zip(xyz, l)]
dhi = [min(xx + rr, ss) for xx, rr, ss in zip(xyz, r, dat.shape)]
nbh = dat[dlo[0]:dhi[0], dlo[1]:dhi[1], dlo[2]:dhi[2]]
nxyz = np.where(nbh)
nxyz = [nn + dl for nn, dl in zip(nxyz, dlo)]
nbi = np.ravel_multi_index(nxyz, dat.shape)
nbs = ld.findNeighbours(indices, c_id, dat.shape, dat.strides, tuple(l))
nbs.sort()
print(np.all(nbs == nbi))
print(nbs)
print(nbi)
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.DataProcessing.LargeData as ld
reload(ld)
data = np.random.rand(100)
values = np.random.rand(50)
indices = np.arange(50)
ld.setArray(data, indices, values, cutoff=1)
print(np.all(data[:50] == values))
import numpy as np
from ClearMap.Utils.Timer import Timer
import ClearMap.DataProcessing.LargeData as ld
reload(ld)
m = np.array([1, 3, 6, 7, 10])
i = np.array([1, 2, 3, 4, 6, 7, 8, 9])
o = ld.match(m, i)
o2 = [np.where(i == l)[0][0] for l in m]
def detectSpots(img, detectSpotsParameter = None, correctIlluminationParameter = None, removeBackgroundParameter = None,
filterDoGParameter = None, findExtendedMaximaParameter = None, detectCellShapeParameter = None,
verbose = False, out = sys.stdout, **parameter):
"""Detect Cells in 3d grayscale image using DoG filtering and maxima detection
Effectively this function performs the following steps:
* illumination correction via :func:`~ClearMap.ImageProcessing.IlluminationCorrection.correctIllumination`
* background removal via :func:`~ClearMap.ImageProcessing.BackgroundRemoval.removeBackground`
* difference of Gaussians (DoG) filter via :func:`~ClearMap.ImageProcessing.Filter.filterDoG`
* maxima detection via :func:`~ClearMap.ImageProcessing.MaximaDetection.findExtendedMaxima`
* cell shape detection via :func:`~ClearMap.ImageProcessing.CellSizeDetection.detectCellShape`
* cell intensity and size measurements via: :func:`~ClearMap.ImageProcessing.CellSizeDetection.findCellIntensity`,
:func:`~ClearMap.ImageProcessing.CellSizeDetection.findCellSize`.
Note:
Processing steps are done in place to save memory.
Arguments:
img (array): image data
detectSpotParameter: image processing parameter as described in the individual sub-routines
verbose (bool): print progress information
out (object): object to print progress information to
Returns:
tuple: tuple of arrays (cell coordinates, raw intensity, fully filtered intensty, illumination and background corrected intensity [, cell size])
"""
timer = Timer();
# normalize data -> to check
#img = img.astype('float');
#dmax = 0.075 * 65535;
#ids = img > dmax;
#img[ids] = dmax;
#img /= dmax;
#out.write(timer.elapsedTime(head = 'Normalization'));
#img = dataset[600:1000,1600:1800,800:830];
#img = dataset[600:1000,:,800:830];
# correct illumination
correctIlluminationParameter = getParameter(detectSpotsParameter, "correctIlluminationParameter", correctIlluminationParameter);
img1 = img.copy();
img1 = correctIllumination(img1, correctIlluminationParameter = correctIlluminationParameter, verbose = verbose, out = out, **parameter)
# background subtraction in each slice
#img2 = img.copy();
removeBackgroundParameter = getParameter(detectSpotsParameter, "removeBackgroundParameter", removeBackgroundParameter);
img2 = removeBackground(img1, removeBackgroundParameter = removeBackgroundParameter, verbose = verbose, out = out, **parameter)
# mask
#timer.reset();
#if mask == None: #explicit mask
# mask = img > 0.01;
# mask = binary_opening(mask, self.structureELement('Disk', (3,3,3)));
#img[img < 0.01] = 0; # masking in place # extended maxima
#out.write(timer.elapsedTime(head = 'Mask'));
#DoG filter
filterDoGParameter = getParameter(detectSpotsParameter, "filterDoGParameter", filterDoGParameter);
dogSize = getParameter(filterDoGParameter, "size", None);
#img3 = img2.copy();
img3 = filterDoG(img2, filterDoGParameter = filterDoGParameter, verbose = verbose, out = out, **parameter);
# normalize
# imax = img.max();
# if imax == 0:
# imax = 1;
# img /= imax;
# extended maxima
findExtendedMaximaParameter = getParameter(detectSpotsParameter, "findExtendedMaximaParameter", findExtendedMaximaParameter);
hMax = getParameter(findExtendedMaximaParameter, "hMax", None);
imgmax = findExtendedMaxima(img3, findExtendedMaximaParameter = findExtendedMaximaParameter, verbose = verbose, out = out, **parameter);
#center of maxima
if not hMax is None:
centers = findCenterOfMaxima(img, imgmax, verbose = verbose, out = out, **parameter);
else:
centers = findPixelCoordinates(imgmax, verbose = verbose, out = out, **parameter);
#cell size detection
detectCellShapeParameter = getParameter(detectSpotsParameter, "detectCellShapeParameter", detectCellShapeParameter);
cellShapeThreshold = getParameter(detectCellShapeParameter, "threshold", None);
if not cellShapeThreshold is None:
# cell shape via watershed
imgshape = detectCellShape(img2, centers, detectCellShapeParameter = detectCellShapeParameter, verbose = verbose, out = out, **parameter);
#size of cells
csize = findCellSize(imgshape, maxLabel = centers.shape[0], out = out, **parameter);
#intensity of cells
cintensity = findCellIntensity(img, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
#intensity of cells in background image
cintensity2 = findCellIntensity(img2, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
#intensity of cells in dog filtered image
if dogSize is None:
cintensity3 = cintensity2;
else:
cintensity3 = findCellIntensity(img3, imgshape, maxLabel = centers.shape[0], verbose = verbose, out = out, **parameter);
if verbose:
out.write(timer.elapsedTime(head = 'Spot Detection') + '\n');
#remove cell;s of size 0
idz = csize > 0;
return ( centers[idz], numpy.vstack((cintensity[idz], cintensity3[idz], cintensity2[idz], csize[idz])).transpose());
else:
#intensity of cells
cintensity = findIntensity(img, centers, verbose = verbose, out = out, **parameter);
#intensity of cells in background image
cintensity2 = findIntensity(img2, centers, verbose = verbose, out = out, **parameter);
#intensity of cells in dog filtered image
if dogSize is None:
cintensity3 = cintensity2;
else:
cintensity3 = findIntensity(img3, centers, verbose = verbose, out = out, **parameter);
if verbose:
out.write(timer.elapsedTime(head = 'Spot Detection') + '\n');
return ( centers, numpy.vstack((cintensity, cintensity3, cintensity2)).transpose());
def detectCells(jobid,
source,
sink=None,
method="SpotDetection",
processMethod="sequential",
verbose=False,
**parameter):
"""Detect cells in data
This is a main script to start running the cell detection.
Arguments:
source (str or array): Image source
sink (str or None): destination for the results
method (str or function):
================ ============================================================
Method Description
================ ============================================================
"SpotDetection" uses predefined spot detection pipline
"Ilastik" uses predefined pipline with cell classification via Ilastik
function a user defined function
================ ============================================================
processMethod (str or all): 'sequential' or 'parallel'. if all its choosen
automatically
verbose (bool): print info
**parameter (dict): parameter for the image procesing sub-routines
TP: added JOBID for cluster jobs
Returns:
"""
timer = Timer()
# run segmentation
if method == "SpotDetection":
detectCells = ClearMap.ImageProcessing.SpotDetection.detectSpots
elif method == 'Ilastik':
if ClearMap.ImageProcessing.Ilastik.Initialized:
detectCells = ClearMap.ImageProcessing.IlastikClassification.classifyCells
processMethod = 'sequential'
#ilastik does parallel processing so force sequential processing here
else:
raise RuntimeError(
"detectCells: Ilastik not initialized, fix in Settings.py or use SpotDectection method instead!"
)
print('Ilastik functionality disabled')
else:
raise RuntimeError("detectCells: invalid method %s" % str(method))
print('ProcessMethod = {}'.format(processMethod))
if processMethod == 'cluster':
result, substack = sequentiallyProcessStack_usingCluster(
jobid,
pth_update(source),
sink=pth_update(sink),
function=detectCells,
verbose=verbose,
**parameter)
if result == 'ENDPROCESS': return 'ENDPROCESS', 'ENDPROCESS'
elif processMethod == 'sequential':
result = sequentiallyProcessStack(source,
sink=sink,
function=detectCells,
verbose=verbose,
**parameter)
elif processMethod is all or processMethod == 'parallel':
result = parallelProcessStack(source,
sink=sink,
function=detectCells,
verbose=verbose,
**parameter)
else:
raise RuntimeError("detectCells: invalid processMethod %s" %
str(processMethod))
if verbose:
timer.printElapsedTime("Total Cell Detection")
if processMethod == 'cluster':
return result, substack
else:
return result
def filterLinear(img, filterLinearParameter = None, ftype = None, size = None, sigma = None, sigma2 = None, save = None,
subStack = None, verbose = False, out = sys.stdout, **parameter):
"""Applies a linear filter to the image
Arguments:
img (array): image data
filterLinearParameter (dict):
========= ==================== ================================================================
Name Type Descritption
========= ==================== ================================================================
*ftype* (str or None) the type of the filter, see :ref:`FilterTypes`
if None do ot perform any fitlering
*size* (tuple or None) size for the filter
if None, do not perform filtering
*sigma* (tuple or None) std of outer Guassian, if None autmatically determined from size
*sigma2* (tuple or None) std of inner Guassian, if None autmatically determined from size
*save* (str or None) file name to save result of this operation
if None dont save to file
*verbose* (bool or int) print progress information
========= ==================== ================================================================
subStack (dict or None): sub-stack information
verbose (bool): print progress info
out (object): object to write progress info to
Returns:
array: filtered image
Note:
Converts image to float32 type if filter is active!
"""
timer = Timer();
ftype = getParameter(filterLinearParameter, "ftype", ftype);
size = getParameter(filterLinearParameter, "size", size);
sigma = getParameter(filterLinearParameter, "sigma", sigma);
sigma2 = getParameter(filterLinearParameter, "sigma2", sigma2);
save = getParameter(filterLinearParameter, "save", save);
verbose = getParameter(filterLinearParameter, "verbose",verbose);
if verbose:
writeParameter(out = out, head = 'Linear Filter:', ftype = ftype, size = size, sigma = sigma, sigma2 = sigma2, save = save);
if ftype is None:
return img;
#DoG filter
img = img.astype('float32'); # always convert to float for downstream processing
if not size is None:
fil = filterKernel(ftype = ftype, size = size, sigma = sigma, sigma2 = sigma2);
fil = fil.astype('float32');
#img = correlate(img, fdog);
#img = scipy.signal.correlate(img, fdog);
img = correlate(img, fil);
#img = convolve(img, fdog, mode = 'same');
img[img < 0] = 0;
if verbose > 1:
plotTiling(img);
if not save is None:
writeSubStack(save, img, subStack = subStack);
if verbose:
out.write(timer.elapsedTime(head = 'Linear Filter') + '\n');
return img