def classifyPixel(project,
source,
sink=None,
processingDirectory=None,
cleanup=True):
"""Run pixel classification in headless moded using a trained project file
Arguments:
project (str): ilastik project .ilp file
source (str or array): image source
sink (str or array or None): image sink
Returns:
str or array: classified image sink
"""
#generate source image file if source is array
if isinstance(source, str):
inpfile = source
else:
#generate temporary file
if processingDirectory is None:
processingDirectory = tempfile.mkdtemp()
inpfile = os.path.join(processingDirectory, 'ilastik.npy')
io.writePoints(inpfile, source.transpose((2, 1, 0)))
if isinstance(sink, str):
outfile = sink
else:
#outdir = tempfile.mkdtemp();
#outfile = os.path.join(outdir, 'result\d*.tif');
outfile = tempfile.mktemp('.h5')
ilinp = fileNameToIlastikInput(inpfile)
ilout = fileNameToIlastikOuput(outfile)
args = '--project="' + project + '" ' + ilout + ' ' + ilinp
run(args)
#clean up
if cleanup and processingDirectory is not None:
shutil.rmtree(processingDirectory)
if not isinstance(sink, str):
sink = readResultH5(outfile)
if cleanup:
os.remove(outfile)
return sink
def classifyPixel(project, source, sink = None, processingDirectory = None, cleanup = True):
"""Run pixel classification in headless moded using a trained project file
Arguments:
project (str): ilastik project .ilp file
source (str or array): image source
sink (str or array or None): image sink
Returns:
str or array: classified image sink
"""
#generate source image file if source is array
if isinstance(source, str):
inpfile = source;
else:
#generate temporary file
if processingDirectory is None:
processingDirectory = tempfile.mkdtemp();
inpfile = os.path.join(processingDirectory, 'ilastik.npy')
io.writePoints(inpfile, source.transpose((2,1,0)));
if isinstance(sink, str):
outfile = sink;
else:
#outdir = tempfile.mkdtemp();
#outfile = os.path.join(outdir, 'result\d*.tif');
outfile = tempfile.mktemp('.h5');
ilinp = fileNameToIlastikInput(inpfile);
ilout = fileNameToIlastikOuput(outfile);
args = '--project="' + project + '" ' + ilout + ' ' + ilinp;
run(args);
#clean up
if cleanup and processingDirectory is not None:
shutil.rmtree(processingDirectory);
if not isinstance(sink, str):
sink = readResultH5(outfile);
if cleanup:
os.remove(outfile);
return sink;
def parallelProcessStack(source, x = all, y = all, z = all, sink = None,
processes = 2, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
chunkOptimization = True, chunkOptimizationSize = all,
function = noProcessing, join = joinPoints, verbose = False, **parameter):
"""Parallel process a image stack
Main routine that distributes image processing on paralllel processes.
Arguments:
source (str): image source
x,y,z (tuple or all): range specifications
sink (str or None): destination for the result
processes (int): number of parallel processes
chunkSizeMax (int): maximal size of a sub-stack
chunkSizeMin (int): minial size of a sub-stack
chunkOverlap (int): minimal sub-stack overlap
chunkOptimization (bool): optimize chunck sizes to best fit number of processes
chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
function (function): the main image processing script
join (function): the fuction to join the results from the image processing script
verbose (bool): print information on sub-stack generation
Returns:
str or array: results of the image processing
"""
subStacks = calculateSubStacks(source, x = x, y = y, z = z,
processes = processes, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
chunkOptimization = chunkOptimization, chunkOptimizationSize = chunkOptimizationSize, verbose = verbose);
nSubStacks = len(subStacks);
if verbose:
print "Number of SubStacks: %d" % nSubStacks;
#for i in range(nSubStacks):
# self.printSubStackInfo(subStacks[i]);
argdata = [];
for i in range(nSubStacks):
argdata.append((function, parameter, subStacks[i], verbose));
#print argdata
# process in parallel
pool = Pool(processes = processes);
results = pool.map(_processSubStack, argdata);
#print '=========== results';
#print results;
#join the results
results = join(results, subStacks = subStacks, **parameter);
#write / or return
return io.writePoints(sink, results);
def parallelProcessStack(source, x = all, y = all, z = all, sink = None,
processes = 2, chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
chunkOptimization = True, chunkOptimizationSize = all,
function = noProcessing, join = joinPoints, verbose = False, **parameter):
"""Parallel process a image stack
Main routine that distributes image processing on paralllel processes.
Arguments:
source (str): image source
x,y,z (tuple or all): range specifications
sink (str or None): destination for the result
processes (int): number of parallel processes
chunkSizeMax (int): maximal size of a sub-stack
chunkSizeMin (int): minial size of a sub-stack
chunkOverlap (int): minimal sub-stack overlap
chunkOptimization (bool): optimize chunck sizes to best fit number of processes
chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
function (function): the main image processing script
join (function): the fuction to join the results from the image processing script
verbose (bool): print information on sub-stack generation
Returns:
str or array: results of the image processing
"""
subStacks = calculateSubStacks(source, x = x, y = y, z = z,
processes = processes, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
chunkOptimization = chunkOptimization, chunkOptimizationSize = chunkOptimizationSize, verbose = verbose);
nSubStacks = len(subStacks);
if verbose:
print("Number of SubStacks: %d" % nSubStacks);
#for i in range(nSubStacks):
# self.printSubStackInfo(subStacks[i]);
argdata = [];
for i in range(nSubStacks):
argdata.append((function, parameter, subStacks[i], verbose));
#print argdata
# process in parallel
pool = Pool(processes = processes);
results = pool.map(_processSubStack, argdata);
#print '=========== results';
#print results;
#join the results
results = join(results, subStacks = subStacks, **parameter);
#write / or return
return io.writePoints(sink, results);
def join_results_from_cluster_helper(source,
x=all,
y=all,
z=all,
sink=None,
chunkSizeMax=100,
chunkSizeMin=30,
chunkOverlap=15,
function=noProcessing,
join=joinPoints,
verbose=False,
**parameter):
#calc num of substacks
subStacks = calculateSubStacks(source,
x=x,
y=y,
z=z,
processes=1,
chunkSizeMax=chunkSizeMax,
chunkSizeMin=chunkSizeMin,
chunkOverlap=chunkOverlap,
chunkOptimization=False,
verbose=verbose)
#load all cell detection job results
if type(sink) == tuple:
pckfld = os.path.join(sink[0][:sink[0].rfind("/")], "cell_detection")
makedir(pckfld)
elif type(sink) == str:
pckfld = os.path.join(sink[:sink.rfind("/")], "cell_detection")
makedir(pckfld)
results = []
fls = listdirfull(pckfld)
fls.sort()
for fl in fls:
if "~" not in fl:
with open(fl, "rb") as pckl:
results.append(pickle.load(pckl))
pckl.close()
print(results[0])
#reformat
results = [xx for yy in results for xx in yy]
#join the results
print("Length of results: {}".format(len(results)))
results = join(results, subStacks=subStacks, **parameter)
#write / or return
return io.writePoints(sink, results)
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None,
chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
function = noProcessing, join = joinPoints, verbose = False, **parameter):
"""Sequential image processing on a stack
Main routine that sequentially processes a large image on sub-stacks.
Arguments:
source (str): image source
x,y,z (tuple or all): range specifications
sink (str or None): destination for the result
processes (int): number of parallel processes
chunkSizeMax (int): maximal size of a sub-stack
chunkSizeMin (int): minial size of a sub-stack
chunkOverlap (int): minimal sub-stack overlap
chunkOptimization (bool): optimize chunck sizes to best fit number of processes
chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
function (function): the main image processing script
join (function): the fuction to join the results from the image processing script
verbose (bool): print information on sub-stack generation
Returns:
str or array: results of the image processing
"""
#determine z ranges
subStacks = calculateSubStacks(source, x = x, y = y, z = z,
processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
chunkOptimization = False, verbose = verbose);
nSubStacks = len(subStacks);
#print nSubStacks;
argdata = [];
for i in range(nSubStacks):
argdata.append((function, parameter, subStacks[i], verbose));
#run sequentially
results = [];
for i in range(nSubStacks):
results.append(_processSubStack(argdata[i]));
#join the results
results = join(results, subStacks = subStacks, **parameter);
#write / or return
return io.writePoints(sink, results);
def output_analysis_helper(threshold=(20, 900), row=(3, 3), **params):
'''
Function to change elastix result directory before running 'step 6' i.e. point transformix to atlas.
'''
dct = pth_update(set_parameters_for_clearmap(**params))
dct['RegistrationAlignmentParameter']["resultDirectory"] = os.path.join(
params["outputdirectory"],
'clearmap_cluster_output/elastix_auto_to_sim_atlas')
points, intensities = io.readPoints(
dct['ImageProcessingParameter']["sink"])
#Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
#row = (0,0) : peak intensity from the raw data
#row = (1,1) : peak intensity from the DoG filtered data
#row = (2,2) : peak intensity from the background subtracted data
#row = (3,3) : voxel size from the watershed
points, intensities = thresholdPoints(points,
intensities,
threshold=threshold,
row=row)
#points, intensities = thresholdPoints(points, intensities, threshold = (20, 900), row = (2,2));
io.writePoints(dct['FilteredCellsFile'], (points, intensities))
# Transform point coordinates
#############################
points = io.readPoints(
dct['CorrectionResamplingPointsParameter']["pointSource"])
points = resamplePoints(**dct['CorrectionResamplingPointsParameter'])
points = transformPoints(
points,
transformDirectory=dct['CorrectionAlignmentParameter']
["resultDirectory"],
indices=False,
resultDirectory=None)
dct['CorrectionResamplingPointsInverseParameter']["pointSource"] = points
points = resamplePointsInverse(
**dct['CorrectionResamplingPointsInverseParameter'])
dct['RegistrationResamplingPointParameter']["pointSource"] = points
points = resamplePoints(**dct['RegistrationResamplingPointParameter'])
points = transformPoints(
points,
transformDirectory=dct['RegistrationAlignmentParameter']
["resultDirectory"],
indices=False,
resultDirectory=None)
io.writePoints(dct['TransformedCellsFile'], points)
# Heat map generation
#####################
points = io.readPoints(dct['TransformedCellsFile'])
intensities = io.readPoints(dct['FilteredCellsFile'][1])
#Without weigths:
vox = voxelize(points, dct['AtlasFile'], **dct['voxelizeParameter'])
if not isinstance(vox, basestring):
io.writeData(os.path.join(dct['OutputDirectory'], 'cells_heatmap.tif'),
vox.astype('int32'))
#With weigths from the intensity file (here raw intensity):
dct['voxelizeParameter']["weights"] = intensities[:, 0].astype(float)
vox = voxelize(points, dct['AtlasFile'], **dct['voxelizeParameter'])
if not isinstance(vox, basestring):
io.writeData(
os.path.join(dct['OutputDirectory'], 'cells_heatmap_weighted.tif'),
vox.astype('int32'))
#Table generation:
##################
#With integrated weigths from the intensity file (here raw intensity):
ids, counts = countPointsInRegions(points,
labeledImage=dct['AnnotationFile'],
intensities=intensities,
intensityRow=0)
table = np.zeros(ids.shape,
dtype=[('id', 'int64'), ('counts', 'f8'),
('name', 'a256')])
table["id"] = ids
table["counts"] = counts
table["name"] = labelToName(ids)
io.writeTable(
os.path.join(dct['OutputDirectory'],
'Annotated_counts_intensities.csv'), table)
#Without weigths (pure cell number):
ids, counts = countPointsInRegions(points,
labeledImage=dct['AnnotationFile'],
intensities=None)
table = np.zeros(ids.shape,
dtype=[('id', 'int64'), ('counts', 'f8'),
('name', 'a256')])
table["id"] = ids
table["counts"] = counts
table["name"] = labelToName(ids)
io.writeTable(os.path.join(dct['OutputDirectory'], 'Annotated_counts.csv'),
table)
print('Analysis Completed')
return
def transformPoints(source,
sink=None,
transformParameterFile=None,
transformDirectory=None,
indices=True,
resultDirectory=None,
tmpFile=None):
"""Transform coordinates math:`x` via elastix estimated transformation to :math:`T(x)`
Note the transformation is from the fixed image coorindates to the moving image coordiantes.
Arguments:
source (str): source of the points
sink (str or None): sink for transformed points
transformParameterFile (str or None): parameter file for the primary transformation, if None, the file is determined from the transformDirectory.
transformDirectory (str or None): result directory of elastix alignment, if None the transformParameterFile has to be given.
indices (bool): if True use points as pixel coordinates otherwise spatial coordinates.
resultDirectory (str or None): elastic result directory
tmpFile (str or None): file name for the elastix point file.
Returns:
array or str: array or file name of transformed points
"""
global TransformixBinary
checkElastixInitialized()
global ElastixSettings
if tmpFile == None:
tmpFile = os.path.join(tempfile.gettempdir(), 'elastix_input.txt')
# write text file
if isinstance(source, basestring):
#check if we have elastix signature
with open(source) as f:
line = f.readline()
f.close()
if line[:5] == 'point' or line[:5] != 'index':
txtfile = source
else:
points = io.readPoints(source)
#points = points[:,[1,0,2]];
txtfile = tmpFile
writePoints(txtfile, points)
elif isinstance(source, numpy.ndarray):
txtfile = tmpFile
#points = source[:,[1,0,2]];
writePoints(txtfile, source)
else:
raise RuntimeError('transformPoints: source not string or array!')
if resultDirectory == None:
outdirname = os.path.join(tempfile.gettempdir(), 'elastix_output')
else:
outdirname = resultDirectory
if not os.path.exists(outdirname):
os.makedirs(outdirname)
if transformParameterFile == None:
if transformDirectory == None:
RuntimeError(
'neither alignment directory and transformation parameter file specified!'
)
transformparameterdir = transformDirectory
transformparameterfile = getTransformParameterFile(
transformparameterdir)
else:
transformparameterdir = os.path.split(transformParameterFile)
transformparameterdir = transformparameterdir[0]
transformparameterfile = transformParameterFile
#transform
#make path in parameterfiles absolute
setPathTransformParameterFiles(transformparameterdir)
#run transformix
cmd = TransformixBinary + ' -threads 8 -def ' + txtfile + ' -out ' + outdirname + ' -tp ' + transformparameterfile
res = os.system(cmd)
if res != 0:
raise RuntimeError('failed executing ' + cmd)
#read data / file
if sink == []:
return io.path.join(outdirname, 'outputpoints.txt')
else:
#read coordinates
transpoints = parseElastixOutputPoints(os.path.join(
outdirname, 'outputpoints.txt'),
indices=indices)
#correct x,y,z to y,x,z
#transpoints = transpoints[:,[1,0,2]];
#cleanup
for f in os.listdir(outdirname):
os.remove(os.path.join(outdirname, f))
os.rmdir(outdirname)
return io.writePoints(sink, transpoints)
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None,
chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
function = noProcessing, join = joinPoints, verbose = False, **parameter):
"""Sequential image processing on a stack
Main routine that sequentially processes a large image on sub-stacks.
Arguments:
source (str): image source
x,y,z (tuple or all): range specifications
sink (str or None): destination for the result
processes (int): number of parallel processes
chunkSizeMax (int): maximal size of a sub-stack
chunkSizeMin (int): minial size of a sub-stack
chunkOverlap (int): minimal sub-stack overlap
chunkOptimization (bool): optimize chunck sizes to best fit number of processes
chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
function (function): the main image processing script
join (function): the fuction to join the results from the image processing script
verbose (bool): print information on sub-stack generation
Returns:
str or array: results of the image processing
"""
#determine z ranges
subStacks = calculateSubStacks(source, x = x, y = y, z = z,
processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
chunkOptimization = False, verbose = verbose);
nSubStacks = len(subStacks);
#print nSubStacks;
argdata = [];
for i in range(nSubStacks):
argdata.append((function, parameter, subStacks[i], verbose));
#run sequentially
results = [];
for i in range(nSubStacks):
results.append(_processSubStack(argdata[i]));
#join the results
results = join(results, subStacks = subStacks, **parameter);
#write / or return
return io.writePoints(sink, results);
### Pickle does not like classes:
## sub stack information
#class SubStack(object):
# """Class containing all info of a sub stack usefull for the image processing and result joining functions"""
#
# # sub stack id
# stackId = None;
#
# # number of stacks
# nStacks = None;
#
# # tuple of x,y,z range of this sub stack
# z = all;
# x = all;
# y = all;
#
# #original source
# source = None;
#
# # tuple of center point of the overlaping regions
# zCenters = None;
#
# # tuple of z indices that would generate full image without overlaps
# zCenterIndices = None;
#
# # tuple of z indices in the sub image as returned by readData that would generate full image without overlaps
# zSubCenterIndices = None;
#
#
# def __init__(slf, stackId = 0, nStacks = 1, source = None, x = all, y = all, z = all, zCenters = all, zCenterIndices = all):
# slf.stackId = stackId;
# slf.nStacks = nStacks;
# slf.source = source;
# slf.x = x;
# slf.y = y;
# slf.z = z;
# slf.zCenters = zCenters;
# slf.zCenterIndices = zCenterIndices;
# if not zCenterIndices is all and not z is all:
# slf.zSubCenterIndices = (c - z[0] for c in zCenterIndices);
# else:
# slf.zSubCenterIndices = all;
#
#
#def printSubStackInfo(slf, out = sys.stdout):
# out.write("Sub Stack: %d / %d\n" % (slf.stackId, slf.nStacks));
# out.write("source: %s\n" % slf.source);
# out.write("x,y,z: %s, %s, %s\n" % (str(slf.x), str(slf.y), str(slf.z)));
# out.write("zCenters: %s\n" % str(slf.zCenters));
# out.write("zCenterIndices: %s\n" % str(slf.zCenterIndices));
def transformPoints(source, sink = None, transformParameterFile = None, transformDirectory = None, indices = True, resultDirectory = None, tmpFile = None):
"""Transform coordinates math:`x` via elastix estimated transformation to :math:`T(x)`
Note the transformation is from the fixed image coorindates to the moving image coordiantes.
Arguments:
source (str): source of the points
sink (str or None): sink for transformed points
transformParameterFile (str or None): parameter file for the primary transformation, if None, the file is determined from the transformDirectory.
transformDirectory (str or None): result directory of elastix alignment, if None the transformParameterFile has to be given.
indices (bool): if True use points as pixel coordinates otherwise spatial coordinates.
resultDirectory (str or None): elastic result directory
tmpFile (str or None): file name for the elastix point file.
Returns:
array or str: array or file name of transformed points
"""
global TransformixBinary;
checkElastixInitialized();
global ElastixSettings;
if tmpFile == None:
tmpFile = os.path.join(tempfile.tempdir, 'elastix_input.txt');
# write text file
if isinstance(source, basestring):
#check if we have elastix signature
with open(source) as f:
line = f.readline();
f.close();
if line[:5] == 'point' or line[:5] != 'index':
txtfile = source;
else:
points = io.readPoints(source);
#points = points[:,[1,0,2]];
txtfile = tmpFile;
writePoints(txtfile, points);
elif isinstance(source, numpy.ndarray):
txtfile = tmpFile;
#points = source[:,[1,0,2]];
writePoints(txtfile, source);
else:
raise RuntimeError('transformPoints: source not string or array!');
if resultDirectory == None:
outdirname = os.path.join(tempfile.tempdir, 'elastix_output');
else:
outdirname = resultDirectory;
if not os.path.exists(outdirname):
os.makedirs(outdirname);
if transformParameterFile == None:
if transformDirectory == None:
RuntimeError('neither alignment directory and transformation parameter file specified!');
transformparameterdir = transformDirectory
transformparameterfile = getTransformParameterFile(transformparameterdir);
else:
transformparameterdir = os.path.split(transformParameterFile);
transformparameterdir = transformparameterdir[0];
transformparameterfile = transformParameterFile;
#transform
#make path in parameterfiles absolute
setPathTransformParameterFiles(transformparameterdir);
#run transformix
cmd = TransformixBinary + ' -def ' + txtfile + ' -out ' + outdirname + ' -tp ' + transformparameterfile;
res = os.system(cmd);
if res != 0:
raise RuntimeError('failed executing ' + cmd);
#read data / file
if sink == []:
return io.path.join(outdirname, 'outputpoints.txt')
else:
#read coordinates
transpoints = parseElastixOutputPoints(os.path.join(outdirname, 'outputpoints.txt'), indices = indices);
#correct x,y,z to y,x,z
#transpoints = transpoints[:,[1,0,2]];
#cleanup
for f in os.listdir(outdirname):
os.remove(os.path.join(outdirname, f));
os.rmdir(outdirname)
return io.writePoints(sink, transpoints);
#pdb.set_trace()
#io.writeData(os.path.join(homeDir, 'Results/OverlayWatershed.tif'), overlay_Img);
overlay_Img = plt.fredOverlayPoints(cfos_fn,
points,
pointColor=[200, 0, 0])
io.writeData(os.path.join(resultDir, fName + '_PointsOriginalImg.tif'),
overlay_Img)
overlay_Img = plt.overlayPoints(imgD, points, pointColor=[200, 0, 0])
io.writeData(os.path.join(resultDir, fName + '_PointsFilterDoG.tif'),
overlay_Img)
#Convert points from 3d to 2d
points = numpy.delete(points, 2, axis=1)
io.writePoints(points_fn, points)
##############################################################333
imgR = io.readData(auto_fn)
imgR = imgR[..., numpy.newaxis]
imgR = imgR.astype('int16')
imgR, scaleFactor = resampleData(imgR,
auto_R_fn,
orientation=(1, 2, 3),
dataSizeSink=None,
resolutionSource=pixelRes,
resolutionSink=(25, 25, 1),
processingDirectory=None,
processes=4,
cleanup=True,
if result == "ENDPROCESS": print("Jobid > # of jobs required, ending job")
except:
print("Jobid > # of jobs required, ending job")
print("\n finished step 4 - cell detection \n")
###########################################STEP 5########################################################
out = join_results_from_cluster_helper(**dct["ImageProcessingParameter"])
print("\n finished step 5 \n")
###########################################STEP 6########################################################
threshold = (500,3000); row = (2,2)
points, intensities = io.readPoints(dct["ImageProcessingParameter"]["sink"])
#Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
#row = (0,0) : peak intensity from the raw data
#row = (1,1) : peak intensity from the DoG filtered data
#row = (2,2) : peak intensity from the background subtracted data
#row = (3,3) : voxel size from the watershed
points, intensities = thresholdPoints(points, intensities, threshold = threshold,
row = row)
#change dst to match parameters sweeped
dst = (os.path.join(brain,
"clearmap_cluster_output/cells_rBP%s_fIPmethod%s_fIPsize%s_dCSP%s.npy" % (rBP_size,
fIP_method, fIP_size, dCSP_threshold)),
os.path.join(brain,
"clearmap_cluster_output/intensities_rBP%s_fIPmethod%s_fIPsize%s_dCSP%s.npy" % (rBP_size,
fIP_method, fIP_size, dCSP_threshold)))
io.writePoints(dst, (points, intensities));
print("\n finished step 6 \n")
#io.writeData(os.path.join(homeDir, 'Results/OverlayWatershed.tif'), overlay_Img);
overlay_Img = plt.fredOverlayPoints(input_fn, points, pointColor=[200, 0, 0])
io.writeData(os.path.join(homeDir, 'Results/PointsOriginalImg.tif'),
overlay_Img)
overlay_Img = plt.fredOverlayPoints(os.path.join(homeDir,
'Results/FilterDoG.tif'),
points,
pointColor=[200, 0, 0])
io.writeData(os.path.join(homeDir, 'Results/PointsFilterDoG.tif'), overlay_Img)
#################################################################################################
#Convert points from 3d to 2d
points = numpy.delete(points, 2, axis=1)
io.writePoints(points_fn, points)
#Resampled Img
imgR = io.readData(auto_fn)
imgR = imgR[..., numpy.newaxis]
imgR = imgR.astype('int16')
imgR, scaleFactor = resampleData(imgR,
auto_R_fn,
orientation=(1, 2, 3),
dataSizeSink=None,
resolutionSource=(2.3719, 2.3719, 1),
resolutionSink=(25, 25, 1),
processingDirectory=None,
processes=4,
cleanup=True,
verbose=True,
interpolation='linear')
def flatfieldLineFromRegression(data, sink = None, method = 'polynomial', reverse = None, verbose = False):
"""Create flat field line fit from a list of positions and intensities
The fit is either to be assumed to be a Gaussian:
.. math:
I(x) = a \\exp^{- (x- x_0)^2 / (2 \\sigma)) + b"
or follows a order 6 radial polynomial
.. math:
I(x) = a + b (x- x_0)^2 + c (x- x_0)^4 + d (x- x_0)^6
Arguments:
data (array): intensity data as vector of intensities or (n,2) dim array of positions d=0 and intensities measurements d=1:-1
sink (str or None): destination to write the result of the fit
method (str): method to fit intensity data, 'Gaussian' or 'Polynomial'
reverse (bool): reverse the line fit after fitting
verbose (bool): print and plot information for the fit
Returns:
array: fitted intensities on points
"""
data = io.readPoints(data);
# split data
if len(data.shape) == 1:
x = numpy.arange(0, data.shape[0]);
y = data;
elif len(data.shape) == 2:
x = data[:,0]
y = data[:,1:-1];
else:
raise RuntimeError('flatfieldLineFromRegression: input data not a line or array of x,i data');
#calculate mean of the intensity measurements
ym = numpy.mean(y, axis = 1);
if verbose:
plt.figure()
for i in range(1,data.shape[1]):
plt.plot(x, data[:,i]);
plt.plot(x, ym, 'k');
if method.lower() == 'polynomial':
## fit r^6
mean = sum(ym * x)/sum(ym)
def f(x,m,a,b,c,d):
return a + b * (x-m)**2 + c * (x-m)**4 + d * (x-m)**6;
popt, pcov = curve_fit(f, x, ym, p0 = (mean, 1, 1, 1, .1));
m = popt[0]; a = popt[1]; b = popt[2];
c = popt[3]; d = popt[4];
if verbose:
print "polynomial fit: %f + %f (x- %f)^2 + %f (x- %f)^4 + %f (x- %f)^6" % (a, b, m, c, m, d, m);
def fopt(x):
return f(x, m = m, a = a, b = b, c = c, d = d);
flt = map(fopt, range(0, int(x[-1])));
else:
## Gaussian fit
mean = sum(ym * x)/sum(ym)
sigma = sum(ym * (x-mean)**2)/(sum(ym))
def f(x, a, m, s, b):
return a * numpy.exp(- (x - m)**2 / 2 / s) + b;
popt, pcov = curve_fit(f, x, ym, p0 = (1000, mean, sigma, 400));
a = popt[0]; m = popt[1]; s = popt[2]; b = popt[3];
if verbose:
print "Gaussian fit: %f exp(- (x- %f)^2 / (2 %f)) + %f" % (a, m, s, b);
def fopt(x):
return f(x, a = a, m = m, s = s, b = b);
if reverse:
flt.reverse();
if verbose:
plt.plot(x, flt);
plt.title('flatfieldLineFromRegression')
return io.writePoints(sink, flt);
def output_analysis(
threshold=(20, 900), row=(3, 3), check_cell_detection=False, **params):
"""Wrapper for analysis:
Inputs
-------------------
Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
Row:
row = (0,0) : peak intensity from the raw data
row = (1,1) : peak intensity from the DoG filtered data
row = (2,2) : peak intensity from the background subtracted data
row = (3,3) : voxel size from the watershed
Check Cell detection: (For the testing phase only, remove when running on the full size dataset)
"""
dct = pth_update(set_parameters_for_clearmap(**params))
points, intensities = io.readPoints(
dct["ImageProcessingParameter"]["sink"])
#Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
#row = (0,0) : peak intensity from the raw data
#row = (1,1) : peak intensity from the DoG filtered data
#row = (2,2) : peak intensity from the background subtracted data
#row = (3,3) : voxel size from the watershed
points, intensities = thresholdPoints(points,
intensities,
threshold=threshold,
row=row)
#points, intensities = thresholdPoints(points, intensities, threshold = (20, 900), row = (2,2));
io.writePoints(dct["FilteredCellsFile"], (points, intensities))
## Check Cell detection (For the testing phase only, remove when running on the full size dataset)
#######################
# if check_cell_detection:
# import ClearMap.Visualization.Plot as plt
# pointSource= os.path.join(BaseDirectory, FilteredCellsFile[0]);
# data = plt.overlayPoints(cFosFile, pointSource, pointColor = None, **cFosFileRange);
# io.writeData(os.path.join(BaseDirectory, "cells_check.tif"), data);
# Transform point coordinates
#############################
points = io.readPoints(
dct["CorrectionResamplingPointsParameter"]["pointSource"])
points = resamplePoints(**dct["CorrectionResamplingPointsParameter"])
points = transformPoints(
points,
transformDirectory=dct["CorrectionAlignmentParameter"]
["resultDirectory"],
indices=False,
resultDirectory=None)
dct["CorrectionResamplingPointsInverseParameter"]["pointSource"] = points
points = resamplePointsInverse(
**dct["CorrectionResamplingPointsInverseParameter"])
dct["RegistrationResamplingPointParameter"]["pointSource"] = points
points = resamplePoints(**dct["RegistrationResamplingPointParameter"])
points = transformPoints(
points,
transformDirectory=dct["RegistrationAlignmentParameter"]
["resultDirectory"],
indices=False,
resultDirectory=None)
io.writePoints(dct["TransformedCellsFile"], points)
# Heat map generation
#####################
points = io.readPoints(dct["TransformedCellsFile"])
intensities = io.readPoints(dct["FilteredCellsFile"][1])
#Without weigths:
vox = voxelize(points, dct["AtlasFile"], **dct["voxelizeParameter"])
if not isinstance(vox, str):
io.writeData(os.path.join(dct["OutputDirectory"], "cells_heatmap.tif"),
vox.astype("int32"))
#With weigths from the intensity file (here raw intensity):
dct["voxelizeParameter"]["weights"] = intensities[:, 0].astype(float)
vox = voxelize(points, dct["AtlasFile"], **dct["voxelizeParameter"])
if not isinstance(vox, str):
io.writeData(
os.path.join(dct["OutputDirectory"], "cells_heatmap_weighted.tif"),
vox.astype("int32"))
#Table generation:
##################
#With integrated weigths from the intensity file (here raw intensity):
try:
ids, counts = countPointsInRegions(points,
labeledImage=dct["AnnotationFile"],
intensities=intensities,
intensityRow=0)
table = numpy.zeros(ids.shape,
dtype=[("id", "int64"), ("counts", "f8"),
("name", "a256")])
table["id"] = ids
table["counts"] = counts
table["name"] = labelToName(ids)
io.writeTable(
os.path.join(dct["OutputDirectory"],
"Annotated_counts_intensities.csv"), table)
#Without weigths (pure cell number):
ids, counts = countPointsInRegions(points,
labeledImage=dct["AnnotationFile"],
intensities=None)
table = numpy.zeros(ids.shape,
dtype=[("id", "int64"), ("counts", "f8"),
("name", "a256")])
table["id"] = ids
table["counts"] = counts
table["name"] = labelToName(ids)
io.writeTable(
os.path.join(dct["OutputDirectory"], "Annotated_counts.csv"),
table)
except:
print("Table not generated.\n")
print("Analysis Completed")
return
ImageProcessingParameter = joinParameter(StackProcessingParameter,
SpotDetectionParameter)
#Cell detection:
################
detectCells(**ImageProcessingParameter)
#Thresholding: the threshold parameter is either intensity or size in voxel, depending on the chosen "row"
#row = (0,0) : peak intensity from the raw data
#row = (1,1) : peak intensity from the DoG filtered data
#row = (2,2) : peak intensity from the background subtracted data
#row = (3,3) : voxel size from the watershed
points, intensities = thresholdPoints(points,
intensities,
threshold=(20, 900),
row=(3, 3))
io.writePoints(FilteredCellsFile, (points, intensities))
# Transform point coordinates
#############################
points = io.readPoints(CorrectionResamplingPointsParameter["pointSource"])
points = resamplePoints(**CorrectionResamplingPointsParameter)
points = transformPoints(
points,
transformDirectory=CorrectionAlignmentParameter["resultDirectory"],
indices=False,
resultDirectory=None)
CorrectionResamplingPointsInverseParameter["pointSource"] = points
points = resamplePointsInverse(**CorrectionResamplingPointsInverseParameter)
RegistrationResamplingPointParameter["pointSource"] = points
points = resamplePoints(**RegistrationResamplingPointParameter)
points = transformPoints(
points,
def flatfieldLineFromRegression(data,
sink=None,
method='polynomial',
reverse=None,
verbose=False):
"""Create flat field line fit from a list of positions and intensities
The fit is either to be assumed to be a Gaussian:
.. math:
I(x) = a \\exp^{- (x- x_0)^2 / (2 \\sigma)) + b"
or follows a order 6 radial polynomial
.. math:
I(x) = a + b (x- x_0)^2 + c (x- x_0)^4 + d (x- x_0)^6
Arguments:
data (array): intensity data as vector of intensities or (n,2) dim array of positions d=0 and intensities measurements d=1:-1
sink (str or None): destination to write the result of the fit
method (str): method to fit intensity data, 'Gaussian' or 'Polynomial'
reverse (bool): reverse the line fit after fitting
verbose (bool): print and plot information for the fit
Returns:
array: fitted intensities on points
"""
data = io.readPoints(data)
# split data
if len(data.shape) == 1:
x = numpy.arange(0, data.shape[0])
y = data
elif len(data.shape) == 2:
x = data[:, 0]
y = data[:, 1:-1]
else:
raise RuntimeError(
'flatfieldLineFromRegression: input data not a line or array of x,i data'
)
#calculate mean of the intensity measurements
ym = numpy.mean(y, axis=1)
if verbose:
plt.figure()
for i in range(1, data.shape[1]):
plt.plot(x, data[:, i])
plt.plot(x, ym, 'k')
if method.lower() == 'polynomial':
## fit r^6
mean = sum(ym * x) / sum(ym)
def f(x, m, a, b, c, d):
return a + b * (x - m)**2 + c * (x - m)**4 + d * (x - m)**6
popt, pcov = curve_fit(f, x, ym, p0=(mean, 1, 1, 1, .1))
m = popt[0]
a = popt[1]
b = popt[2]
c = popt[3]
d = popt[4]
if verbose:
print "polynomial fit: %f + %f (x- %f)^2 + %f (x- %f)^4 + %f (x- %f)^6" % (
a, b, m, c, m, d, m)
def fopt(x):
return f(x, m=m, a=a, b=b, c=c, d=d)
flt = map(fopt, range(0, int(x[-1])))
else:
## Gaussian fit
mean = sum(ym * x) / sum(ym)
sigma = sum(ym * (x - mean)**2) / (sum(ym))
def f(x, a, m, s, b):
return a * numpy.exp(-(x - m)**2 / 2 / s) + b
popt, pcov = curve_fit(f, x, ym, p0=(1000, mean, sigma, 400))
a = popt[0]
m = popt[1]
s = popt[2]
b = popt[3]
if verbose:
print "Gaussian fit: %f exp(- (x- %f)^2 / (2 %f)) + %f" % (a, m, s,
b)
def fopt(x):
return f(x, a=a, m=m, s=s, b=b)
if reverse:
flt.reverse()
if verbose:
plt.plot(x, flt)
plt.title('flatfieldLineFromRegression')
return io.writePoints(sink, flt)
def sequentiallyProcessStack(source, x = all, y = all, z = all, sink = None,
chunkSizeMax = 100, chunkSizeMin = 30, chunkOverlap = 15,
function = noProcessing, join = joinPoints, verbose = False, **parameter):
"""Sequential image processing on a stack
Main routine that sequentially processes a large image on sub-stacks.
Arguments:
source (str): image source
x,y,z (tuple or all): range specifications
sink (str or None): destination for the result
processes (int): number of parallel processes
chunkSizeMax (int): maximal size of a sub-stack
chunkSizeMin (int): minial size of a sub-stack
chunkOverlap (int): minimal sub-stack overlap
chunkOptimization (bool): optimize chunck sizes to best fit number of processes
chunkOptimizationSize (bool or all): if True only decrease the chunk size when optimizing
function (function): the main image processing script
join (function): the fuction to join the results from the image processing script
verbose (bool): print information on sub-stack generation
Returns:
str or array: results of the image processing
"""
#determine z ranges
subStacks = calculateSubStacks(source, x = x, y = y, z = z,
processes = 1, chunkSizeMax = chunkSizeMax, chunkSizeMin = chunkSizeMin, chunkOverlap = chunkOverlap,
chunkOptimization = False, verbose = verbose);
nSubStacks = len(subStacks);
#print nSubStacks;
argdata = [];
for i in range(nSubStacks):
argdata.append((function, parameter, subStacks[i], verbose));
#run sequentially
results = [];
for i in range(nSubStacks):
results.append(_processSubStack(argdata[i]));
#join the results
results = join(results, subStacks = subStacks, **parameter);
#write / or return
return io.writePoints(sink, results);
### Pickle does not like classes:
## sub stack information
#class SubStack(object):
# """Class containing all info of a sub stack usefull for the image processing and result joining functions"""
#
# # sub stack id
# stackId = None;
#
# # number of stacks
# nStacks = None;
#
# # tuple of x,y,z range of this sub stack
# z = all;
# x = all;
# y = all;
#
# #original source
# source = None;
#
# # tuple of center point of the overlaping regions
# zCenters = None;
#
# # tuple of z indices that would generate full image without overlaps
# zCenterIndices = None;
#
# # tuple of z indices in the sub image as returned by readData that would generate full image without overlaps
# zSubCenterIndices = None;
#
#
# def __init__(slf, stackId = 0, nStacks = 1, source = None, x = all, y = all, z = all, zCenters = all, zCenterIndices = all):
# slf.stackId = stackId;
# slf.nStacks = nStacks;
# slf.source = source;
# slf.x = x;
# slf.y = y;
# slf.z = z;
# slf.zCenters = zCenters;
# slf.zCenterIndices = zCenterIndices;
# if not zCenterIndices is all and not z is all:
# slf.zSubCenterIndices = (c - z[0] for c in zCenterIndices);
# else:
# slf.zSubCenterIndices = all;
#
#
#def printSubStackInfo(slf, out = sys.stdout):
# out.write("Sub Stack: %d / %d\n" % (slf.stackId, slf.nStacks));
# out.write("source: %s\n" % slf.source);
# out.write("x,y,z: %s, %s, %s\n" % (str(slf.x), str(slf.y), str(slf.z)));
# out.write("zCenters: %s\n" % str(slf.zCenters));
# out.write("zCenterIndices: %s\n" % str(slf.zCenterIndices));