def preload(cachedCellImg, loader):
"""
Find which is the last cell index in the cache, identify to which block
(given the blockSize[2] AKA Z dimension) that index belongs to,
and concurrently load all cells (sections) that the Z dimension of the blockSize will need.
If they are already loaded, these operations are insignificant.
"""
# The SoftRefLoaderCache.map is a ConcurrentHashMap with Long keys, aka numbers
cache = cachedCellImg.getCache()
f1 = cache.getClass().getDeclaredField(
"cache") # LoaderCacheAsCacheAdapter.cache
f1.setAccessible(True)
softCache = f1.get(cache)
f2 = softCache.getClass().getDeclaredField(
"map") # SoftRefLoaderCache.map
f2.setAccessible(True)
keys = sorted(f2.get(softCache).getKeys())
if 0 == len(keys):
return
first = keys[-1] - (keys[-1] % blockSize[2])
syncPrint("Preloading %i-%i" % (first, first + blockSize[2] - 1))
exe = newFixedThreadPool(n_threads=-1, name="preloader")
try:
for index in xrange(first, first + blockSize[2]):
exe.submit(Task, softCache.get, index, loader)
except:
syncPrint(sys.exc_info())
finally:
exe.shutdown()
def ensurePointMatches(filepaths, csvDir, params, n_adjacent):
""" If a pointmatches csv file doesn't exist, will create it. """
w = ParallelTasks("ensurePointMatches", exe=newFixedThreadPool(numCPUs()))
exeload = newFixedThreadPool() # uses as many threads as CPUs
try:
count = 1
for result in w.chunkConsume(
numCPUs() * 2,
pointmatchingTasks(filepaths, csvDir, params, n_adjacent,
exeload)):
if result: # is False when CSV file already exists
syncPrint("Completed %i/%i" %
(count, len(filepaths) * n_adjacent))
count += 1
syncPrint("Awaiting all remaining pointmatching tasks to finish.")
w.awaitAll()
syncPrint("Finished all pointmatching tasks.")
except:
print sys.exc_info()
finally:
exeload.shutdown()
w.destroy()
def preload(cachedCellImg, loader, block_size, filepaths):
"""
Find which is the last cell index in the cache, identify to which block
(given the blockSize[2] AKA Z dimension) that index belongs to,
and concurrently load all cells (sections) that the Z dimension of the blockSize will need.
If they are already loaded, these operations are insignificant.
"""
exe = newFixedThreadPool(n_threads=min(block_size[2], numCPUs()),
name="preloader")
try:
# The SoftRefLoaderCache.map is a ConcurrentHashMap with Long keys, aka numbers
cache = cachedCellImg.getCache()
f1 = cache.getClass().getDeclaredField(
"cache") # LoaderCacheAsCacheAdapter.cache
f1.setAccessible(True)
softCache = f1.get(cache)
cache = None
f2 = softCache.getClass().getDeclaredField(
"map") # SoftRefLoaderCache.map
f2.setAccessible(True)
keys = sorted(f2.get(softCache).keySet())
if 0 == len(keys):
return
first = keys[-1] - (keys[-1] % block_size[2])
last = max(len(filepaths), first + block_size[2] - 1)
keys = None
msg = "Preloading %i-%i" % (first, first + block_size[2] - 1)
futures = []
for index in xrange(first, first + block_size[2]):
futures.append(
exe.submit(TimeItTask(softCache.get, index, loader)))
softCache = None
# Wait for all
count = 1
while len(futures) > 0:
r, t = futures.pop(0).get()
# t in miliseconds
if t > 500:
if msg:
syncPrint(msg)
msg = None
syncPrint("preloaded index %i in %f ms" %
(first + count, t))
count += 1
if not msg: # msg was printed
syncPrint("Completed preloading %i-%i" %
(first, first + block_size[2] - 1))
except:
syncPrint(sys.exc_info())
finally:
exe.shutdown()
def register(view_index,
filepaths,
modelclass,
csv_dir,
params,
n_threads=0,
workaround2487=True):
# Work around jython bug https://bugs.jython.org/issue2487 , a race condition on type initialization
if 0 == view_index and workaround2487:
# bogus call for first two filepaths: single-threaded execution to initalize types
register(view_index, {view_index: filepaths[view_index][0:2]},
modelclass,
"/tmp/",
params,
n_threads=1,
workaround2487=False)
exe = newFixedThreadPool(n_threads)
paths = filepaths[view_index]
try:
name = "matrices-view-%i" % view_index
matrices = loadMatrices(name, csv_dir)
if not matrices:
matrices = computeForwardTransforms(paths, klb_loader,
getCalibration, csv_dir, exe,
modelclass, params)
# Debug: print identity transforms
identity_indices = [
i for i, m in enumerate(matrices) if 1.0 == m[0]
and 0.0 == m[1] and 0.0 == m[2] and 0.0 == m[3] and 0.0 == m[4]
and 1.0 == m[5] and 0.0 == m[6] and 0.0 == m[7] and 0.0 == m[8]
and 0.0 == m[9] and 1.0 == m[10] and 0.0 == m[11]
]
syncPrint("View %i: identity matrices at [%s]" %
(view_index, ", ".join(
str(index) for index in identity_indices)))
saveMatrices(name, matrices, csv_dir)
finally:
exe.shutdown()
transforms = asBackwardConcatTransforms(matrices,
transformclass=Translation3D)
path_transforms = dict(izip(paths, transforms))
registered_loader = RegisteredLoader(klb_loader, path_transforms)
return Load.lazyStack(paths, registered_loader)
def __init__(self,
filepaths,
loadImg,
matrices,
img_dimensions,
cell_dimensions,
interval,
copy_threads=1,
preload=None):
self.filepaths = filepaths
self.loadImg = loadImg # function to load images
self.matrices = matrices
self.img_dimensions = img_dimensions
self.cell_dimensions = cell_dimensions # x,y must match dims of interval
self.interval = interval # when smaller than the image, will crop
self.copy_threads = max(1, copy_threads)
self.cache = SoftMemoize(partial(TranslatedSectionGet.makeCell, self),
maxsize=256)
self.exe = newFixedThreadPool(
-1) # BEWARE native memory leak if not closed
self.preload = preload
with open(csv_sums_path, 'r') as csvfile:
reader = csv.reader(csvfile, delimiter=',', quotechar="\"")
header = reader.next() # skip
sums = [(filename, float(s)) for i, (filename, s) in enumerate(reader)]
else:
# Compute:
TMs = []
for foldername in sorted(os.listdir(srcDir)):
index = foldername[2:]
filename = os.path.join(
foldername,
"SPM00_TM%s_CM00_CM01_CHN00.weightFused.TimeRegistration.klb" %
index)
TMs.append(filename)
exe = newFixedThreadPool(-1)
try:
def computeSum(filename, aimg=None):
syncPrint(filename)
img = aimg if aimg is not None else klb.readFull(
os.path.join(srcDir, filename))
try:
return filename, sum(
img.getImg().update(None).getCurrentStorageArray())
except:
syncPrint("Failed to compute sum: retry")
return computeSum(filename, aimg=img)
futures = [exe.submit(Task(computeSum, filename)) for filename in TMs]
def deconvolve(images, kernels, name, n_iterations): # Bayesian-based multi-view deconvolution exe = newFixedThreadPool(Runtime.getRuntime().availableProcessors() -2)
# RANSAC parameters: reduce list of pointmatches to a spatially coherent subset
paramsModel = {
"maxEpsilon": somaDiameter, # max allowed alignment error in calibrated units (a distance)
"minInlierRatio": 0.0000001, # ratio inliers/candidates
"minNumInliers": 5, # minimum number of good matches to accept the result
"n_iterations": 2000, # for estimating the model
"maxTrust": 4, # for rejecting candidates
}
# Joint dictionary of parameters
params = {}
params.update(paramsDoG)
params.update(paramsFeatures)
params.update(paramsModel)
exe = newFixedThreadPool(2)
#viewInBDV(imgB0, imgB1, imgB2, imgB3)
# Parameter exploration target: increase dramatically the number of inlier point matches,
# in order to be able to reliably estimate a TranslationModel3D (and RigidiModel3D) across camera views.
# Which requires extracting features optimized for the overlapping regions.
# Strategy 1: allow more features per peak
params["max_per_peak"] = 4 # was 3
# Did increase the number of features but not by much, and not the number of point matches
# Strategy 2: allow constellations with smaller angles to better capture whatever feature could be available
params["min_angle"] = 0.25 # was: 1.57
# Did increase the number of features by a lot (about double), and a tiny bit the point matches
def maxProjectLastDimension(img, strategy="1by1", chunk_size=0):
last_dimension = img.numDimensions() -1
if "1by1" == strategy:
exe = newFixedThreadPool()
try:
n_threads = exe.getCorePoolSize()
imgTs = [ArrayImgs.unsignedShorts(list(Intervals.dimensionsAsLongArray(img))[:-1]) for i in xrange(n_threads)]
def mergeMax(img1, img2, imgT):
return compute(maximum(img1, img2)).into(imgT)
def hyperSlice(index):
return Views.hyperSlice(img, last_dimension, index)
# The first n_threads mergeMax:
futures = [exe.submit(Task(mergeMax, hyperSlice(i*2), hyperSlice(i*2 +1), imgTs[i]))
for i in xrange(n_threads)]
# As soon as one finishes, merge it with the next available hyperSlice
next = n_threads
while len(futures) > 0: # i.e. not empty
imgT = futures.pop(0).get()
if next < img.dimension(last_dimension):
futures.append(exe.submit(Task(mergeMax, imgT, hyperSlice(next), imgT)))
next += 1
else:
# Run out of hyperSlices to merge
if 0 == len(futures):
return imgT # done
# Merge imgT to each other until none remain
futures.append(exe.submit(Task(mergeMax, imgT, futures.pop(0).get(), imgT)))
finally:
exe.shutdownNow()
else:
# By chunks
imglibtype = img.randomAccess().get().getClass()
# The Converter class
reduce_max = makeCompositeToRealConverter(reducer_class=Math,
reducer_method="max",
reducer_method_signature="(DD)D")
if chunk_size > 0:
# map reduce approach
exe = newFixedThreadPool()
try:
def projectMax(img, minC, maxC, reduce_max):
imgA = ArrayImgs.unsignedSorts(Intervals.dimensionsAsLongArray(imgC))
ImgUtil.copy(ImgView.wrap(convert(Views.collapseReal(Views.interval(img, minC, maxC)), reduce_max.newInstance(), imglibtype), img.factory()), imgA)
return imgA
# The min and max coordinates of all dimensions except the last one
minCS = [0 for d in xrange(last_dimension)]
maxCS = [img.dimension(d) -1 for d in xrange(last_dimension)]
# Process every chunk in parallel
futures = [exe.submit(Task(projectMax, img, minCS + [offset], maxCS + [min(offset + chunk_size, img.dimension(last_dimension)) -1]))
for offset in xrange(0, img.dimension(last_dimension), chunk_size)]
return reduce(lambda f1, f2: compute(maximum(f1.get(), f2.get())).into(f1.get(), futures))
finally:
exe.shutdownNow()
else:
# One chunk: all at once
# Each sample of img3DV is a virtual vector over all time frames at that 3D coordinate
# Reduce each vector to a single scalar, using a Converter
img3DC = convert(Views.collapseReal(img), reduce_max.newInstance(), imglibtype)
imgA = ArrayImgs.unsignedShorts([img.dimension(d) for d in xrange(last_dimension)])
ImgUtil.copy(ImgView.wrap(imgV, img.factory()), imgA)
return imgA
# RANSAC parameters: reduce list of pointmatches to a spatially coherent subset
paramsModel = {
"maxEpsilon": somaDiameter, # max allowed alignment error in calibrated units (a distance)
"minInlierRatio": 0.0000001, # ratio inliers/candidates
"minNumInliers": 5, # minimum number of good matches to accept the result
"n_iterations": 2000, # for estimating the model
"maxTrust": 4, # for rejecting candidates
}
# Joint dictionary of parameters
params = {}
params.update(paramsDoG)
params.update(paramsFeatures)
params.update(paramsModel)
exe = newFixedThreadPool(2)
#viewInBDV(imgB0, imgB1, imgB2, imgB3)
# Parameter exploration target: increase dramatically the number of inlier point matches,
# in order to be able to reliably estimate a TranslationModel3D (and RigidiModel3D) across camera views.
# Which requires extracting features optimized for the overlapping regions.
# Strategy 1: allow more features per peak
params["max_per_peak"] = 4 # was 3
# Did increase the number of features but not by much, and not the number of point matches
# Strategy 2: allow constellations with smaller angles to better capture whatever feature could be available
params["min_angle"] = 0.25 # was: 1.57
# Did increase the number of features by a lot (about double), and a tiny bit the point matches
# Ensure target directory exists
if not os.path.exists(targetFolder):
os.mkdir(targetFolder)
def convert(filepath, targetFolder):
imp = IJ.openImage(filepath)
if imp.getType() == ImagePlus.GRAY32:
ic = ImageConverter(imp)
ic.setDoScaling(False)
ic.convertToGray16()
filename = os.path.basename(filepath)
writeZip(imp, os.path.join(targetFolder, filename), title=filename)
else:
syncPrint("Not a 32-bit image: " + filename)
exe = newFixedThreadPool()
try:
futures = []
for filename in sorted(os.listdir(sourceFolder)):
filepath = os.path.join(sourceFolder, filename)
if accept(filepath):
futures.append(exe.submit(Task(convert, filepath, targetFolder)))
for f in futures:
f.get() # wait
finally:
exe.shutdownNow()