def createRandomForestClassifier(img, samples, class_names, n_samples=0, ops=None, filepath=None, params={}):
rf = FastRandomForest()
rf.setNumTrees(params.get("n_trees", 200))
rf.setNumFeatures(params.get("n_features", 2))
#rf.seed(params.get("seed", 67778))
rf.setNumThreads(params.get("n_threads", numCPUs()))
return trainClassifier(rf, img, samples, class_names, n_samples=n_samples, ops=ops, filepath=filepath)
def ensurePointMatches(filepaths, csvDir, params, n_adjacent):
""" If a pointmatches csv file doesn't exist, will create it. """
w = ParallelTasks("ensurePointMatches", exe=newFixedThreadPool(numCPUs()))
exeload = newFixedThreadPool()
try:
count = 1
for result in w.chunkConsume(
numCPUs() * 2,
pointmatchingTasks(filepaths, csvDir, params, n_adjacent,
exeload)):
if result: # is False when CSV file already exists
syncPrint("Completed %i/%i" %
(count, len(filepaths) * n_adjacent))
count += 1
syncPrint("Awaiting all remaining pointmatching tasks to finish.")
w.awaitAll()
syncPrint("Finished all pointmatching tasks.")
except:
print sys.exc_info()
finally:
exeload.shutdown()
w.destroy()
def preload(cachedCellImg, loader, block_size, filepaths):
"""
Find which is the last cell index in the cache, identify to which block
(given the blockSize[2] AKA Z dimension) that index belongs to,
and concurrently load all cells (sections) that the Z dimension of the blockSize will need.
If they are already loaded, these operations are insignificant.
"""
exe = newFixedThreadPool(n_threads=min(block_size[2], numCPUs()),
name="preloader")
try:
# The SoftRefLoaderCache.map is a ConcurrentHashMap with Long keys, aka numbers
cache = cachedCellImg.getCache()
f1 = cache.getClass().getDeclaredField(
"cache") # LoaderCacheAsCacheAdapter.cache
f1.setAccessible(True)
softCache = f1.get(cache)
cache = None
f2 = softCache.getClass().getDeclaredField(
"map") # SoftRefLoaderCache.map
f2.setAccessible(True)
keys = sorted(f2.get(softCache).keySet())
if 0 == len(keys):
return
first = keys[-1] - (keys[-1] % block_size[2])
last = max(len(filepaths), first + block_size[2] - 1)
keys = None
msg = "Preloading %i-%i" % (first, first + block_size[2] - 1)
futures = []
for index in xrange(first, first + block_size[2]):
futures.append(
exe.submit(TimeItTask(softCache.get, index, loader)))
softCache = None
# Wait for all
count = 1
while len(futures) > 0:
r, t = futures.pop(0).get()
# t in miliseconds
if t > 500:
if msg:
syncPrint(msg)
msg = None
syncPrint("preloaded index %i in %f ms" %
(first + count, t))
count += 1
if not msg: # msg was printed
syncPrint("Completed preloading %i-%i" %
(first, first + block_size[2] - 1))
except:
syncPrint(sys.exc_info())
finally:
exe.shutdown()
def export8bitN5(
filepaths,
loadFn,
img_dimensions,
matrices,
name,
exportDir,
interval,
gzip_compression=6,
invert=True,
CLAHE_params=[400, 256, 3.0],
n5_threads=0, # 0 means as many as CPU cores
block_size=[128, 128, 128]):
"""
Export into an N5 volume, in parallel, in 8-bit.
filepaths: the ordered list of filepaths, one per serial section.
loadFn: a function to load a filepath into an ImagePlus.
name: name to assign to the N5 volume.
matrices: the list of transformation matrices (each one is an array), one per section
exportDir: the directory into which to save the N5 volume.
interval: for cropping.
gzip_compression: defaults to 6 as suggested by Saalfeld. 0 means no compression.
invert: Defaults to True (necessary for FIBSEM). Whether to invert the images upon loading.
CLAHE_params: defaults to [400, 256, 3.0]. If not None, the a list of the 3 parameters needed for a CLAHE filter to apply to each image.
n5_threads: defaults to 0, meaning as many as CPU cores.
block_size: defaults to 128x128x128 px. A list of 3 integer numbers, the dimensions of each individual block.
"""
dims = Intervals.dimensionsAsLongArray(interval)
voldims = [dims[0], dims[1], len(filepaths)]
cell_dimensions = [dims[0], dims[1], 1]
def asNormalizedUnsignedByteArrayImg(interval, invert, blockRadius, n_bins,
slope, matrices, index, imp):
sp = imp.getProcessor() # ShortProcessor
# Crop to interval if needed
x = interval.min(0)
y = interval.min(1)
width = interval.max(0) - interval.min(0) + 1
height = interval.max(1) - interval.min(1) + 1
if 0 != x or 0 != y or sp.getWidth() != width or sp.getHeight(
) != height:
sp.setRoi(x, y, width, height)
sp = sp.crop()
if invert:
sp.invert()
CLAHE.run(
ImagePlus("", sp), blockRadius, n_bins, slope, None
) # far less memory requirements than NormalizeLocalContrast, and faster.
minimum, maximum = autoAdjust(sp)
# Transform and convert image to 8-bit, mapping to display range
img = ArrayImgs.unsignedShorts(
sp.getPixels(), [sp.getWidth(), sp.getHeight()])
sp = None
imp = None
# Must use linear interpolation for subpixel precision
affine = AffineTransform2D()
affine.set(matrices[index])
imgI = Views.interpolate(Views.extendZero(img),
NLinearInterpolatorFactory())
imgA = RealViews.transform(imgI, affine)
imgT = Views.zeroMin(Views.interval(imgA, img))
# Convert to 8-bit
imgMinMax = convert2(imgT,
RealUnsignedByteConverter(minimum, maximum),
UnsignedByteType,
randomAccessible=False) # use IterableInterval
aimg = ArrayImgs.unsignedBytes(Intervals.dimensionsAsLongArray(img))
# ImgUtil copies multi-threaded, which is not appropriate here as there are many other images being copied too
#ImgUtil.copy(ImgView.wrap(imgMinMax, aimg.factory()), aimg)
# Single-threaded copy
copier = createBiConsumerTypeSet(UnsignedByteType)
LoopBuilder.setImages(imgMinMax, aimg).forEachPixel(copier)
img = imgI = imgA = imgMinMax = imgT = None
return aimg
blockRadius, n_bins, slope = CLAHE_params
# A CacheLoader that interprets the list of filepaths as a 3D volume: a stack of 2D slices
loader = SectionCellLoader(
filepaths,
asArrayImg=partial(asNormalizedUnsignedByteArrayImg, interval, invert,
blockRadius, n_bins, slope, matrices),
loadFn=loadFn)
# How to preload block_size[2] files at a time? Or at least as many as numCPUs()?
# One possibility is to query the SoftRefLoaderCache.map for its entries, using a ScheduledExecutorService,
# and preload sections ahead for the whole blockSize[2] dimension.
cachedCellImg = lazyCachedCellImg(loader, voldims, cell_dimensions,
UnsignedByteType, BYTE)
exe_preloader = newFixedThreadPool(n_threads=min(
block_size[2], n5_threads if n5_threads > 0 else numCPUs()),
name="preloader")
def preload(cachedCellImg, loader, block_size, filepaths, exe):
"""
Find which is the last cell index in the cache, identify to which block
(given the blockSize[2] AKA Z dimension) that index belongs to,
and concurrently load all cells (sections) that the Z dimension of the blockSize will need.
If they are already loaded, these operations are insignificant.
"""
try:
# The SoftRefLoaderCache.map is a ConcurrentHashMap with Long keys, aka numbers
cache = cachedCellImg.getCache()
f1 = cache.getClass().getDeclaredField(
"cache") # LoaderCacheAsCacheAdapter.cache
f1.setAccessible(True)
softCache = f1.get(cache)
cache = None
f2 = softCache.getClass().getDeclaredField(
"map") # SoftRefLoaderCache.map
f2.setAccessible(True)
keys = sorted(f2.get(softCache).keySet())
if 0 == len(keys):
return
first = max(0, keys[-1] - (keys[-1] % block_size[2]))
last = min(len(filepaths), first + block_size[2]) - 1
keys = None
syncPrintQ("### Preloading %i-%i ###" % (first, last))
futures = []
for index in xrange(first, last + 1):
futures.append(
exe.submit(TimeItTask(softCache.get, index, loader)))
softCache = None
# Wait for all
loaded_any = False
count = 0
while len(futures) > 0:
r, t = futures.pop(0).get() # waits for the image to load
if t > 1000: # in miliseconds. Less than this is for sure a cache hit, more a cache miss and reload
loaded_any = True
r = None
# t in miliseconds
syncPrintQ("preloaded index %i in %f ms" % (first + count, t))
count += 1
if not loaded_any:
syncPrintQ("Completed preloading %i-%i" %
(first, first + block_size[2] - 1))
except:
syncPrintQ(sys.exc_info())
preloader = Executors.newSingleThreadScheduledExecutor()
preloader.scheduleWithFixedDelay(
RunTask(preload, cachedCellImg, loader, block_size, filepaths,
exe_preloader), 10, 60, TimeUnit.SECONDS)
try:
syncPrint("N5 directory: " + exportDir + "\nN5 dataset name: " + name +
"\nN5 blockSize: " + str(block_size))
writeN5(cachedCellImg,
exportDir,
name,
block_size,
gzip_compression_level=gzip_compression,
n_threads=n5_threads)
finally:
preloader.shutdown()
exe_preloader.shutdown()
def run():
exe = newFixedThreadPool(min(len(cropped), numCPUs()))
try:
# Dummy for in-RAM reading of isotropic images
img_filenames = [str(i) for i in xrange(len(cropped))]
loader = InRAMLoader(dict(zip(img_filenames, cropped)))
getCalibration = params.get("getCalibration", None)
if not getCalibration:
getCalibration = lambda img: [1.0] * cropped[0].numDimensions()
csv_dir = params["csv_dir"]
modelclass = params["modelclass"]
# Matrices describing the registration on the basis of the cropped images
matrices = computeOptimizedTransforms(img_filenames, loader, getCalibration,
csv_dir, exe, modelclass, params)
# Store outside, so they can be e.g. printed, and used beyond here
for matrix, affine in zip(matrices, affines):
affine.set(*matrix)
# Combine the transforms: scaling (by calibration)
# + the coarse registration (i.e. manual translations)
# + the translation introduced by the ROI cropping
# + the affine matrices computed above over the cropped images.
coarse_matrices = []
for coarse_affine in coarse_affines:
matrix = zeros(12, 'd')
coarse_affine.toArray(matrix)
coarse_matrices.append(matrix)
# NOTE: both coarse_matrices and matrices are from the camera X to camera 0. No need to invert them.
# NOTE: uses identity calibration because the coarse_matrices already include the calibration scaling to isotropy
transforms = mergeTransforms([1.0, 1.0, 1.0], coarse_matrices, [minC, maxC], matrices, invert2=False)
print "calibration:", [1.0, 1.0, 1.0]
print "cmTransforms:\n %s\n %s\n %s\n %s" % tuple(str(m) for m in coarse_matrices)
print "ROI", [minC, maxC]
print "fineTransformsPostROICrop:\n %s\n %s\n %s\n %s" % tuple(str(m) for m in matrices)
print "invert2:", False
# Show registered images
registered = [transformedView(img, transform, interval=cropped[0])
for img, transform in izip(original_images, transforms)]
registered_imp = showAsStack(registered, title="Registered with %s" % params["modelclass"].getSimpleName())
registered_imp.setDisplayRange(cropped_imp.getDisplayRangeMin(), cropped_imp.getDisplayRangeMax())
"""
# TEST: same as above, but without merging the transforms. WORKS, same result
# Copy into ArrayImg, otherwise they are rather slow to browse
def copy(img1, affine):
# Copy in two steps. Otherwise the nearest neighbor interpolation on top of another
# nearest neighbor interpolation takes a huge amount of time
dimensions = Intervals.dimensionsAsLongArray(img1)
aimg1 = ArrayImgs.unsignedShorts(dimensions)
ImgUtil.copy(ImgView.wrap(img1, aimg1.factory()), aimg1)
img2 = transformedView(aimg1, affine)
aimg2 = ArrayImgs.unsignedShorts(dimensions)
ImgUtil.copy(ImgView.wrap(img2, aimg2.factory()), aimg2)
return aimg2
futures = [exe.submit(Task(copy, img, affine)) for img, affine in izip(cropped, affines)]
aimgs = [f.get() for f in futures]
showAsStack(aimgs, title="DEBUG Registered with %s" % params["modelclass"].getSimpleName())
"""
except:
print sys.exc_info()
finally:
exe.shutdown()
SwingUtilities.invokeLater(lambda: run_button.setEnabled(True))