def createRandomForestClassifier(img, samples, class_names, n_samples=0, ops=None, filepath=None, params={}): rf = FastRandomForest() rf.setNumTrees(params.get("n_trees", 200)) rf.setNumFeatures(params.get("n_features", 2)) #rf.seed(params.get("seed", 67778)) rf.setNumThreads(params.get("n_threads", numCPUs())) return trainClassifier(rf, img, samples, class_names, n_samples=n_samples, ops=ops, filepath=filepath)
def ensurePointMatches(filepaths, csvDir, params, n_adjacent): """ If a pointmatches csv file doesn't exist, will create it. """ w = ParallelTasks("ensurePointMatches", exe=newFixedThreadPool(numCPUs())) exeload = newFixedThreadPool() try: count = 1 for result in w.chunkConsume( numCPUs() * 2, pointmatchingTasks(filepaths, csvDir, params, n_adjacent, exeload)): if result: # is False when CSV file already exists syncPrint("Completed %i/%i" % (count, len(filepaths) * n_adjacent)) count += 1 syncPrint("Awaiting all remaining pointmatching tasks to finish.") w.awaitAll() syncPrint("Finished all pointmatching tasks.") except: print sys.exc_info() finally: exeload.shutdown() w.destroy()
def preload(cachedCellImg, loader, block_size, filepaths): """ Find which is the last cell index in the cache, identify to which block (given the blockSize[2] AKA Z dimension) that index belongs to, and concurrently load all cells (sections) that the Z dimension of the blockSize will need. If they are already loaded, these operations are insignificant. """ exe = newFixedThreadPool(n_threads=min(block_size[2], numCPUs()), name="preloader") try: # The SoftRefLoaderCache.map is a ConcurrentHashMap with Long keys, aka numbers cache = cachedCellImg.getCache() f1 = cache.getClass().getDeclaredField( "cache") # LoaderCacheAsCacheAdapter.cache f1.setAccessible(True) softCache = f1.get(cache) cache = None f2 = softCache.getClass().getDeclaredField( "map") # SoftRefLoaderCache.map f2.setAccessible(True) keys = sorted(f2.get(softCache).keySet()) if 0 == len(keys): return first = keys[-1] - (keys[-1] % block_size[2]) last = max(len(filepaths), first + block_size[2] - 1) keys = None msg = "Preloading %i-%i" % (first, first + block_size[2] - 1) futures = [] for index in xrange(first, first + block_size[2]): futures.append( exe.submit(TimeItTask(softCache.get, index, loader))) softCache = None # Wait for all count = 1 while len(futures) > 0: r, t = futures.pop(0).get() # t in miliseconds if t > 500: if msg: syncPrint(msg) msg = None syncPrint("preloaded index %i in %f ms" % (first + count, t)) count += 1 if not msg: # msg was printed syncPrint("Completed preloading %i-%i" % (first, first + block_size[2] - 1)) except: syncPrint(sys.exc_info()) finally: exe.shutdown()
def export8bitN5( filepaths, loadFn, img_dimensions, matrices, name, exportDir, interval, gzip_compression=6, invert=True, CLAHE_params=[400, 256, 3.0], n5_threads=0, # 0 means as many as CPU cores block_size=[128, 128, 128]): """ Export into an N5 volume, in parallel, in 8-bit. filepaths: the ordered list of filepaths, one per serial section. loadFn: a function to load a filepath into an ImagePlus. name: name to assign to the N5 volume. matrices: the list of transformation matrices (each one is an array), one per section exportDir: the directory into which to save the N5 volume. interval: for cropping. gzip_compression: defaults to 6 as suggested by Saalfeld. 0 means no compression. invert: Defaults to True (necessary for FIBSEM). Whether to invert the images upon loading. CLAHE_params: defaults to [400, 256, 3.0]. If not None, the a list of the 3 parameters needed for a CLAHE filter to apply to each image. n5_threads: defaults to 0, meaning as many as CPU cores. block_size: defaults to 128x128x128 px. A list of 3 integer numbers, the dimensions of each individual block. """ dims = Intervals.dimensionsAsLongArray(interval) voldims = [dims[0], dims[1], len(filepaths)] cell_dimensions = [dims[0], dims[1], 1] def asNormalizedUnsignedByteArrayImg(interval, invert, blockRadius, n_bins, slope, matrices, index, imp): sp = imp.getProcessor() # ShortProcessor # Crop to interval if needed x = interval.min(0) y = interval.min(1) width = interval.max(0) - interval.min(0) + 1 height = interval.max(1) - interval.min(1) + 1 if 0 != x or 0 != y or sp.getWidth() != width or sp.getHeight( ) != height: sp.setRoi(x, y, width, height) sp = sp.crop() if invert: sp.invert() CLAHE.run( ImagePlus("", sp), blockRadius, n_bins, slope, None ) # far less memory requirements than NormalizeLocalContrast, and faster. minimum, maximum = autoAdjust(sp) # Transform and convert image to 8-bit, mapping to display range img = ArrayImgs.unsignedShorts( sp.getPixels(), [sp.getWidth(), sp.getHeight()]) sp = None imp = None # Must use linear interpolation for subpixel precision affine = AffineTransform2D() affine.set(matrices[index]) imgI = Views.interpolate(Views.extendZero(img), NLinearInterpolatorFactory()) imgA = RealViews.transform(imgI, affine) imgT = Views.zeroMin(Views.interval(imgA, img)) # Convert to 8-bit imgMinMax = convert2(imgT, RealUnsignedByteConverter(minimum, maximum), UnsignedByteType, randomAccessible=False) # use IterableInterval aimg = ArrayImgs.unsignedBytes(Intervals.dimensionsAsLongArray(img)) # ImgUtil copies multi-threaded, which is not appropriate here as there are many other images being copied too #ImgUtil.copy(ImgView.wrap(imgMinMax, aimg.factory()), aimg) # Single-threaded copy copier = createBiConsumerTypeSet(UnsignedByteType) LoopBuilder.setImages(imgMinMax, aimg).forEachPixel(copier) img = imgI = imgA = imgMinMax = imgT = None return aimg blockRadius, n_bins, slope = CLAHE_params # A CacheLoader that interprets the list of filepaths as a 3D volume: a stack of 2D slices loader = SectionCellLoader( filepaths, asArrayImg=partial(asNormalizedUnsignedByteArrayImg, interval, invert, blockRadius, n_bins, slope, matrices), loadFn=loadFn) # How to preload block_size[2] files at a time? Or at least as many as numCPUs()? # One possibility is to query the SoftRefLoaderCache.map for its entries, using a ScheduledExecutorService, # and preload sections ahead for the whole blockSize[2] dimension. cachedCellImg = lazyCachedCellImg(loader, voldims, cell_dimensions, UnsignedByteType, BYTE) exe_preloader = newFixedThreadPool(n_threads=min( block_size[2], n5_threads if n5_threads > 0 else numCPUs()), name="preloader") def preload(cachedCellImg, loader, block_size, filepaths, exe): """ Find which is the last cell index in the cache, identify to which block (given the blockSize[2] AKA Z dimension) that index belongs to, and concurrently load all cells (sections) that the Z dimension of the blockSize will need. If they are already loaded, these operations are insignificant. """ try: # The SoftRefLoaderCache.map is a ConcurrentHashMap with Long keys, aka numbers cache = cachedCellImg.getCache() f1 = cache.getClass().getDeclaredField( "cache") # LoaderCacheAsCacheAdapter.cache f1.setAccessible(True) softCache = f1.get(cache) cache = None f2 = softCache.getClass().getDeclaredField( "map") # SoftRefLoaderCache.map f2.setAccessible(True) keys = sorted(f2.get(softCache).keySet()) if 0 == len(keys): return first = max(0, keys[-1] - (keys[-1] % block_size[2])) last = min(len(filepaths), first + block_size[2]) - 1 keys = None syncPrintQ("### Preloading %i-%i ###" % (first, last)) futures = [] for index in xrange(first, last + 1): futures.append( exe.submit(TimeItTask(softCache.get, index, loader))) softCache = None # Wait for all loaded_any = False count = 0 while len(futures) > 0: r, t = futures.pop(0).get() # waits for the image to load if t > 1000: # in miliseconds. Less than this is for sure a cache hit, more a cache miss and reload loaded_any = True r = None # t in miliseconds syncPrintQ("preloaded index %i in %f ms" % (first + count, t)) count += 1 if not loaded_any: syncPrintQ("Completed preloading %i-%i" % (first, first + block_size[2] - 1)) except: syncPrintQ(sys.exc_info()) preloader = Executors.newSingleThreadScheduledExecutor() preloader.scheduleWithFixedDelay( RunTask(preload, cachedCellImg, loader, block_size, filepaths, exe_preloader), 10, 60, TimeUnit.SECONDS) try: syncPrint("N5 directory: " + exportDir + "\nN5 dataset name: " + name + "\nN5 blockSize: " + str(block_size)) writeN5(cachedCellImg, exportDir, name, block_size, gzip_compression_level=gzip_compression, n_threads=n5_threads) finally: preloader.shutdown() exe_preloader.shutdown()
def run(): exe = newFixedThreadPool(min(len(cropped), numCPUs())) try: # Dummy for in-RAM reading of isotropic images img_filenames = [str(i) for i in xrange(len(cropped))] loader = InRAMLoader(dict(zip(img_filenames, cropped))) getCalibration = params.get("getCalibration", None) if not getCalibration: getCalibration = lambda img: [1.0] * cropped[0].numDimensions() csv_dir = params["csv_dir"] modelclass = params["modelclass"] # Matrices describing the registration on the basis of the cropped images matrices = computeOptimizedTransforms(img_filenames, loader, getCalibration, csv_dir, exe, modelclass, params) # Store outside, so they can be e.g. printed, and used beyond here for matrix, affine in zip(matrices, affines): affine.set(*matrix) # Combine the transforms: scaling (by calibration) # + the coarse registration (i.e. manual translations) # + the translation introduced by the ROI cropping # + the affine matrices computed above over the cropped images. coarse_matrices = [] for coarse_affine in coarse_affines: matrix = zeros(12, 'd') coarse_affine.toArray(matrix) coarse_matrices.append(matrix) # NOTE: both coarse_matrices and matrices are from the camera X to camera 0. No need to invert them. # NOTE: uses identity calibration because the coarse_matrices already include the calibration scaling to isotropy transforms = mergeTransforms([1.0, 1.0, 1.0], coarse_matrices, [minC, maxC], matrices, invert2=False) print "calibration:", [1.0, 1.0, 1.0] print "cmTransforms:\n %s\n %s\n %s\n %s" % tuple(str(m) for m in coarse_matrices) print "ROI", [minC, maxC] print "fineTransformsPostROICrop:\n %s\n %s\n %s\n %s" % tuple(str(m) for m in matrices) print "invert2:", False # Show registered images registered = [transformedView(img, transform, interval=cropped[0]) for img, transform in izip(original_images, transforms)] registered_imp = showAsStack(registered, title="Registered with %s" % params["modelclass"].getSimpleName()) registered_imp.setDisplayRange(cropped_imp.getDisplayRangeMin(), cropped_imp.getDisplayRangeMax()) """ # TEST: same as above, but without merging the transforms. WORKS, same result # Copy into ArrayImg, otherwise they are rather slow to browse def copy(img1, affine): # Copy in two steps. Otherwise the nearest neighbor interpolation on top of another # nearest neighbor interpolation takes a huge amount of time dimensions = Intervals.dimensionsAsLongArray(img1) aimg1 = ArrayImgs.unsignedShorts(dimensions) ImgUtil.copy(ImgView.wrap(img1, aimg1.factory()), aimg1) img2 = transformedView(aimg1, affine) aimg2 = ArrayImgs.unsignedShorts(dimensions) ImgUtil.copy(ImgView.wrap(img2, aimg2.factory()), aimg2) return aimg2 futures = [exe.submit(Task(copy, img, affine)) for img, affine in izip(cropped, affines)] aimgs = [f.get() for f in futures] showAsStack(aimgs, title="DEBUG Registered with %s" % params["modelclass"].getSimpleName()) """ except: print sys.exc_info() finally: exe.shutdown() SwingUtilities.invokeLater(lambda: run_button.setEnabled(True))