def main():
parser = argparse.ArgumentParser()
parser.add_argument('grayscale', help='example: my-grayscale.h5/volume')
parser.add_argument('classifier', help='example: my-file.h5/forest')
parser.add_argument('filter_specs',
help='json file containing filter list')
parser.add_argument('output_path',
help='example: my-predictions.h5/volume')
parser.add_argument('--compute-blockwise',
help='Compute blockwise instead of as a whole',
action='store_true')
parser.add_argument('--thread-count',
help='The threadpool size',
default=0,
type=int)
args = parser.parse_args()
# Show log messages on the console.
logger.setLevel(logging.INFO)
logger.addHandler(logging.StreamHandler(sys.stdout))
Request.reset_thread_pool(args.thread_count)
load_and_predict(args.grayscale, args.classifier, args.filter_specs,
args.output_path, args.compute_blockwise)
logger.info("DONE.")
def teardown_method(self, method):
# reset cleanup frequency to sane value
# reset max memory
Memory.setAvailableRamCaches(-1)
mgr = _CacheMemoryManager()
mgr.setRefreshInterval(default_refresh_interval)
mgr.enable()
Request.reset_thread_pool()
def teardown_method(self, method):
# reset cleanup frequency to sane value
# reset max memory
Memory.setAvailableRamCaches(-1)
mgr = CacheMemoryManager()
mgr.setRefreshInterval(default_refresh_interval)
mgr.enable()
Request.reset_thread_pool()
def test_RequestLock():
assert Request.global_thread_pool.num_workers > 0, \
"This test must be used with the real threadpool."
lockA = RequestLock()
lockB = RequestLock()
def log_request_system_status():
status = (
"*************************\n" +
'lockA.pending: {}\n'.format(len(lockA._pendingRequests)) +
'lockB.pending: {}\n'.format(len(lockB._pendingRequests))
#+ "suspended Requests: {}\n".format( len(Request.global_suspend_set) )
+ "global job queue: {}\n".format(
len(Request.global_thread_pool.unassigned_tasks)))
for worker in Request.global_thread_pool.workers:
status += "{} queued tasks: {}\n".format(worker.name,
len(worker.job_queue))
status += "*****************************************************"
logger.debug(status)
running = [True]
def periodic_status():
while running[0]:
time.sleep(0.5)
log_request_system_status()
# Uncomment these lines to print periodic status while the test runs...
status_thread = threading.Thread(target=periodic_status)
status_thread.daemon = True
status_thread.start()
try:
_impl_test_lock(lockA, lockB, Request, 1000)
except:
log_request_system_status()
running[0] = False
status_thread.join()
global paused
paused = False
Request.reset_thread_pool(Request.global_thread_pool.num_workers)
if lockA.locked():
lockA.release()
if lockB.locked():
lockB.release()
raise
log_request_system_status()
running[0] = False
status_thread.join()
def setup(self):
super(SkeletonAssociationProcess, self).setup()
self.inner_logger.debug(
'Setting up opPixelClassification and multicut_shell...')
# todo: replace opPixelClassification with catpy tile-getter
self.opPixelClassification, self.multicut_shell = setup_classifier_and_multicut(
*self.setup_args)
self.inner_logger.debug(
'opPixelClassification and multicut_shell set up')
Request.reset_thread_pool(1)
def test_RequestLock():
assert Request.global_thread_pool.num_workers > 0, "This test must be used with the real threadpool."
lockA = RequestLock()
lockB = RequestLock()
def log_request_system_status():
status = (
"*************************\n"
+ "lockA.pending: {}\n".format(len(lockA._pendingRequests))
+ "lockB.pending: {}\n".format(len(lockB._pendingRequests))
# + "suspended Requests: {}\n".format( len(Request.global_suspend_set) )
+ "global job queue: {}\n".format(len(Request.global_thread_pool.unassigned_tasks))
)
for worker in Request.global_thread_pool.workers:
status += "{} queued tasks: {}\n".format(worker.name, len(worker.job_queue))
status += "*****************************************************"
logger.debug(status)
running = [True]
def periodic_status():
while running[0]:
time.sleep(0.5)
log_request_system_status()
# Uncomment these lines to print periodic status while the test runs...
status_thread = threading.Thread(target=periodic_status)
status_thread.daemon = True
status_thread.start()
try:
_impl_test_lock(lockA, lockB, Request, 1000)
except:
log_request_system_status()
running[0] = False
status_thread.join()
global paused
paused = False
Request.reset_thread_pool(Request.global_thread_pool.num_workers)
if lockA.locked():
lockA.release()
if lockB.locked():
lockB.release()
raise
log_request_system_status()
running[0] = False
status_thread.join()
def main():
parser = argparse.ArgumentParser()
parser.add_argument('grayscale', help='example: my-grayscale.h5/volume')
parser.add_argument('classifier', help='example: my-file.h5/forest')
parser.add_argument('filter_specs', help='json file containing filter list')
parser.add_argument('output_path', help='example: my-predictions.h5/volume')
parser.add_argument('--compute-blockwise', help='Compute blockwise instead of as a whole', action='store_true')
parser.add_argument('--thread-count', help='The threadpool size', default=0, type=int)
args = parser.parse_args()
# Show log messages on the console.
logger.setLevel(logging.INFO)
logger.addHandler( logging.StreamHandler(sys.stdout) )
Request.reset_thread_pool(args.thread_count)
load_and_predict( args.grayscale, args.classifier, args.filter_specs, args.output_path, args.compute_blockwise )
logger.info("DONE.")
def ilastik_simple_predict(gray_vol, mask, classifier_path, filter_specs_path, selected_channels=None, normalize=True,
LAZYFLOW_THREADS=0, LAZYFLOW_TOTAL_RAM_MB=None, logfile="/dev/null"):
"""
gray_vol: A 3D numpy array with axes zyx
mask: A binary image where 0 means "no prediction necessary".
'None' can be given, which means "predict everything".
classifier_path: Path to a vigra RandomForest classifier, in HDF5.
Example: /path/to/myclassifier.h5/classifiers/my_rf
filter_specs_path: Path to "filter specs" json file. The json structure is like this:
[ ['GaussianSmoothing', 0.3],
['GaussianSmoothing', 0.7],
['LaplacianOfGaussian', 1.6] ]
(See ilastik's simple_predict.py for valid filter names.)
selected_channels: A list of channel indexes to select and return from the prediction results.
'None' can also be given, which means "return all prediction channels".
You may also return a *nested* list, in which case groups of channels can be
combined (summed) into their respective output channels.
For example: selected_channels=[0,3,[2,4],7] means the output will have 4 channels:
0,3,2+4,7 (channels 5 and 6 are simply dropped).
normalize: Renormalize all outputs so the channels sum to 1 everywhere.
That is, (predictions.sum(axis=-1) == 1.0).all()
Note: Pixels with 0.0 in all channels will be simply given a value of 1/N in all channels.
LAZYFLOW_THREADS, LAZYFLOW_TOTAL_RAM_MB: Same meanings as in ilastik_predict_with_array().
(although we have to configure them in a different way)
"""
print("ilastik_simple_predict(): Starting with raw data: dtype={}, shape={}".format(str(gray_vol.dtype), gray_vol.shape))
import os
from collections import OrderedDict
import uuid
import platform
import vigra
from ilastik.utility.simple_predict import load_and_predict
from lazyflow.request import Request
print("ilastik_simple_predict(): Done with imports")
_prepare_lazyflow_config(LAZYFLOW_THREADS, LAZYFLOW_TOTAL_RAM_MB, 10)
Request.reset_thread_pool(LAZYFLOW_THREADS)
# The process_name argument is prefixed to all log messages.
# For now, just use the machine name and a uuid
# FIXME: It would be nice to provide something more descriptive, like the ROI of the current spark job...
process_name = platform.node() + "-" + str(uuid.uuid1())
# To avoid conflicts between processes, give each process it's own logfile to write to.
if logfile != "/dev/null":
base, ext = os.path.splitext(logfile)
logfile = base + '.' + process_name + ext
_init_logging(logfile, process_name)
# Construct an OrderedDict of role-names -> DatasetInfos
# (See PixelClassificationWorkflow.ROLE_NAMES)
raw_data_array = vigra.taggedView(gray_vol, 'zyx')
print("ilastik_simple_predict(): Starting export...")
predictions = load_and_predict( raw_data_array, classifier_path, filter_specs_path, compute_blockwise=True )
selected_predictions = select_channels(predictions, selected_channels)
if normalize:
normalize_channels_in_place(selected_predictions)
return selected_predictions
def testBadMemoryConditions(self):
"""
TestCacheMemoryManager.testBadMemoryConditions
This test is a proof of the proposition in
https://github.com/ilastik/lazyflow/issue/185
which states that, given certain memory constraints, the cache
cleanup strategy in use is inefficient. An advanced strategy
should pass the test.
"""
mgr = _CacheMemoryManager()
mgr.setRefreshInterval(0.01)
mgr.enable()
d = 2
tags = "xy"
shape = (999,) * d
blockshape = (333,) * d
# restrict memory for computation to one block (including fudge
# factor 2 of bigRequestStreamer)
cacheMem = np.prod(shape)
Memory.setAvailableRam(np.prod(blockshape) * 2 + cacheMem)
# restrict cache memory to the whole volume
Memory.setAvailableRamCaches(cacheMem)
# to ease observation, do everything single threaded
Request.reset_thread_pool(num_workers=1)
x = np.zeros(shape, dtype=np.uint8)
x = vigra.taggedView(x, axistags=tags)
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
pipe.Input.setValue(x)
pipe.Output.meta.ideal_blockshape = blockshape
# simulate BlockedArrayCache behaviour without caching
# cache = OpSplitRequestsBlockwise(True, graph=g)
# cache.BlockShape.setValue(blockshape)
# cache.Input.connect(pipe.Output)
cache = OpBlockedArrayCache(graph=g)
cache.Input.connect(pipe.Output)
cache.BlockShape.setValue(blockshape)
op = OpEnlarge(graph=g)
op.Input.connect(cache.Output)
split = OpSplitRequestsBlockwise(True, graph=g)
split.BlockShape.setValue(blockshape)
split.Input.connect(op.Output)
streamer = BigRequestStreamer(split.Output, [(0,) * len(shape), shape])
streamer.execute()
# in the worst case, we have 4*4 + 4*6 + 9 = 49 requests to pipe
# in the best case, we have 9
np.testing.assert_equal(pipe.accessCount, 9)
def setup(self):
super(SynapseDetectionProcess, self).setup()
self.opPixelClassification = setup_classifier(*self.setup_args)
Request.reset_thread_pool(0) # todo: set to 0?
def testBadMemoryConditions(self):
"""
TestCacheMemoryManager.testBadMemoryConditions
This test is a proof of the proposition in
https://github.com/ilastik/lazyflow/issue/185
which states that, given certain memory constraints, the cache
cleanup strategy in use is inefficient. An advanced strategy
should pass the test.
"""
mgr = CacheMemoryManager()
mgr.setRefreshInterval(0.01)
mgr.enable()
d = 2
tags = "xy"
shape = (999,) * d
blockshape = (333,) * d
# restrict memory for computation to one block (including fudge
# factor 2 of bigRequestStreamer)
cacheMem = np.prod(shape)
Memory.setAvailableRam(np.prod(blockshape) * 2 + cacheMem)
# restrict cache memory to the whole volume
Memory.setAvailableRamCaches(cacheMem)
# to ease observation, do everything single threaded
Request.reset_thread_pool(num_workers=1)
x = np.zeros(shape, dtype=np.uint8)
x = vigra.taggedView(x, axistags=tags)
g = Graph()
pipe = OpArrayPiperWithAccessCount(graph=g)
pipe.Input.setValue(x)
pipe.Output.meta.ideal_blockshape = blockshape
# simulate BlockedArrayCache behaviour without caching
# cache = OpSplitRequestsBlockwise(True, graph=g)
# cache.BlockShape.setValue(blockshape)
# cache.Input.connect(pipe.Output)
cache = OpBlockedArrayCache(graph=g)
cache.Input.connect(pipe.Output)
cache.BlockShape.setValue(blockshape)
op = OpEnlarge(graph=g)
op.Input.connect(cache.Output)
split = OpSplitRequestsBlockwise(True, graph=g)
split.BlockShape.setValue(blockshape)
split.Input.connect(op.Output)
streamer = BigRequestStreamer(split.Output, [(0,) * len(shape), shape])
streamer.execute()
# in the worst case, we have 4*4 + 4*6 + 9 = 49 requests to pipe
# in the best case, we have 9
np.testing.assert_equal(pipe.accessCount, 9)
