def testUsingPreloadedArryasWhenScriptingBatchProcessing(self):
args = app.parse_args([])
args.headless = True
args.project = self.PROJECT_FILE
shell = app.main(args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
# Obtain the training operator
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
# Sanity checks
assert len(opPixelClassification.InputImages) > 0
assert opPixelClassification.Classifier.ready()
input_data1 = numpy.random.randint(0, 255, (2, 20, 20, 5, 1)).astype(numpy.uint8)
input_data2 = numpy.random.randint(0, 255, (2, 20, 20, 5, 1)).astype(numpy.uint8)
role_data_dict = {
"Raw Data": [
PreloadedArrayDatasetInfo(preloaded_array=input_data1, axistags=vigra.AxisTags("tzyxc")),
PreloadedArrayDatasetInfo(preloaded_array=input_data2, axistags=vigra.AxisTags("tzyxc")),
]
}
predictions = shell.workflow.batchProcessingApplet.run_export(role_data_dict, export_to_array=True)
for result in predictions:
assert result.shape == (2, 20, 20, 5, 2)
def analyze(conn, images, model, new_dataset, extension=".tar", resolution=0):
# Prepare ilastik
# temporary directory where to download files
path = tempfile.mkdtemp()
if not os.path.exists(path):
os.makedirs(path)
os.environ["LAZYFLOW_THREADS"] = "2"
os.environ["LAZYFLOW_TOTAL_RAM_MB"] = "2000"
args = app.parse_args([])
args.headless = True
args.project = model
args.readonly = True
shell = app.main(args)
start = time.time()
for image in images:
input_data = load_from_s3(image, path)
# run ilastik headless
print('running ilastik using %s and %s' % (model, image.getName()))
data = OrderedDict([(
"Raw Data",
[PreloadedArrayDatasetInfo(preloaded_array=input_data)],
)])
shell.workflow.batchProcessingApplet.run_export(
data, export_to_array=True) # noqa
elapsed = time.time() - start
print(elapsed)
def analyze(image_id, model):
args = app.parse_args([])
args.headless = True
args.project = model
args.readonly = True
shell = app.main(args)
input_data = load_from_s3(image_id)
# run ilastik headless
data = [{
"Raw Data":
PreloadedArrayDatasetInfo(preloaded_array=input_data,
axistags=vigra.defaultAxistags("tzyxc"))
}] # noqa
return shell.workflow.batchProcessingApplet.run_export(
data, export_to_array=True) # noqa
def analyze(conn, images, model, new_dataset):
# Prepare ilastik
os.environ["LAZYFLOW_THREADS"] = "2"
os.environ["LAZYFLOW_TOTAL_RAM_MB"] = "2000"
args = app.parse_args([])
args.headless = True
args.project = model
shell = app.main(args)
for image in images:
input_data = load_numpy_array(image)
# run ilastik headless
print('running ilastik using %s and %s' % (model, image.getName()))
data = [ {"Raw Data": PreloadedArrayDatasetInfo(preloaded_array=input_data, axistags=vigra.defaultAxistags("tzyxc"))}] # noqa
predictions = shell.workflow.batchProcessingApplet.run_export(data,
export_to_array=True) # noqa
for d in predictions:
save_results(conn, image, d, new_dataset)
def generate_trained_project_file(new_project_path,
raw_data_paths,
label_data_paths,
feature_selections,
classifier_factory=None):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(
label_data_paths
), "Number of label images must match number of raw images."
from ilastik import app
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = app.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = app.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args(
[], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(
data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue(classifier_factory)
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue(cwd)
opReader.FilePath.setValue(label_data_path)
print("Reading label volume: {}".format(label_data_path))
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape) - 1
label_volume = label_volume[..., None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print("Applying label volume to lane #{}".format(lane))
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][
entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = list(map(str, list(range(max_label_class))))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
def runWorkflow(cluster_args):
ilastik_main_args = app.parse_args([])
# Copy relevant args from cluster cmdline options to app cmdline options
ilastik_main_args.headless = True
ilastik_main_args.project = cluster_args.project
ilastik_main_args.process_name = cluster_args.process_name
# Nodes should not write to a common logfile.
# Override with /dev/null
if cluster_args._node_work_ is None:
ilastik_main_args.logfile = cluster_args.logfile
else:
ilastik_main_args.logfile = "/dev/null"
assert cluster_args.project is not None, "Didn't get a project file."
# Read the config file
configFilePath = cluster_args.option_config_file
config = parseClusterConfigFile(configFilePath)
# Update the monkey_patch settings
ilastik.monkey_patches.apply_setting_dict(config.__dict__)
# Configure the thread count.
# Nowadays, this is done via an environment variable setting for app to detect.
if cluster_args._node_work_ is not None and config.task_threadpool_size is not None:
os.environ["LAZYFLOW_THREADS"] = str(config.task_threadpool_size)
if cluster_args._node_work_ is not None and config.task_total_ram_mb is not None:
os.environ["LAZYFLOW_TOTAL_RAM_MB"] = str(config.task_total_ram_mb)
# Instantiate 'shell' by calling app with our
shell = app.main(ilastik_main_args)
workflow = shell.projectManager.workflow
# Attach cluster operators
resultSlot = None
finalOutputSlot = workflow.getHeadlessOutputSlot(config.output_slot_id)
assert finalOutputSlot is not None
clusterOperator = None
try:
if cluster_args._node_work_ is not None:
clusterOperator, resultSlot = prepare_node_cluster_operator(
config, cluster_args, finalOutputSlot)
else:
clusterOperator, resultSlot = prepare_master_cluster_operator(
cluster_args, finalOutputSlot)
# Get the result
logger.info("Starting task")
result = resultSlot[
0].value # FIXME: The image index is hard-coded here.
finally:
logger.info("Cleaning up")
global stop_background_tasks
stop_background_tasks = True
try:
if clusterOperator is not None:
clusterOperator.cleanUp()
except:
logger.error("Errors during cleanup.")
try:
logger.info("Closing project...")
shell.closeCurrentProject()
except:
logger.error("Errors while closing project.")
logger.info("FINISHED with result {}".format(result))
if not result:
logger.error("FAILED TO COMPLETE!")
from collections import OrderedDict import numpy import vigra from ilastik import app from ilastik.applets.dataSelection import DatasetInfo, PreloadedArrayDatasetInfo from ilastik.workflows.pixelClassification import PixelClassificationWorkflow # Before we start ilastik, optionally prepare these environment variable settings. os.environ["LAZYFLOW_THREADS"] = "2" os.environ["LAZYFLOW_TOTAL_RAM_MB"] = "2000" # Programmatically set the command-line arguments directly into the argparse.Namespace object # Provide your project file, and don't forget to specify headless. args = app.parse_args([]) args.headless = True args.project = "/Users/bergs/MyProject.ilp" # REPLACE WITH YOUR PROJECT FILE # Instantiate the 'shell', (in this case, an instance of ilastik.shell.HeadlessShell) # This also loads the project file into shell.projectManager shell = app.main(args) assert isinstance(shell.workflow, PixelClassificationWorkflow) # Obtain the training operator opPixelClassification = shell.workflow.pcApplet.topLevelOperator # Sanity checks assert len(opPixelClassification.InputImages) > 0 assert opPixelClassification.Classifier.ready()
