def testUsingPreloadedArryasWhenScriptingBatchProcessing(self):
args = ilastik_main.parse_args([])
args.headless = True
args.project = self.PROJECT_FILE
shell = ilastik_main.main(args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
# Obtain the training operator
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
# Sanity checks
assert len(opPixelClassification.InputImages) > 0
assert opPixelClassification.Classifier.ready()
input_data1 = numpy.random.randint(0, 255, (2, 20, 20, 5, 1)).astype(
numpy.uint8)
input_data2 = numpy.random.randint(0, 255, (2, 20, 20, 5, 1)).astype(
numpy.uint8)
role_data_dict = {
"Raw Data": [
PreloadedArrayDatasetInfo(preloaded_array=input_data1,
axistags=vigra.AxisTags("tzyxc")),
PreloadedArrayDatasetInfo(preloaded_array=input_data2,
axistags=vigra.AxisTags("tzyxc")),
]
}
predictions = shell.workflow.batchProcessingApplet.run_export(
role_data_dict, export_to_array=True)
for result in predictions:
assert result.shape == (2, 20, 20, 5, 2)
def generate_trained_project_file(new_project_path,
raw_data_paths,
label_data_paths,
feature_selections,
classifier_factory=None):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(
label_data_paths
), "Number of label images must match number of raw images."
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = ilastik_main.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args(
[], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(
data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue(classifier_factory)
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue(cwd)
opReader.FilePath.setValue(label_data_path)
print("Reading label volume: {}".format(label_data_path))
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape) - 1
label_volume = label_volume[..., None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print("Applying label volume to lane #{}".format(lane))
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][
entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = list(map(str, list(range(max_label_class))))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
def runWorkflow(cluster_args):
ilastik_main_args = ilastik_main.parse_args([])
# Copy relevant args from cluster cmdline options to ilastik_main cmdline options
ilastik_main_args.headless = True
ilastik_main_args.project = cluster_args.project
ilastik_main_args.process_name = cluster_args.process_name
# Nodes should not write to a common logfile.
# Override with /dev/null
if cluster_args._node_work_ is None:
ilastik_main_args.logfile = cluster_args.logfile
else:
ilastik_main_args.logfile = "/dev/null"
assert cluster_args.project is not None, "Didn't get a project file."
# Read the config file
configFilePath = cluster_args.option_config_file
config = parseClusterConfigFile(configFilePath)
# Update the monkey_patch settings
ilastik.monkey_patches.apply_setting_dict(config.__dict__)
# Configure the thread count.
# Nowadays, this is done via an environment variable setting for ilastik_main to detect.
if cluster_args._node_work_ is not None and config.task_threadpool_size is not None:
os.environ["LAZYFLOW_THREADS"] = str(config.task_threadpool_size)
if cluster_args._node_work_ is not None and config.task_total_ram_mb is not None:
os.environ["LAZYFLOW_TOTAL_RAM_MB"] = str(config.task_total_ram_mb)
# Instantiate 'shell' by calling ilastik_main with our
shell = ilastik_main.main(ilastik_main_args)
workflow = shell.projectManager.workflow
# Attach cluster operators
resultSlot = None
finalOutputSlot = workflow.getHeadlessOutputSlot(config.output_slot_id)
assert finalOutputSlot is not None
clusterOperator = None
try:
if cluster_args._node_work_ is not None:
clusterOperator, resultSlot = prepare_node_cluster_operator(
config, cluster_args, finalOutputSlot)
else:
clusterOperator, resultSlot = prepare_master_cluster_operator(
cluster_args, finalOutputSlot)
# Get the result
logger.info("Starting task")
result = resultSlot[
0].value # FIXME: The image index is hard-coded here.
finally:
logger.info("Cleaning up")
global stop_background_tasks
stop_background_tasks = True
try:
if clusterOperator is not None:
clusterOperator.cleanUp()
except:
logger.error("Errors during cleanup.")
try:
logger.info("Closing project...")
shell.closeCurrentProject()
except:
logger.error("Errors while closing project.")
logger.info("FINISHED with result {}".format(result))
if not result:
logger.error("FAILED TO COMPLETE!")
def runWorkflow(cluster_args):
ilastik_main_args = ilastik_main.parse_args([])
# Copy relevant args from cluster cmdline options to ilastik_main cmdline options
ilastik_main_args.headless = True
ilastik_main_args.project = cluster_args.project
ilastik_main_args.process_name = cluster_args.process_name
# Nodes should not write to a common logfile.
# Override with /dev/null
if cluster_args._node_work_ is None:
ilastik_main_args.logfile = cluster_args.logfile
else:
ilastik_main_args.logfile = "/dev/null"
assert cluster_args.project is not None, "Didn't get a project file."
# Read the config file
configFilePath = cluster_args.option_config_file
config = parseClusterConfigFile( configFilePath )
# Update the monkey_patch settings
ilastik.monkey_patches.apply_setting_dict( config.__dict__ )
# Configure the thread count.
# Nowadays, this is done via an environment variable setting for ilastik_main to detect.
if cluster_args._node_work_ is not None and config.task_threadpool_size is not None:
os.environ["LAZYFLOW_THREADS"] = str(config.task_threadpool_size)
if cluster_args._node_work_ is not None and config.task_total_ram_mb is not None:
os.environ["LAZYFLOW_TOTAL_RAM_MB"] = str(config.task_total_ram_mb)
# Instantiate 'shell' by calling ilastik_main with our
shell = ilastik_main.main( ilastik_main_args )
workflow = shell.projectManager.workflow
# Attach cluster operators
resultSlot = None
finalOutputSlot = workflow.getHeadlessOutputSlot( config.output_slot_id )
assert finalOutputSlot is not None
clusterOperator = None
try:
if cluster_args._node_work_ is not None:
clusterOperator, resultSlot = prepare_node_cluster_operator(config, cluster_args, finalOutputSlot)
else:
clusterOperator, resultSlot = prepare_master_cluster_operator(cluster_args, finalOutputSlot)
# Get the result
logger.info("Starting task")
result = resultSlot[0].value # FIXME: The image index is hard-coded here.
finally:
logger.info("Cleaning up")
global stop_background_tasks
stop_background_tasks = True
try:
if clusterOperator is not None:
clusterOperator.cleanUp()
except:
logger.error("Errors during cleanup.")
try:
logger.info("Closing project...")
shell.closeCurrentProject()
except:
logger.error("Errors while closing project.")
logger.info("FINISHED with result {}".format(result))
if not result:
logger.error( "FAILED TO COMPLETE!" )
def generate_untrained_project_file(
new_project_path, raw_data_paths, feature_selections, label_names
):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
labels: list of label names
"""
import ilastik_main as app
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from ilastik.applets.dataSelection.opDataSelection import RelativeFilesystemDatasetInfo
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = app.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = app.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE FILE PATHS
##
data_selection_applet = shell.workflow.dataSelectionApplet
input_infos = [RelativeFilesystemDatasetInfo(filePath=path) for path
in raw_data_paths]
opDataSelection = data_selection_applet.topLevelOperator
existing_lanes = len(opDataSelection.DatasetGroup)
opDataSelection.DatasetGroup.resize(max(len(input_infos), existing_lanes))
# Not sure if assuming role_index = 0 is allways valid
role_index = 0
for lane_index, info in enumerate(input_infos):
if info:
opDataSelection.DatasetGroup[lane_index][role_index].setValue(info)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
##
## READ/APPLY LABEL VOLUMES
##
opPixelClassification.LabelNames.setValue(label_names)
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
from collections import OrderedDict import numpy import vigra import ilastik_main from ilastik.applets.dataSelection import DatasetInfo, PreloadedArrayDatasetInfo from ilastik.workflows.pixelClassification import PixelClassificationWorkflow # Before we start ilastik, optionally prepare these environment variable settings. os.environ["LAZYFLOW_THREADS"] = "2" os.environ["LAZYFLOW_TOTAL_RAM_MB"] = "2000" # Programmatically set the command-line arguments directly into the argparse.Namespace object # Provide your project file, and don't forget to specify headless. args = ilastik_main.parse_args([]) args.headless = True args.project = "/Users/bergs/MyProject.ilp" # REPLACE WITH YOUR PROJECT FILE # Instantiate the 'shell', (in this case, an instance of ilastik.shell.HeadlessShell) # This also loads the project file into shell.projectManager shell = ilastik_main.main(args) assert isinstance(shell.workflow, PixelClassificationWorkflow) # Obtain the training operator opPixelClassification = shell.workflow.pcApplet.topLevelOperator # Sanity checks assert len(opPixelClassification.InputImages) > 0 assert opPixelClassification.Classifier.ready()
from collections import OrderedDict import numpy import vigra import ilastik_main from ilastik.applets.dataSelection import DatasetInfo from ilastik.workflows.pixelClassification import PixelClassificationWorkflow # Before we start ilastik, optionally prepare these environment variable settings. os.environ["LAZYFLOW_THREADS"] = "2" os.environ["LAZYFLOW_TOTAL_RAM_MB"] = "2000" # Programmatically set the command-line arguments directly into the argparse.Namespace object # Provide your project file, and don't forget to specify headless. args = ilastik_main.parse_args([]) args.headless = True args.project = '/Users/bergs/MyProject.ilp' # REPLACE WITH YOUR PROJECT FILE # Instantiate the 'shell', (in this case, an instance of ilastik.shell.HeadlessShell) # This also loads the project file into shell.projectManager shell = ilastik_main.main( args ) assert isinstance(shell.workflow, PixelClassificationWorkflow) # Obtain the training operator opPixelClassification = shell.workflow.pcApplet.topLevelOperator # Sanity checks assert len(opPixelClassification.InputImages) > 0 assert opPixelClassification.Classifier.ready()
def generate_trained_loo_project_file(
project_path,
name_loo_img
):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.roi import fullSlicing
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parse_args([])
ilastik_args.project = project_path
ilastik_args.headless = True
ilastik_args.readonly = False
shell = ilastik_main.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE FILE PATHS
##
data_selection_applet = shell.workflow.dataSelectionApplet
opDataSelection = data_selection_applet.topLevelOperator
existing_lanes = len(opDataSelection.DatasetGroup)
# Not sure if assuming role_index = 0 is allways valid
role_index = 0
cur_lane = None
for lane, dataset in enumerate(opDataSelection.DatasetGroup):
dat = dataset[role_index][0].wait()[0]
if dat.nickname == name_loo_img:
cur_lane = lane
break
if cur_lane is None:
raise ValueError(f'{name_loo_img} not found in project.')
# Set delete the label fro this image by setting all labels to 0
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
#label_input_slot = opPixelClassification.LabelInputs[cur_lane]
#label_output_slot = opPixelClassification.LabelImages[cur_lane]
#shape = label_output_slot.meta.shape
#zero_labels = np.zeros(shape=shape, dtype=np.uint8)
#label_input_slot[fullSlicing(shape)] = zero_labels
#label_input_slot.setDirty()
#label_output_slot.disconnect()
#label_output_slot.setValue(zero_labels)
#label_output_slot.setDirty()
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Mark the classifier as dirty to force re-training it
cur_labs = opPixelClassification.opTrain.Labels[cur_lane]
up_lab=cur_labs.upstream_slot.upstream_slot.upstream_slot
zero_labels = np.zeros(shape=up_lab.meta.shape, dtype=np.uint8)
up_lab.setValue(zero_labels)
up_lab.setDirty()
#cur_labs.disconnect()
#cur_labs.value[:] = 0
#opPixelClassification.opTrain.ClassifierFactory.setDirty()
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=True)
def generate_trained_project_file( new_project_path,
raw_data_paths,
label_data_paths,
feature_selections,
classifier_factory=None ):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(label_data_paths), \
"Number of label images must match number of raw images."
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = 'Pixel Classification'
shell = ilastik_main.main( ilastik_args )
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args([], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue( ScalesList )
opFeatures.FeatureIds.setValue( FeatureIds )
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue( classifier_factory )
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue( cwd )
opReader.FilePath.setValue( label_data_path )
print("Reading label volume: {}".format( label_data_path ))
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape)-1
label_volume = label_volume[...,None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print("Applying label volume to lane #{}".format(lane))
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = list(map(str, list(range(max_label_class))))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
