def updateWorkingDirectory(self,newpath,oldpath):
newdir = PathComponents(newpath).externalDirectory
olddir = PathComponents(oldpath).externalDirectory
if newdir==olddir:
return
# Disconnect the working directory while we make these changes.
# All the changes will take effect when we set the new working directory.
self.topLevelOperator.WorkingDirectory.disconnect()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if not slot.ready():
# Skip if there is no dataset in this lane/role combination yet.
continue
datasetInfo = slot.value
if datasetInfo.location == DatasetInfo.Location.FileSystem:
#construct absolute path and recreate relative to the new path
fp = PathComponents(datasetInfo.filePath,olddir).totalPath()
abspath,relpath = getPathVariants(fp,newdir)
# Same convention as in dataSelectionGui:
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([fp, abspath])) > 1:
datasetInfo.filePath = relpath
else:
datasetInfo.filePath = abspath
slot.setValue(datasetInfo, check_changed=False)
self.topLevelOperator.WorkingDirectory.setValue(newdir)
self._projectFilePath = newdir
def updateWorkingDirectory(self,newpath,oldpath):
newpath,oldpath = PathComponents(newpath).externalDirectory,PathComponents(oldpath).externalDirectory
if newpath==oldpath:
return
for slot in self.topLevelOperator.Dataset:
datasetInfo = slot.value
if datasetInfo.location == DatasetInfo.Location.FileSystem:
#construct absolute path and recreate relative to the new path
fp = PathComponents(datasetInfo.filePath,oldpath).totalPath()
abspath,relpath = getPathVariants(fp,newpath)
# Same convention as in dataSelectionGui:
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([fp, abspath])) > 1: datasetInfo.filePath = relpath
else:datasetInfo.filePath = abspath
slot.setValue(datasetInfo)
self.topLevelOperator.WorkingDirectory.setValue(newpath)
self._projectFilePath = newpath
for slot in self.topLevelOperator.Dataset:
self.topLevelOperator.applet._gui.updateTableForSlot(slot = slot)
def updateInternalPathComboBox( self, row, externalPath, internalPath ):
with Tracer(traceLogger):
combo = QComboBox()
datasetNames = []
# Make sure we're dealing with the absolute path (to make this simple)
absPath, relPath = getPathVariants(externalPath, self.mainOperator.WorkingDirectory.value)
ext = os.path.splitext(absPath)[1]
h5Exts = ['.ilp', '.h5', '.hdf5']
if ext in h5Exts:
datasetNames = self.getPossibleInternalPaths(absPath)
# Add each dataset option to the combo
for path in datasetNames:
combo.addItem( path )
# If the internal path we used previously is in the combo list, select it.
prevSelection = combo.findText( internalPath )
if prevSelection != -1:
combo.setCurrentIndex( prevSelection )
# Define response to changes and add it to the GUI.
# Pass in the corresponding the table item so we can figure out which row this came from
combo.currentIndexChanged.connect( bind(self.handleComboSelectionChanged, combo) )
self.fileInfoTableWidget.setCellWidget( row, Column.InternalID, combo )
def addFileNames(self, fileNames):
"""
Add the given filenames to both the GUI table and the top-level operator inputs.
"""
with Tracer(traceLogger):
# Allocate additional subslots in the operator inputs.
oldNumFiles = len(self.mainOperator.Dataset)
self.mainOperator.Dataset.resize( oldNumFiles+len(fileNames) )
# Assign values to the new inputs we just allocated.
# The GUI will be updated by callbacks that are listening to slot changes
for i, filePath in enumerate(fileNames):
datasetInfo = DatasetInfo()
cwd = self.mainOperator.WorkingDirectory.value
absPath, relPath = getPathVariants(filePath, cwd)
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([cwd, absPath])) > 1:
datasetInfo.filePath = relPath
else:
datasetInfo.filePath = absPath
h5Exts = ['.ilp', '.h5', '.hdf5']
if os.path.splitext(datasetInfo.filePath)[1] in h5Exts:
datasetNames = self.getPossibleInternalPaths( absPath )
if len(datasetNames) > 0:
datasetInfo.filePath += str(datasetNames[0])
else:
raise RuntimeError("HDF5 file has no image datasets")
# Allow labels by default if this gui isn't being used for batch data.
datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal )
self.mainOperator.Dataset[i+oldNumFiles].setValue( datasetInfo )
def updateInternalPathComboBox( self, row, externalPath, internalPath ):
with Tracer(traceLogger):
combo = QComboBox()
datasetNames = []
# Make sure we're dealing with the absolute path (to make this simple)
absPath, relPath = getPathVariants(externalPath, self.mainOperator.WorkingDirectory.value)
ext = os.path.splitext(absPath)[1]
h5Exts = ['.ilp', '.h5', '.hdf5']
if ext in h5Exts:
# Open the file as a read-only so we can get a list of the internal paths
f = h5py.File(absPath, 'r')
# Define a closure to collect all of the dataset names in the file.
def accumulateDatasetPaths(name, val):
if type(val) == h5py._hl.dataset.Dataset and 3 <= len(val.shape) <= 5:
datasetNames.append( '/' + name )
# Visit every group/dataset in the file
f.visititems(accumulateDatasetPaths)
f.close()
# Add each dataset option to the combo
for path in datasetNames:
combo.addItem( path )
# If the internal path we used previously is in the combo list, select it.
prevSelection = combo.findText( internalPath )
if prevSelection != -1:
combo.setCurrentIndex( prevSelection )
# Define response to changes and add it to the GUI.
# Pass in the corresponding the table item so we can figure out which row this came from
combo.currentIndexChanged.connect( bind(self.handleComboSelectionChanged, combo) )
self.fileInfoTableWidget.setCellWidget( row, Column.InternalID, combo )
def updateFilePath(self, index):
"""
Update the operator's filePath input to match the gui
"""
with Tracer(traceLogger):
oldLocationSetting = self.mainOperator.Dataset[index].value.location
# Get the directory by inspecting the original operator path
oldTotalPath = self.mainOperator.Dataset[index].value.filePath
# Split into directory, filename, extension, and internal path
lastDotIndex = oldTotalPath.rfind('.')
extensionAndInternal = oldTotalPath[lastDotIndex:]
extension = extensionAndInternal.split('/')[0]
oldFilePath = oldTotalPath[:lastDotIndex] + extension
fileNameText = str(self.fileInfoTableWidget.item(index, Column.Name).text())
internalPathCombo = self.fileInfoTableWidget.cellWidget(index, Column.InternalID)
#internalPath = str(self.fileInfoTableWidget.item(index, Column.InternalID).text())
internalPath = str(internalPathCombo.currentText())
directory = os.path.split(oldFilePath)[0]
newFileNamePath = fileNameText
if directory != '':
newFileNamePath = directory + '/' + fileNameText
newTotalPath = newFileNamePath
if internalPath != '':
if internalPath[0] != '/':
newTotalPath += '/'
newTotalPath += internalPath
cwd = self.mainOperator.WorkingDirectory.value
absTotalPath, relTotalPath = getPathVariants( newTotalPath, cwd )
# Check the location setting
locationCombo = self.fileInfoTableWidget.cellWidget(index, Column.Location)
comboIndex = locationCombo.currentIndex()
newLocationSelection = locationCombo.itemData(comboIndex).toInt()[0] # In PyQt, toInt() returns a tuple
if newLocationSelection == LocationOptions.Project:
newLocationSetting = DatasetInfo.Location.ProjectInternal
elif newLocationSelection == LocationOptions.AbsolutePath:
newLocationSetting = DatasetInfo.Location.FileSystem
newTotalPath = absTotalPath
elif newLocationSelection == LocationOptions.RelativePath:
newLocationSetting = DatasetInfo.Location.FileSystem
newTotalPath = relTotalPath
if newTotalPath != oldTotalPath or newLocationSetting != oldLocationSetting:
# Be sure to copy so the slot notices the change when we setValue()
datasetInfo = copy.copy(self.mainOperator.Dataset[index].value)
datasetInfo.filePath = newTotalPath
datasetInfo.location = newLocationSetting
# TODO: First check to make sure this file exists!
self.mainOperator.Dataset[index].setValue( datasetInfo )
# Update the storage option combo to show the new path
self.updateStorageOptionComboBox(index, newFileNamePath)
def addFileNames(self, fileNames):
"""
Add the given filenames to both the GUI table and the top-level operator inputs.
"""
with Tracer(traceLogger):
# Allocate additional subslots in the operator inputs.
oldNumFiles = len(self.mainOperator.Dataset)
self.mainOperator.Dataset.resize(oldNumFiles + len(fileNames))
# Assign values to the new inputs we just allocated.
# The GUI will be updated by callbacks that are listening to slot changes
for i, filePath in enumerate(fileNames):
datasetInfo = DatasetInfo()
cwd = self.mainOperator.WorkingDirectory.value
absPath, relPath = getPathVariants(filePath, cwd)
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([cwd, absPath])) > 1:
datasetInfo.filePath = relPath
else:
datasetInfo.filePath = absPath
# Allow labels by default if this gui isn't being used for batch data.
datasetInfo.allowLabels = (self.guiMode == GuiMode.Normal)
self.mainOperator.Dataset[i +
oldNumFiles].setValue(datasetInfo)
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath, headless):
self._projectFilePath = projectFilePath
self.initWithoutTopGroup(hdf5File, projectFilePath)
# normally the serializer is not dirty after loading a project file
# however, when the file was corrupted, the user has the possibility
# to save the fixed file after loading it.
dirty = False
infoDir = topGroup['infos']
self.topLevelOperator.Dataset.resize( len(infoDir) )
for index, (infoGroupName, infoGroup) in enumerate( sorted(infoDir.items()) ):
datasetInfo = DatasetInfo()
# Make a reverse-lookup of the location storage strings
LocationLookup = { v:k for k,v in self.LocationStrings.items() }
datasetInfo.location = LocationLookup[ str(infoGroup['location'].value) ]
# Write to the 'private' members to avoid resetting the dataset id
datasetInfo._filePath = str(infoGroup['filePath'].value)
datasetInfo._datasetId = str(infoGroup['datasetId'].value)
# Deserialize the "allow labels" flag
try:
datasetInfo.allowLabels = infoGroup['allowLabels'].value
except KeyError:
pass
# Deserialize the axisorder (if present)
try:
datasetInfo.axisorder = infoGroup['axisorder'].value
except KeyError:
pass
# If the data is supposed to be in the project,
# check for it now.
if datasetInfo.location == DatasetInfo.Location.ProjectInternal:
if not datasetInfo.datasetId in topGroup['local_data'].keys():
raise RuntimeError("Corrupt project file. Could not find data for " + infoGroupName)
# If the data is supposed to exist outside the project, make sure it really does.
if datasetInfo.location == DatasetInfo.Location.FileSystem:
pathData = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0])
filePath = pathData.externalPath
if not os.path.exists(filePath):
if headless:
raise RuntimeError("Could not find data at " + filePath)
filt = "Image files (" + ' '.join('*.' + x for x in OpDataSelection.SupportedExtensions) + ')'
newpath = self.repairFile(filePath, filt)
newpath = newpath+pathData.internalPath
datasetInfo._filePath = getPathVariants(newpath , os.path.split(projectFilePath)[0])[0]
dirty = True
# Give the new info to the operator
self.topLevelOperator.Dataset[index].setValue(datasetInfo)
self._dirty = dirty
def execute(self, slot, subindex, roi, result):
dtypeBytes = self._getDtypeBytes()
totalBytes = dtypeBytes * numpy.prod(self.Input.meta.shape)
totalMB = totalBytes / (1000*1000)
logger.info( "Clusterizing computation of {} MB dataset, outputting according to {}".format(totalMB, self.OutputDatasetDescription.value) )
configFilePath = self.ConfigFilePath.value
self._config = parseClusterConfigFile( configFilePath )
self._validateConfig()
# Create the destination file if necessary
blockwiseFileset, taskInfos = self._prepareDestination()
try:
# Figure out which work doesn't need to be recomputed (if any)
unneeded_rois = []
for roi in taskInfos.keys():
if blockwiseFileset.getBlockStatus(roi[0]) == BlockwiseFileset.BLOCK_AVAILABLE \
or blockwiseFileset.isBlockLocked(roi[0]): # We don't attempt to process currently locked blocks.
unneeded_rois.append( roi )
# Remove any tasks that we don't need to compute (they were finished in a previous run)
for roi in unneeded_rois:
logger.info( "No need to run task: {} for roi: {}".format( taskInfos[roi].taskName, roi ) )
del taskInfos[roi]
absWorkDir, relWorkDir = getPathVariants(self._config.server_working_directory, os.path.split( configFilePath )[0] )
if self._config.task_launch_server == "localhost":
def localCommand( cmd ):
cwd = os.getcwd()
os.chdir( absWorkDir )
subprocess.call( cmd, shell=True )
os.chdir( cwd )
launchFunc = localCommand
else:
# We use fabric for executing remote tasks
# Import it here because it isn't required that the nodes can use it.
import fabric.api as fab
@fab.hosts( self._config.task_launch_server )
def remoteCommand( cmd ):
with fab.cd( absWorkDir ):
fab.run( cmd )
launchFunc = functools.partial( fab.execute, remoteCommand )
# Spawn each task
for taskInfo in taskInfos.values():
logger.info("Launching node task: " + taskInfo.command )
launchFunc( taskInfo.command )
# Return immediately. We do not attempt to monitor the task progress.
result[0] = True
return result
finally:
blockwiseFileset.close()
def addFileNames(self, fileNames):
"""
Add the given filenames to both the GUI table and the top-level operator inputs.
"""
with Tracer(traceLogger):
infos = []
oldNumFiles = len(self.topLevelOperator.Dataset)
# HACK: If the filePath isn't valid, replace it
# This is to work around the scenario where two independent data selection applets are coupled, causing mutual resizes.
# This will be fixed when a multi-file data selection applet gui replaces this gui.
for i in reversed( range( oldNumFiles ) ):
if not self.topLevelOperator.Dataset[i].ready():
oldNumFiles -= 1
else:
break
# Assign values to the new inputs we just allocated.
# The GUI will be updated by callbacks that are listening to slot changes
for i, filePath in enumerate(fileNames):
datasetInfo = DatasetInfo()
cwd = self.topLevelOperator.WorkingDirectory.value
if not areOnSameDrive(filePath,cwd):
QMessageBox.critical(self, "Drive Error","Data must be on same drive as working directory.")
return
absPath, relPath = getPathVariants(filePath, cwd)
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([cwd, absPath])) > 1:
datasetInfo.filePath = relPath
else:
datasetInfo.filePath = absPath
h5Exts = ['.ilp', '.h5', '.hdf5']
if os.path.splitext(datasetInfo.filePath)[1] in h5Exts:
datasetNames = self.getPossibleInternalPaths( absPath )
if len(datasetNames) > 0:
datasetInfo.filePath += str(datasetNames[0])
else:
raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath)
# Allow labels by default if this gui isn't being used for batch data.
datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal )
infos.append(datasetInfo)
#if no exception was thrown, set up the operator now
self.topLevelOperator.Dataset.resize( oldNumFiles+len(fileNames) )
for i in range(len(infos)):
self.topLevelOperator.Dataset[i+oldNumFiles].setValue( infos[i] )
def _prepareTaskInfos(self, roiList):
# Divide up the workload into large pieces
logger.info("Dividing into {} node jobs.".format(len(roiList)))
taskInfos = collections.OrderedDict()
for roiIndex, roi in enumerate(roiList):
roi = (tuple(roi[0]), tuple(roi[1]))
taskInfo = OpClusterize.TaskInfo()
taskInfo.subregion = SubRegion(None, start=roi[0], stop=roi[1])
taskName = "J{:02}".format(roiIndex)
commandArgs = []
commandArgs.append("--option_config_file=" +
self.ConfigFilePath.value)
commandArgs.append("--project=" + self.ProjectFilePath.value)
commandArgs.append("--_node_work_=\"" +
Roi.dumps(taskInfo.subregion) + "\"")
commandArgs.append("--process_name={}".format(taskName))
commandArgs.append("--output_description_file={}".format(
self.OutputDatasetDescription.value))
for slot in self.SecondaryOutputDescriptions:
commandArgs.append(
"--secondary_output_description_file={}".format(
slot.value))
# Check the command format string: We need to know where to put our args...
commandFormat = self._config.command_format
assert commandFormat.find("{task_args}") != -1
# Output log directory might be a relative path (relative to config file)
absLogDir, relLogDir = getPathVariants(
self._config.output_log_directory,
os.path.split(self.ConfigFilePath.value)[0])
taskOutputLogFilename = taskName + ".log"
taskOutputLogPath = os.path.join(absLogDir, taskOutputLogFilename)
allArgs = " " + " ".join(commandArgs) + " "
taskInfo.taskName = taskName
taskInfo.command = commandFormat.format(
task_args=allArgs,
task_name=taskName,
task_output_file=taskOutputLogPath)
taskInfos[roi] = taskInfo
return taskInfos
def updateStorageOptionComboBox(self, row, filePath):
"""
Create and add the combobox for storage location options
"""
with Tracer(traceLogger):
# Determine the relative path to this file
absPath, relPath = getPathVariants(
filePath, self.mainOperator.WorkingDirectory.value)
# Add a prefixes to make the options clear
absPath = "Absolute Link: " + absPath
relPath = "Relative Link: <project directory>/" + relPath
combo = QComboBox()
options = {} # combo data -> combo text
options[LocationOptions.AbsolutePath] = absPath
options[LocationOptions.RelativePath] = relPath
options[LocationOptions.Project] = "Store in Project File"
for option, text in sorted(options.items()):
# Add to the combo, storing the option as the item data
combo.addItem(text, option)
# Select the combo index that matches the current setting
location = self.mainOperator.Dataset[row].value.location
if location == DatasetInfo.Location.ProjectInternal:
comboData = LocationOptions.Project
elif location == DatasetInfo.Location.FileSystem:
# Determine if the path is relative or absolute
if os.path.isabs(
self.mainOperator.Dataset[row].value.filePath[0]):
comboData = LocationOptions.AbsolutePath
else:
comboData = LocationOptions.RelativePath
comboIndex = combo.findData(QVariant(comboData))
combo.setCurrentIndex(comboIndex)
combo.currentIndexChanged.connect(
partial(self.handleComboSelectionChanged, combo))
self.fileInfoTableWidget.setCellWidget(row, Column.Location, combo)
def updateStorageOptionComboBox(self, row, filePath):
"""
Create and add the combobox for storage location options
"""
assert threading.current_thread().name == "MainThread"
with Tracer(traceLogger):
# Determine the relative path to this file
absPath, relPath = getPathVariants(filePath, self.topLevelOperator.WorkingDirectory.value)
# Add a prefixes to make the options clear
absPath = "Absolute Link: " + absPath
relPath = "Relative Link: <project directory>/" + relPath
combo = QComboBox()
options = {} # combo data -> combo text
options[ LocationOptions.AbsolutePath ] = absPath
options[ LocationOptions.RelativePath ] = relPath
options[ LocationOptions.Project ] = "Store in Project File"
for option, text in sorted(options.items()):
# Add to the combo, storing the option as the item data
combo.addItem(text, option)
# Select the combo index that matches the current setting
location = self.topLevelOperator.Dataset[row].value.location
if location == DatasetInfo.Location.ProjectInternal:
comboData = LocationOptions.Project
elif location == DatasetInfo.Location.FileSystem:
# Determine if the path is relative or absolute
if os.path.isabs(self.topLevelOperator.Dataset[row].value.filePath[0]):
comboData = LocationOptions.AbsolutePath
else:
comboData = LocationOptions.RelativePath
comboIndex = combo.findData( QVariant(comboData) )
combo.setCurrentIndex( comboIndex )
combo.currentIndexChanged.connect( partial(self.handleComboSelectionChanged, combo) )
self.fileInfoTableWidget.setCellWidget( row, Column.Location, combo )
def updateInternalPathComboBox(self, row, externalPath, internalPath):
with Tracer(traceLogger):
combo = QComboBox()
datasetNames = []
# Make sure we're dealing with the absolute path (to make this simple)
absPath, relPath = getPathVariants(
externalPath, self.mainOperator.WorkingDirectory.value)
ext = os.path.splitext(absPath)[1]
h5Exts = ['.ilp', '.h5', '.hdf5']
if ext in h5Exts:
# Open the file as a read-only so we can get a list of the internal paths
f = h5py.File(absPath, 'r')
# Define a closure to collect all of the dataset names in the file.
def accumulateDatasetPaths(name, val):
if type(val) == h5py._hl.dataset.Dataset and 3 <= len(
val.shape) <= 5:
datasetNames.append('/' + name)
# Visit every group/dataset in the file
f.visititems(accumulateDatasetPaths)
f.close()
# Add each dataset option to the combo
for path in datasetNames:
combo.addItem(path)
# If the internal path we used previously is in the combo list, select it.
prevSelection = combo.findText(internalPath)
if prevSelection != -1:
combo.setCurrentIndex(prevSelection)
# Define response to changes and add it to the GUI.
# Pass in the corresponding the table item so we can figure out which row this came from
combo.currentIndexChanged.connect(
bind(self.handleComboSelectionChanged, combo))
self.fileInfoTableWidget.setCellWidget(row, Column.InternalID,
combo)
def updateWorkingDirectory(self, newpath, oldpath):
newdir = PathComponents(newpath).externalDirectory
olddir = PathComponents(oldpath).externalDirectory
if newdir == olddir:
return
# Disconnect the working directory while we make these changes.
# All the changes will take effect when we set the new working directory.
self.topLevelOperator.WorkingDirectory.disconnect()
for laneIndex, multislot in enumerate(
self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if not slot.ready():
# Skip if there is no dataset in this lane/role combination yet.
continue
datasetInfo = slot.value
if datasetInfo.location == DatasetInfo.Location.FileSystem:
#construct absolute path and recreate relative to the new path
fp = PathComponents(datasetInfo.filePath,
olddir).totalPath()
abspath, relpath = getPathVariants(fp, newdir)
# Same convention as in dataSelectionGui:
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([fp, abspath])) > 1:
datasetInfo.filePath = relpath
else:
datasetInfo.filePath = abspath
slot.setValue(datasetInfo, check_changed=False)
self.topLevelOperator.WorkingDirectory.setValue(newdir)
self._projectFilePath = newdir
def _prepareTaskInfos(self, roiList):
# Divide up the workload into large pieces
logger.info( "Dividing into {} node jobs.".format( len(roiList) ) )
taskInfos = collections.OrderedDict()
for roiIndex, roi in enumerate(roiList):
roi = ( tuple(roi[0]), tuple(roi[1]) )
taskInfo = OpClusterize.TaskInfo()
taskInfo.subregion = SubRegion( None, start=roi[0], stop=roi[1] )
taskName = "J{:02}".format(roiIndex)
commandArgs = []
commandArgs.append( "--option_config_file=" + self.ConfigFilePath.value )
commandArgs.append( "--project=" + self.ProjectFilePath.value )
commandArgs.append( "--_node_work_=\"" + Roi.dumps( taskInfo.subregion ) + "\"" )
commandArgs.append( "--process_name={}".format(taskName) )
commandArgs.append( "--output_description_file={}".format( self.OutputDatasetDescription.value ) )
for slot in self.SecondaryOutputDescriptions:
commandArgs.append( "--secondary_output_description_file={}".format( slot.value ) )
# Check the command format string: We need to know where to put our args...
commandFormat = self._config.command_format
assert commandFormat.find("{task_args}") != -1
# Output log directory might be a relative path (relative to config file)
absLogDir, relLogDir = getPathVariants(self._config.output_log_directory, os.path.split( self.ConfigFilePath.value )[0] )
taskOutputLogFilename = taskName + ".log"
taskOutputLogPath = os.path.join( absLogDir, taskOutputLogFilename )
allArgs = " " + " ".join(commandArgs) + " "
taskInfo.taskName = taskName
taskInfo.command = commandFormat.format( task_args=allArgs, task_name=taskName, task_output_file=taskOutputLogPath )
taskInfos[roi] = taskInfo
return taskInfos
def runWorkflow(parsed_args):
args = parsed_args
# Read the config file
configFilePath = args.option_config_file
config = parseClusterConfigFile(configFilePath)
# If we've got a process name, re-initialize the logger from scratch
task_name = "node"
if args.process_name is not None:
task_name = args.process_name
ilastik.ilastik_logging.default_config.init(args.process_name + ' ')
rootLogHandler = None
if args._node_work_ is None:
# This is the master process.
# Tee the log to a file for future reference.
# Output log directory might be a relative path (relative to config file)
absLogDir, relLogDir = getPathVariants(
config.output_log_directory,
os.path.split(configFilePath)[0])
if not os.path.exists(absLogDir):
os.mkdir(absLogDir)
# Copy the config we're using to the output directory
shutil.copy(configFilePath, absLogDir)
logFile = os.path.join(absLogDir, "MASTER.log")
logFileFormatter = logging.Formatter(
"%(levelname)s %(name)s: %(message)s")
rootLogHandler = logging.FileHandler(logFile, 'a')
rootLogHandler.setFormatter(logFileFormatter)
rootLogger = logging.getLogger()
rootLogger.addHandler(rootLogHandler)
logger.info("Launched with sys.argv: {}".format(sys.argv))
# Update the monkey_patch settings
ilastik.utility.monkey_patches.apply_setting_dict(config.__dict__)
# If we're running a node job, set the threadpool size if the user specified one.
# Note that the main thread does not count toward the threadpool total.
if args._node_work_ is not None and config.task_threadpool_size is not None:
lazyflow.request.Request.reset_thread_pool(
num_workers=config.task_threadpool_size)
# Make sure project file exists.
if not os.path.exists(args.project):
raise RuntimeError("Project file '" + args.project +
"' does not exist.")
# Instantiate 'shell'
shell = HeadlessShell(
functools.partial(Workflow.getSubclass(config.workflow_type)))
# Load project (auto-import it if necessary)
logger.info("Opening project: '" + args.project + "'")
shell.openProjectPath(args.project)
workflow = shell.projectManager.workflow
# Attach cluster operators
resultSlot = None
finalOutputSlot = workflow.getHeadlessOutputSlot(config.output_slot_id)
assert finalOutputSlot is not None
secondaryOutputSlots = workflow.getSecondaryHeadlessOutputSlots(
config.output_slot_id)
secondaryOutputDescriptions = args.secondary_output_description_file # This is a list (see 'action' above)
if len(secondaryOutputDescriptions) != len(secondaryOutputSlots):
raise RuntimeError(
"This workflow produces exactly {} SECONDARY outputs. You provided {}."
.format(len(secondaryOutputSlots),
len(secondaryOutputDescriptions)))
clusterOperator = None
try:
if args._node_work_ is not None:
# We're doing node work
opClusterTaskWorker = OperatorWrapper(
OpTaskWorker, parent=finalOutputSlot.getRealOperator().parent)
# FIXME: Image index is hard-coded as 0. We assume we are working with only one (big) dataset in cluster mode.
opClusterTaskWorker.Input.connect(finalOutputSlot)
opClusterTaskWorker.RoiString[0].setValue(args._node_work_)
opClusterTaskWorker.TaskName.setValue(task_name)
opClusterTaskWorker.ConfigFilePath.setValue(
args.option_config_file)
# Configure optional slots first for efficiency (avoid multiple calls to setupOutputs)
opClusterTaskWorker.SecondaryInputs[0].resize(
len(secondaryOutputSlots))
opClusterTaskWorker.SecondaryOutputDescriptions[0].resize(
len(secondaryOutputSlots))
for i in range(len(secondaryOutputSlots)):
opClusterTaskWorker.SecondaryInputs[0][i].connect(
secondaryOutputSlots[i][0])
opClusterTaskWorker.SecondaryOutputDescriptions[0][i].setValue(
secondaryOutputDescriptions[i])
opClusterTaskWorker.OutputFilesetDescription.setValue(
args.output_description_file)
# If we have a way to report task progress (e.g. by updating the job name),
# then subscribe to progress signals
if config.task_progress_update_command is not None:
def report_progress(progress):
cmd = config.task_progress_update_command.format(
progress=int(progress))
def shell_call(shell_cmd):
logger.debug("Executing progress command: " + cmd)
subprocess.call(shell_cmd, shell=True)
background_tasks.put(functools.partial(shell_call, cmd))
opClusterTaskWorker.innerOperators[0].progressSignal.subscribe(
report_progress)
resultSlot = opClusterTaskWorker.ReturnCode
clusterOperator = opClusterTaskWorker
else:
# We're the master
opClusterizeMaster = OperatorWrapper(
OpClusterize, parent=finalOutputSlot.getRealOperator().parent)
opClusterizeMaster.Input.connect(finalOutputSlot)
opClusterizeMaster.ProjectFilePath.setValue(args.project)
opClusterizeMaster.OutputDatasetDescription.setValue(
args.output_description_file)
# Configure optional slots first for efficiency (avoid multiple calls to setupOutputs)
opClusterizeMaster.SecondaryInputs[0].resize(
len(secondaryOutputSlots))
opClusterizeMaster.SecondaryOutputDescriptions[0].resize(
len(secondaryOutputSlots))
for i in range(len(secondaryOutputSlots)):
opClusterizeMaster.SecondaryInputs[0][i].connect(
secondaryOutputSlots[i][0])
opClusterizeMaster.SecondaryOutputDescriptions[0][i].setValue(
secondaryOutputDescriptions[i])
opClusterizeMaster.ConfigFilePath.setValue(args.option_config_file)
resultSlot = opClusterizeMaster.ReturnCode
clusterOperator = opClusterizeMaster
# Get the result
logger.info("Starting task")
result = resultSlot[
0].value # FIXME: The image index is hard-coded here.
finally:
logger.info("Cleaning up")
global stop_background_tasks
stop_background_tasks = True
try:
if clusterOperator is not None:
clusterOperator.cleanUp()
except:
logger.error("Errors during cleanup.")
try:
logger.info("Closing project...")
shell.closeCurrentProject()
except:
logger.error("Errors while closing project.")
logger.info("FINISHED with result {}".format(result))
if not result:
logger.error("FAILED TO COMPLETE!")
if rootLogHandler is not None:
rootLogHandler.close()
def addFileNames(self, fileNames, roleIndex, startingLane=None):
"""
Add the given filenames to both the GUI table and the top-level operator inputs.
If startingLane is None, the filenames will be *appended* to the role's list of files.
"""
infos = []
if startingLane is None:
startingLane = self._findFirstEmptyLane(roleIndex)
endingLane = startingLane + len(fileNames) - 1
else:
assert startingLane < len(self.topLevelOperator.DatasetGroup)
endingLane = startingLane + len(fileNames) - 1
if self._max_lanes and endingLane >= self._max_lanes:
msg = "You may not add more than {} file(s) to this workflow. Please try again.".format(
self._max_lanes)
QMessageBox.critical(self, "Too many files", msg)
return
# Assign values to the new inputs we just allocated.
# The GUI will be updated by callbacks that are listening to slot changes
for i, filePath in enumerate(fileNames):
datasetInfo = DatasetInfo()
cwd = self.topLevelOperator.WorkingDirectory.value
if not areOnSameDrive(filePath, cwd):
QMessageBox.critical(
self, "Drive Error",
"Data must be on same drive as working directory.")
return
absPath, relPath = getPathVariants(filePath, cwd)
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([cwd, absPath])) > 1:
datasetInfo.filePath = relPath
else:
datasetInfo.filePath = absPath
datasetInfo.nickname = PathComponents(absPath).filenameBase
h5Exts = ['.ilp', '.h5', '.hdf5']
if os.path.splitext(datasetInfo.filePath)[1] in h5Exts:
datasetNames = self.getPossibleInternalPaths(absPath)
if len(datasetNames) > 0:
datasetInfo.filePath += str(datasetNames[0])
else:
raise RuntimeError("HDF5 file %s has no image datasets" %
datasetInfo.filePath)
# Allow labels by default if this gui isn't being used for batch data.
datasetInfo.allowLabels = (self.guiMode == GuiMode.Normal)
infos.append(datasetInfo)
# if no exception was thrown, set up the operator now
opTop = self.topLevelOperator
originalSize = len(opTop.DatasetGroup)
if len(opTop.DatasetGroup) < endingLane + 1:
opTop.DatasetGroup.resize(endingLane + 1)
for laneIndex, info in zip(range(startingLane, endingLane + 1), infos):
try:
self.topLevelOperator.DatasetGroup[laneIndex][
roleIndex].setValue(info)
except DatasetConstraintError as ex:
return_val = [False]
# Give the user a chance to fix the problem
self.handleDatasetConstraintError(info, info.filePath, ex,
roleIndex, laneIndex,
return_val)
if not return_val[0]:
# Not successfully repaired. Roll back the changes and give up.
opTop.DatasetGroup.resize(originalSize)
break
except OpDataSelection.InvalidDimensionalityError as ex:
opTop.DatasetGroup.resize(originalSize)
QMessageBox.critical(self,
"Dataset has different dimensionality",
ex.message)
break
except:
QMessageBox.critical(
self, "Dataset Load Error",
"Wasn't able to load your dataset into the workflow. See console for details."
)
opTop.DatasetGroup.resize(originalSize)
raise
self.updateInternalPathVisiblity()
def _readDatasetInfo(self, infoGroup, localDataGroup, projectFilePath,
headless):
# Unready datasets are represented with an empty group.
if len(infoGroup) == 0:
return None, False
datasetInfo = DatasetInfo()
# Make a reverse-lookup of the location storage strings
LocationLookup = {v: k for k, v in self.LocationStrings.items()}
datasetInfo.location = LocationLookup[str(infoGroup['location'].value)]
# Write to the 'private' members to avoid resetting the dataset id
datasetInfo._filePath = str(infoGroup['filePath'].value)
datasetInfo._datasetId = str(infoGroup['datasetId'].value)
try:
datasetInfo.allowLabels = infoGroup['allowLabels'].value
except KeyError:
pass
try:
datasetInfo.drange = tuple(infoGroup['drange'].value)
except KeyError:
pass
try:
datasetInfo.nickname = str(infoGroup['nickname'].value)
except KeyError:
datasetInfo.nickname = PathComponents(
datasetInfo.filePath).filenameBase
try:
tags = vigra.AxisTags.fromJSON(infoGroup['axistags'].value)
datasetInfo.axistags = tags
except KeyError:
# Old projects just have an 'axisorder' field instead of full axistags
try:
axisorder = infoGroup['axisorder'].value
datasetInfo.axistags = vigra.defaultAxistags(axisorder)
except KeyError:
pass
# If the data is supposed to be in the project,
# check for it now.
if datasetInfo.location == DatasetInfo.Location.ProjectInternal:
if not datasetInfo.datasetId in localDataGroup.keys():
raise RuntimeError(
"Corrupt project file. Could not find data for " +
infoGroup.name)
dirty = False
# If the data is supposed to exist outside the project, make sure it really does.
if datasetInfo.location == DatasetInfo.Location.FileSystem:
pathData = PathComponents(datasetInfo.filePath,
os.path.split(projectFilePath)[0])
filePath = pathData.externalPath
if not os.path.exists(filePath):
if headless:
raise RuntimeError("Could not find data at " + filePath)
filt = "Image files (" + ' '.join(
'*.' + x
for x in OpDataSelection.SupportedExtensions) + ')'
newpath = self.repairFile(filePath, filt)
if pathData.internalPath is not None:
newpath += pathData.internalPath
datasetInfo._filePath = getPathVariants(
newpath,
os.path.split(projectFilePath)[0])[0]
dirty = True
return datasetInfo, dirty
def _readDatasetInfo(self, infoGroup, localDataGroup, projectFilePath, headless):
# Unready datasets are represented with an empty group.
if len( infoGroup ) == 0:
return None, False
datasetInfo = DatasetInfo()
# Make a reverse-lookup of the location storage strings
LocationLookup = { v:k for k,v in self.LocationStrings.items() }
datasetInfo.location = LocationLookup[ str(infoGroup['location'].value) ]
# Write to the 'private' members to avoid resetting the dataset id
datasetInfo._filePath = str(infoGroup['filePath'].value)
datasetInfo._datasetId = str(infoGroup['datasetId'].value)
try:
datasetInfo.allowLabels = infoGroup['allowLabels'].value
except KeyError:
pass
try:
datasetInfo.drange = tuple( infoGroup['drange'].value )
except KeyError:
pass
try:
datasetInfo.nickname = str( infoGroup['nickname'].value )
except KeyError:
datasetInfo.nickname = PathComponents(datasetInfo.filePath).filenameBase
try:
tags = vigra.AxisTags.fromJSON( infoGroup['axistags'].value )
datasetInfo.axistags = tags
except KeyError:
# Old projects just have an 'axisorder' field instead of full axistags
try:
axisorder = infoGroup['axisorder'].value
datasetInfo.axistags = vigra.defaultAxistags(axisorder)
except KeyError:
pass
# If the data is supposed to be in the project,
# check for it now.
if datasetInfo.location == DatasetInfo.Location.ProjectInternal:
if not datasetInfo.datasetId in localDataGroup.keys():
raise RuntimeError("Corrupt project file. Could not find data for " + infoGroup.name)
dirty = False
# If the data is supposed to exist outside the project, make sure it really does.
if datasetInfo.location == DatasetInfo.Location.FileSystem:
pathData = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0])
filePath = pathData.externalPath
if not os.path.exists(filePath):
if headless:
raise RuntimeError("Could not find data at " + filePath)
filt = "Image files (" + ' '.join('*.' + x for x in OpDataSelection.SupportedExtensions) + ')'
newpath = self.repairFile(filePath, filt)
if pathData.internalPath is not None:
newpath += pathData.internalPath
datasetInfo._filePath = getPathVariants(newpath , os.path.split(projectFilePath)[0])[0]
dirty = True
return datasetInfo, dirty
def addFileNames(self, fileNames, roleIndex, startingLane=None):
"""
Add the given filenames to both the GUI table and the top-level operator inputs.
If startingLane is None, the filenames will be *appended* to the role's list of files.
"""
infos = []
if startingLane is None:
startingLane = self._findFirstEmptyLane(roleIndex)
endingLane = startingLane+len(fileNames)-1
else:
assert startingLane < len(self.topLevelOperator.DatasetGroup)
endingLane = startingLane+len(fileNames)-1
# Assign values to the new inputs we just allocated.
# The GUI will be updated by callbacks that are listening to slot changes
for i, filePath in enumerate(fileNames):
datasetInfo = DatasetInfo()
cwd = self.topLevelOperator.WorkingDirectory.value
if not areOnSameDrive(filePath,cwd):
QMessageBox.critical(self, "Drive Error","Data must be on same drive as working directory.")
return
absPath, relPath = getPathVariants(filePath, cwd)
# Relative by default, unless the file is in a totally different tree from the working directory.
if len(os.path.commonprefix([cwd, absPath])) > 1:
datasetInfo.filePath = relPath
else:
datasetInfo.filePath = absPath
datasetInfo.nickname = PathComponents(absPath).filenameBase
h5Exts = ['.ilp', '.h5', '.hdf5']
if os.path.splitext(datasetInfo.filePath)[1] in h5Exts:
datasetNames = self.getPossibleInternalPaths( absPath )
if len(datasetNames) > 0:
datasetInfo.filePath += str(datasetNames[0])
else:
raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath)
# Allow labels by default if this gui isn't being used for batch data.
datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal )
infos.append(datasetInfo)
# if no exception was thrown, set up the operator now
opTop = self.topLevelOperator
originalSize = len(opTop.DatasetGroup)
if len( opTop.DatasetGroup ) < endingLane+1:
opTop.DatasetGroup.resize( endingLane+1 )
for laneIndex, info in zip(range(startingLane, endingLane+1), infos):
try:
self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue( info )
except DatasetConstraintError as ex:
# Give the user a chance to fix the problem
if not self.handleDatasetConstraintError(info, info.filePath, ex, roleIndex, laneIndex):
opTop.DatasetGroup.resize( originalSize )
break
except:
QMessageBox.critical( self, "Dataset Load Error", "Wasn't able to load your dataset into the workflow. See console for details." )
opTop.DatasetGroup.resize( originalSize )
raise
self.updateInternalPathVisiblity()
def execute(self, slot, subindex, roi, result):
dtypeBytes = self._getDtypeBytes()
totalBytes = dtypeBytes * numpy.prod(self.Input.meta.shape)
totalMB = totalBytes / (1000 * 1000)
logger.info(
"Clusterizing computation of {} MB dataset, outputting according to {}"
.format(totalMB, self.OutputDatasetDescription.value))
configFilePath = self.ConfigFilePath.value
self._config = parseClusterConfigFile(configFilePath)
self._validateConfig()
# Create the destination file if necessary
blockwiseFileset, taskInfos = self._prepareDestination()
try:
# Figure out which work doesn't need to be recomputed (if any)
unneeded_rois = []
for roi in taskInfos.keys():
if blockwiseFileset.getBlockStatus(roi[0]) == BlockwiseFileset.BLOCK_AVAILABLE \
or blockwiseFileset.isBlockLocked(roi[0]): # We don't attempt to process currently locked blocks.
unneeded_rois.append(roi)
# Remove any tasks that we don't need to compute (they were finished in a previous run)
for roi in unneeded_rois:
logger.info("No need to run task: {} for roi: {}".format(
taskInfos[roi].taskName, roi))
del taskInfos[roi]
absWorkDir, relWorkDir = getPathVariants(
self._config.server_working_directory,
os.path.split(configFilePath)[0])
if self._config.task_launch_server == "localhost":
def localCommand(cmd):
cwd = os.getcwd()
os.chdir(absWorkDir)
subprocess.call(cmd, shell=True)
os.chdir(cwd)
launchFunc = localCommand
else:
# We use fabric for executing remote tasks
# Import it here because it isn't required that the nodes can use it.
import fabric.api as fab
@fab.hosts(self._config.task_launch_server)
def remoteCommand(cmd):
with fab.cd(absWorkDir):
fab.run(cmd)
launchFunc = functools.partial(fab.execute, remoteCommand)
# Spawn each task
for taskInfo in taskInfos.values():
logger.info("Launching node task: " + taskInfo.command)
launchFunc(taskInfo.command)
# Return immediately. We do not attempt to monitor the task progress.
result[0] = True
return result
finally:
blockwiseFileset.close()
def updateFilePath(self, index):
"""
Update the operator's filePath input to match the gui
"""
with Tracer(traceLogger):
oldLocationSetting = self.mainOperator.Dataset[
index].value.location
# Get the directory by inspecting the original operator path
oldTotalPath = self.mainOperator.Dataset[index].value.filePath
# Split into directory, filename, extension, and internal path
lastDotIndex = oldTotalPath.rfind('.')
extensionAndInternal = oldTotalPath[lastDotIndex:]
extension = extensionAndInternal.split('/')[0]
oldFilePath = oldTotalPath[:lastDotIndex] + extension
fileNameText = str(
self.fileInfoTableWidget.item(index, Column.Name).text())
internalPathCombo = self.fileInfoTableWidget.cellWidget(
index, Column.InternalID)
#internalPath = str(self.fileInfoTableWidget.item(index, Column.InternalID).text())
internalPath = str(internalPathCombo.currentText())
directory = os.path.split(oldFilePath)[0]
newFileNamePath = fileNameText
if directory != '':
newFileNamePath = directory + '/' + fileNameText
newTotalPath = newFileNamePath
if internalPath != '':
if internalPath[0] != '/':
newTotalPath += '/'
newTotalPath += internalPath
cwd = self.mainOperator.WorkingDirectory.value
absTotalPath, relTotalPath = getPathVariants(newTotalPath, cwd)
# Check the location setting
locationCombo = self.fileInfoTableWidget.cellWidget(
index, Column.Location)
comboIndex = locationCombo.currentIndex()
newLocationSelection = locationCombo.itemData(comboIndex).toInt()[
0] # In PyQt, toInt() returns a tuple
if newLocationSelection == LocationOptions.Project:
newLocationSetting = DatasetInfo.Location.ProjectInternal
elif newLocationSelection == LocationOptions.AbsolutePath:
newLocationSetting = DatasetInfo.Location.FileSystem
newTotalPath = absTotalPath
elif newLocationSelection == LocationOptions.RelativePath:
newLocationSetting = DatasetInfo.Location.FileSystem
newTotalPath = relTotalPath
if newTotalPath != oldTotalPath or newLocationSetting != oldLocationSetting:
# Be sure to copy so the slot notices the change when we setValue()
datasetInfo = copy.copy(self.mainOperator.Dataset[index].value)
datasetInfo.filePath = newTotalPath
datasetInfo.location = newLocationSetting
# TODO: First check to make sure this file exists!
self.mainOperator.Dataset[index].setValue(datasetInfo)
# Update the storage option combo to show the new path
self.updateStorageOptionComboBox(index, newFileNamePath)
def runWorkflow(parsed_args):
args = parsed_args
# Read the config file
configFilePath = args.option_config_file
config = parseClusterConfigFile( configFilePath )
# If we've got a process name, re-initialize the logger from scratch
task_name = "node"
if args.process_name is not None:
task_name = args.process_name
ilastik.ilastik_logging.default_config.init(args.process_name + ' ')
rootLogHandler = None
if args._node_work_ is None:
# This is the master process.
# Tee the log to a file for future reference.
# Output log directory might be a relative path (relative to config file)
absLogDir, relLogDir = getPathVariants(config.output_log_directory, os.path.split( configFilePath )[0] )
if not os.path.exists(absLogDir):
os.mkdir(absLogDir)
# Copy the config we're using to the output directory
shutil.copy(configFilePath, absLogDir)
logFile = os.path.join( absLogDir, "MASTER.log" )
logFileFormatter = logging.Formatter("%(levelname)s %(name)s: %(message)s")
rootLogHandler = logging.FileHandler(logFile, 'a')
rootLogHandler.setFormatter(logFileFormatter)
rootLogger = logging.getLogger()
rootLogger.addHandler( rootLogHandler )
logger.info( "Launched with sys.argv: {}".format( sys.argv ) )
# Update the monkey_patch settings
ilastik.utility.monkey_patches.apply_setting_dict( config.__dict__ )
# If we're running a node job, set the threadpool size if the user specified one.
# Note that the main thread does not count toward the threadpool total.
if args._node_work_ is not None and config.task_threadpool_size is not None:
lazyflow.request.Request.reset_thread_pool( num_workers = config.task_threadpool_size )
# Make sure project file exists.
if not os.path.exists(args.project):
raise RuntimeError("Project file '" + args.project + "' does not exist.")
# Instantiate 'shell'
shell = HeadlessShell( functools.partial(Workflow.getSubclass(config.workflow_type) ) )
# Load project (auto-import it if necessary)
logger.info("Opening project: '" + args.project + "'")
shell.openProjectPath(args.project)
workflow = shell.projectManager.workflow
# Attach cluster operators
resultSlot = None
finalOutputSlot = workflow.getHeadlessOutputSlot( config.output_slot_id )
assert finalOutputSlot is not None
secondaryOutputSlots = workflow.getSecondaryHeadlessOutputSlots( config.output_slot_id )
secondaryOutputDescriptions = args.secondary_output_description_file # This is a list (see 'action' above)
if len(secondaryOutputDescriptions) != len(secondaryOutputSlots):
raise RuntimeError( "This workflow produces exactly {} SECONDARY outputs. You provided {}.".format( len(secondaryOutputSlots), len(secondaryOutputDescriptions) ) )
clusterOperator = None
try:
if args._node_work_ is not None:
# We're doing node work
opClusterTaskWorker = OperatorWrapper( OpTaskWorker, parent=finalOutputSlot.getRealOperator().parent )
# FIXME: Image index is hard-coded as 0. We assume we are working with only one (big) dataset in cluster mode.
opClusterTaskWorker.Input.connect( finalOutputSlot )
opClusterTaskWorker.RoiString[0].setValue( args._node_work_ )
opClusterTaskWorker.TaskName.setValue( task_name )
opClusterTaskWorker.ConfigFilePath.setValue( args.option_config_file )
# Configure optional slots first for efficiency (avoid multiple calls to setupOutputs)
opClusterTaskWorker.SecondaryInputs[0].resize( len( secondaryOutputSlots ) )
opClusterTaskWorker.SecondaryOutputDescriptions[0].resize( len( secondaryOutputSlots ) )
for i in range( len(secondaryOutputSlots) ):
opClusterTaskWorker.SecondaryInputs[0][i].connect( secondaryOutputSlots[i][0] )
opClusterTaskWorker.SecondaryOutputDescriptions[0][i].setValue( secondaryOutputDescriptions[i] )
opClusterTaskWorker.OutputFilesetDescription.setValue( args.output_description_file )
# If we have a way to report task progress (e.g. by updating the job name),
# then subscribe to progress signals
if config.task_progress_update_command is not None:
def report_progress( progress ):
cmd = config.task_progress_update_command.format( progress=int(progress) )
def shell_call(shell_cmd):
logger.debug( "Executing progress command: " + cmd )
subprocess.call( shell_cmd, shell=True )
background_tasks.put( functools.partial( shell_call, cmd ) )
opClusterTaskWorker.innerOperators[0].progressSignal.subscribe( report_progress )
resultSlot = opClusterTaskWorker.ReturnCode
clusterOperator = opClusterTaskWorker
else:
# We're the master
opClusterizeMaster = OperatorWrapper( OpClusterize, parent=finalOutputSlot.getRealOperator().parent )
opClusterizeMaster.Input.connect( finalOutputSlot )
opClusterizeMaster.ProjectFilePath.setValue( args.project )
opClusterizeMaster.OutputDatasetDescription.setValue( args.output_description_file )
# Configure optional slots first for efficiency (avoid multiple calls to setupOutputs)
opClusterizeMaster.SecondaryInputs[0].resize( len( secondaryOutputSlots ) )
opClusterizeMaster.SecondaryOutputDescriptions[0].resize( len( secondaryOutputSlots ) )
for i in range( len(secondaryOutputSlots) ):
opClusterizeMaster.SecondaryInputs[0][i].connect( secondaryOutputSlots[i][0] )
opClusterizeMaster.SecondaryOutputDescriptions[0][i].setValue( secondaryOutputDescriptions[i] )
opClusterizeMaster.ConfigFilePath.setValue( args.option_config_file )
resultSlot = opClusterizeMaster.ReturnCode
clusterOperator = opClusterizeMaster
# Get the result
logger.info("Starting task")
result = resultSlot[0].value # FIXME: The image index is hard-coded here.
finally:
logger.info("Cleaning up")
global stop_background_tasks
stop_background_tasks = True
try:
if clusterOperator is not None:
clusterOperator.cleanUp()
except:
logger.error("Errors during cleanup.")
try:
logger.info("Closing project...")
shell.closeCurrentProject()
except:
logger.error("Errors while closing project.")
logger.info("FINISHED with result {}".format(result))
if not result:
logger.error( "FAILED TO COMPLETE!" )
if rootLogHandler is not None:
rootLogHandler.close()
