def configure_operator_with_parsed_args(self, parsed_args):
"""
Helper function for headless workflows.
Configures this applet's top-level operator according to the settings provided in ``parsed_args``.
:param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`.
"""
input_paths = parsed_args.input_files
# If the user doesn't want image stacks to be copied inte the project file,
# we generate hdf5 volumes in a temporary directory and use those files instead.
if parsed_args.preconvert_stacks:
import tempfile
input_paths = self.convertStacksToH5( input_paths, tempfile.gettempdir() )
input_infos = []
for p in input_paths:
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
# Convert all paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
comp = PathComponents(p)
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
input_infos.append(info)
opDataSelection = self.topLevelOperator
opDataSelection.DatasetGroup.resize( len(input_infos) )
for lane_index, info in enumerate(input_infos):
# Only one dataset role in pixel classification
opDataSelection.DatasetGroup[lane_index][0].setValue( info )
def updateWorkingDirectory(self,newpath,oldpath):
newdir = PathComponents(newpath).externalDirectory
olddir = PathComponents(oldpath).externalDirectory
if newdir==olddir:
return
# Disconnect the working directory while we make these changes.
# All the changes will take effect when we set the new working directory.
self.topLevelOperator.WorkingDirectory.disconnect()
for laneIndex, multislot in enumerate(self.topLevelOperator.DatasetGroup):
for roleIndex, slot in enumerate(multislot):
if not slot.ready():
# Skip if there is no dataset in this lane/role combination yet.
continue
datasetInfo = slot.value
if datasetInfo.location == DatasetInfo.Location.FileSystem:
#construct absolute path and recreate relative to the new path
fp = PathComponents(datasetInfo.filePath,olddir).totalPath()
abspath, relpath = getPathVariants(fp,newdir)
# Same convention as in dataSelectionGui:
# Relative by default, unless the file is in a totally different tree from the working directory.
if relpath is not None and len(os.path.commonprefix([fp, abspath])) > 1:
datasetInfo.filePath = relpath
else:
datasetInfo.filePath = abspath
slot.setValue(datasetInfo, check_changed=False)
self.topLevelOperator.WorkingDirectory.setValue(newdir)
self._projectFilePath = newdir
def handleImportLabelsAction():
# Find the directory of the most recently opened image file
mostRecentImageFile = PreferencesManager().get(
'DataSelection', 'recent image')
if mostRecentImageFile is not None:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
self, defaultDirectory)
fileNames = map(str, fileNames)
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(
self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DataSelectionGui.getPossibleInternalPaths(
file_path)
if len(internal_paths) == 0:
QMessageBox.critical(
self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = H5VolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue(path_components.totalPath())
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue(
top_op.LabelInputs.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
# Finally, import the labels
top_op.importLabels(top_op.current_view_index(),
op5.Output)
finally:
op5.cleanUp()
opReader.cleanUp()
def post_process_lane_export(self, lane_index):
# FIXME: This probably only works for the non-blockwise export slot.
# We should assert that the user isn't using the blockwise slot.
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(lane_index).get_table_export_settings()
from lazyflow.utility import PathComponents, make_absolute, format_known_keys
if settings:
self.dataExportTrackingApplet.progressSignal.emit(-1)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys( name_format, known_keys )
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(lane_index).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportTrackingApplet.progressSignal.emit(100)
def configure_operator_with_parsed_args(self, parsed_args):
"""
Helper function for headless workflows.
Configures this applet's top-level operator according to the settings provided in ``parsed_args``.
:param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`.
"""
# TODO: Support image stack inputs by checking for globstrings and converting to hdf5.
input_paths = parsed_args.input_files
input_infos = []
for p in input_paths:
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
# Convert all paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
comp = PathComponents(p)
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
input_infos.append(info)
opDataSelection = self.topLevelOperator
opDataSelection.DatasetGroup.resize( len(input_infos) )
for lane_index, info in enumerate(input_infos):
# Only one dataset role in pixel classification
opDataSelection.DatasetGroup[lane_index][0].setValue( info )
def _applyInternalPathToTempOps(self, index):
if index == -1:
return
newInternalPath = str( self.internalDatasetNameComboBox.currentText() )
# Save a copy of our settings
oldInfos = {}
for laneIndex, op in self.tempOps.items():
oldInfos[laneIndex] = copy.copy( op.Dataset.value )
# Attempt to apply to all temp operators
try:
for laneIndex, op in self.tempOps.items():
info = copy.copy( op.Dataset.value )
pathComponents = PathComponents(info.filePath)
if pathComponents.internalPath != newInternalPath:
pathComponents.internalPath = newInternalPath
info.filePath = pathComponents.totalPath()
op.Dataset.setValue( info )
self._error_fields.discard('Internal Dataset Name')
return True
except Exception as e:
# Revert everything back to the previous state
for laneIndex, op in self.tempOps.items():
op.Dataset.setValue( oldInfos[laneIndex] )
msg = "Could not set new internal path settings due to an exception:\n"
msg += "{}".format( e )
log_exception( logger, msg )
QMessageBox.warning(self, "Error", msg)
self._error_fields.add('Internal Dataset Name')
return False
def post_process_lane_export(self, lane_index):
# FIXME: This probably only works for the non-blockwise export slot.
# We should assert that the user isn't using the blockwise slot.
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(lane_index).get_table_export_settings()
from lazyflow.utility import PathComponents, make_absolute, format_known_keys
if settings:
self.dataExportTrackingApplet.progressSignal.emit(-1)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys( name_format, known_keys )
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(lane_index).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportTrackingApplet.progressSignal.emit(100)
def setupOutputs(self):
# FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then
# we have no guarantees about which one will trigger setupOutputs() first.
# It is therefore possible for 'RawDatasetInfo' to appear ready() to us,
# even though it's upstream partner is UNready. We are about to get the
# unready() notification, but it will come too late to prevent our
# setupOutputs method from being called.
# Without proper graph setup transaction semantics, we have to use this
# hack as a workaround.
try:
rawInfo = self.RawDatasetInfo.value
except:
for oslot in list(self.outputs.values()):
if oslot.upstream_slot is None:
oslot.meta.NOTREADY = True
return
selection_index = self.InputSelection.value
if not self.Inputs[selection_index].ready():
for oslot in list(self.outputs.values()):
if oslot.upstream_slot is None:
oslot.meta.NOTREADY = True
return
self._opFormattedExport.Input.connect(self.Inputs[selection_index])
dataset_dir = str(rawInfo.default_output_dir)
abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value)
known_keys = {}
known_keys["dataset_dir"] = abs_dataset_dir
nickname = rawInfo.nickname.replace("*", "")
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys["nickname"] = nickname
result_types = self.SelectionNames.value
known_keys["result_type"] = result_types[selection_index]
self._opFormattedExport.TransactionSlot.disconnect()
# Blank the internal path while we manipulate the external path
# to avoid invalid intermediate states of ExportPath
self._opFormattedExport.OutputInternalPath.setValue("")
# use partial formatting to fill in non-coordinate name fields
name_format = self.OutputFilenameFormat.value
partially_formatted_name = format_known_keys(name_format, known_keys)
# Convert to absolute path before configuring the internal op
abs_path, _ = getPathVariants(partially_formatted_name, self.WorkingDirectory.value)
self._opFormattedExport.OutputFilenameFormat.setValue(abs_path)
# use partial formatting on the internal dataset name, too
internal_dataset_format = self.OutputInternalPath.value
partially_formatted_dataset_name = format_known_keys(internal_dataset_format, known_keys)
self._opFormattedExport.OutputInternalPath.setValue(partially_formatted_dataset_name)
# Re-connect to finish the 'transaction'
self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot)
def _initInternalDatasetNameCombo(self):
# If any dataset is either (1) not hdf5 or (2) project-internal, then we can't change the internal path.
h5Exts = ['.ilp', '.h5', '.hdf5']
for laneIndex in self._laneIndexes:
tmpOp = self.tempOps[laneIndex]
datasetInfo = tmpOp.Dataset.value
externalPath = PathComponents(datasetInfo.filePath).externalPath
if os.path.splitext(externalPath)[1] not in h5Exts \
or datasetInfo.location == DatasetInfo.Location.ProjectInternal:
self.internalDatasetNameComboBox.addItem("N/A")
self.internalDatasetNameComboBox.setEnabled(False)
return
# Enable IFF all datasets have at least one common internal dataset, and only show COMMON datasets
allInternalPaths = set()
commonInternalPaths = None
for laneIndex in self._laneIndexes:
tmpOp = self.tempOps[laneIndex]
datasetInfo = tmpOp.Dataset.value
externalPath = PathComponents(datasetInfo.filePath).externalPath
absPath, _ = getPathVariants(externalPath,
tmpOp.WorkingDirectory.value)
internalPaths = set(self._getPossibleInternalPaths(absPath))
if commonInternalPaths is None:
# Init with the first file's set of paths
commonInternalPaths = internalPaths
# Set operations
allInternalPaths |= internalPaths
commonInternalPaths &= internalPaths
if len(commonInternalPaths) == 0:
self.internalDatasetNameComboBox.addItem(
"Couldn't find a dataset name common to all selected files."
)
self.internalDatasetNameComboBox.setEnabled(False)
return
uncommonInternalPaths = allInternalPaths - commonInternalPaths
# Add all common paths to the combo
for path in sorted(commonInternalPaths):
self.internalDatasetNameComboBox.addItem(path)
# Add the remaining ones, but disable them since they aren't common to all files:
for path in sorted(uncommonInternalPaths):
self.internalDatasetNameComboBox.addItem(path)
# http://theworldwideinternet.blogspot.com/2011/01/disabling-qcombobox-items.html
model = self.internalDatasetNameComboBox.model()
index = model.index(self.internalDatasetNameComboBox.count() - 1,
0)
model.setData(index, 0, Qt.UserRole - 1)
# Finally, initialize with NO item selected
self.internalDatasetNameComboBox.setCurrentIndex(-1)
def handleImportLabelsAction():
# Find the directory of the most recently opened image file
mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
if mostRecentImageFile is not None:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(self, defaultDirectory)
fileNames = list(map(str, fileNames))
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DataSelectionGui.getPossibleInternalPaths(file_path)
if len(internal_paths) == 0:
QMessageBox.critical(self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = H5VolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue( path_components.totalPath() )
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue( top_op.LabelInputs.meta.getAxisKeys() )
op5.Input.connect( opReader.Output )
# Finally, import the labels
top_op.importLabels( top_op.current_view_index(), op5.Output )
finally:
op5.cleanUp()
opReader.cleanUp()
def _append_lane(workflow, input_filepath, axisorder=None):
"""
Add a lane to the project file for the given input file.
If axisorder is given, override the default axisorder for
the file and force the project to use the given one.
Globstrings are supported, in which case the files are converted to HDF5 first.
"""
# If the filepath is a globstring, convert the stack to h5 # todo: skip this?
tmp_dir = tempfile.mkdtemp()
input_filepath = DataSelectionApplet.convertStacksToH5([input_filepath],
tmp_dir)[0]
try:
os.rmdir(tmp_dir)
except OSError as e:
if e.errno == 39:
logger.warning(
'Temporary directory {} was populated: should be deleted')
else:
raise
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug("adding lane: {}".format(info))
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len(opDataSelection.DatasetGroup) + 1
logger.debug("num_lanes: {}".format(num_lanes))
opDataSelection.DatasetGroup.resize(num_lanes)
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue(info)
workflow.handleNewLanesAdded()
def handleImportLabelsAction():
fileNames = ImageFileDialog(
self,
preferences_group="DataSelection",
preferences_setting="recent image").getSelectedPaths()
fileNames = list(map(str, fileNames))
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(
self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DatasetInfo.getPossibleInternalPathsFor(file_path)
if len(internal_paths) == 0:
QMessageBox.critical(
self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = SubvolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue(path_components.totalPath())
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue(top_op.LabelInputs.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
# Finally, import the labels
top_op.importLabels(top_op.current_view_index(), op5.Output)
finally:
op5.cleanUp()
opReader.cleanUp()
def _applyPattern(self):
globStrings = self.patternEdit.text()
H5EXTS = OpStreamingH5N5SequenceReaderM.H5EXTS
N5EXTS = OpStreamingH5N5SequenceReaderM.N5EXTS
filenames = []
# see if some glob strings include HDF5 and/or N5 files
globStrings = globStrings.split(os.path.pathsep)
pcs = [PathComponents(x) for x in globStrings]
is_h5_n5 = [x.extension in (H5EXTS + N5EXTS) for x in pcs]
h5GlobStrings = os.path.pathsep.join(
[x for x, y in zip(globStrings, is_h5_n5) if y is True])
globStrings = os.path.pathsep.join(
[x for x, y in zip(globStrings, is_h5_n5) if y is False])
filenames.extend(OpStackLoader.expandGlobStrings(globStrings))
try:
OpStreamingH5N5SequenceReaderS.checkGlobString(h5GlobStrings)
# OK, if nothing raised there is a single h5 file in h5GlobStrings:
pathComponents = PathComponents(
h5GlobStrings.split(os.path.pathsep)[0])
h5file = OpStreamingH5N5Reader.get_h5_n5_file(
pathComponents.externalPath, mode="r")
filenames.extend(
"{}/{}".format(pathComponents.externalPath, internal)
for internal in OpStreamingH5N5SequenceReaderS.
expandGlobStrings(h5file, h5GlobStrings))
except (
OpStreamingH5N5SequenceReaderS.WrongFileTypeError,
OpStreamingH5N5SequenceReaderS.NotTheSameFileError,
OpStreamingH5N5SequenceReaderS.NoInternalPlaceholderError,
OpStreamingH5N5SequenceReaderS.ExternalPlaceholderError,
):
pass
try:
OpStreamingH5N5SequenceReaderM.checkGlobString(h5GlobStrings)
filenames.extend(
"{}/{}".format(external, internal)
for external, internal in zip(
*OpStreamingH5N5SequenceReaderM.expandGlobStrings(
h5GlobStrings)))
except (
OpStreamingH5N5SequenceReaderM.WrongFileTypeError,
OpStreamingH5N5SequenceReaderM.SameFileError,
OpStreamingH5N5SequenceReaderM.NoExternalPlaceholderError,
OpStreamingH5N5SequenceReaderM.InternalPlaceholderError,
):
pass
self._updateFileList(filenames)
def convert_predictions_to_segmentation(input_paths, parsed_export_args):
"""
Read exported pixel predictions and calculate/export the segmentation.
input_path: The path to the prediction output file. If hdf5, must include the internal dataset name.
parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser.
"""
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue(os.getcwd())
opArgmaxChannel = OpArgmaxChannel(graph=graph)
opArgmaxChannel.Input.connect(opReader.Output)
opExport = OpFormattedDataExport(graph=graph)
opExport.Input.connect(opArgmaxChannel.Output)
# Apply command-line arguments.
DataExportApplet._configure_operator_with_parsed_args(
parsed_export_args, opExport)
last_progress = [-1]
def print_progress(progress_percent):
if progress_percent != last_progress[0]:
last_progress[0] = progress_percent
sys.stdout.write(" {}".format(progress_percent))
opExport.progressSignal.subscribe(print_progress)
for input_path in input_paths:
opReader.FilePath.setValue(input_path)
input_pathcomp = PathComponents(input_path)
opExport.OutputFilenameFormat.setValue(str(
input_pathcomp.externalPath))
output_path = opExport.ExportPath.value
output_pathcomp = PathComponents(output_path)
output_pathcomp.filenameBase += "_Segmentation"
opExport.OutputFilenameFormat.setValue(
str(output_pathcomp.externalPath))
print("Exporting results to : {}".format(opExport.ExportPath.value))
sys.stdout.write("Progress:")
# Begin export
opExport.run_export()
sys.stdout.write("\n")
print("DONE.")
def format_path(self, path_template: str) -> str:
dataset_dir = str(self._dataset_info.default_output_dir)
abs_dataset_dir, _ = getPathVariants(dataset_dir, self._working_dir)
nickname = self._dataset_info.nickname.replace("*", "")
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(
os.path.pathsep)[0]).fileNameBase
known_keys = {"dataset_dir": abs_dataset_dir, "nickname": nickname}
if self._result_type:
known_keys["result_type"] = self._result_type
return format_known_keys(path_template, known_keys)
def _export_hdf5(self):
self.progressSignal(0)
# Create and open the hdf5 file
export_components = PathComponents(self.ExportPath.value)
try:
os.remove(export_components.externalPath)
except OSError as ex:
# It's okay if the file isn't there.
if ex.errno != 2:
raise
try:
with h5py.File(export_components.externalPath, 'w') as hdf5File:
# Create a temporary operator to do the work for us
opH5Writer = OpH5WriterBigDataset(parent=self)
try:
opH5Writer.hdf5File.setValue(hdf5File)
opH5Writer.hdf5Path.setValue(
export_components.internalPath)
opH5Writer.Image.connect(self.Input)
# The H5 Writer provides it's own progress signal, so just connect ours to it.
opH5Writer.progressSignal.subscribe(self.progressSignal)
# Perform the export and block for it in THIS THREAD.
opH5Writer.WriteImage[:].wait()
finally:
opH5Writer.cleanUp()
self.progressSignal(100)
except IOError as ex:
import sys
msg = "\nException raised when attempting to export to {}: {}\n"\
.format( export_components.externalPath, str(ex) )
sys.stderr.write(msg)
raise
def testBasic_Hdf5(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags("xy"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("hdf5")
opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
opExport.OutputInternalPath.setValue("volume/data")
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
export_file = PathComponents(opExport.ExportPath.value).externalPath
assert os.path.split(export_file)[1] == "test_export_x10-110_y20-120.h5"
# print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test(self):
graph = Graph()
op = OpTiledVolumeReader(graph=graph)
op.DescriptionFilePath.setValue(
self.data_setup.TRANSPOSED_VOLUME_DESCRIPTION_FILE)
op.tiled_volume.TEST_MODE = True
roi = numpy.array([(10, 150, 100), (30, 550, 550)])
roi_t = numpy.array([tuple(reversed(roi[0])), tuple(reversed(roi[1]))])
result_out_t = op.Output(*roi_t).wait()
result_out = result_out_t.transpose()
# We expect a channel dimension to be added automatically...
assert (result_out_t.shape == roi_t[1] - roi_t[0]).all()
ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
with h5py.File(ref_path_comp.externalPath, 'r') as f:
ref_data = f[ref_path_comp.internalPath][:]
expected = ref_data[roiToSlice(*roi)]
#numpy.save('/tmp/expected.npy', expected)
#numpy.save('/tmp/result_out.npy', result_out)
# We can't expect the pixels to match exactly because compression was used to create the tiles...
assert (expected == result_out).all()
def exportRoiToHdf5(self, roi, exportDirectory, use_view_coordinates=True):
"""
Export an arbitrary roi to a single hdf5 file.
The file will be placed in the given exportDirectory,
and will be named according to the exported roi.
:param roi: The roi to export
:param exportDirectory: The directory in which the result should be placed.
:param use_view_coordinates: If True, assume the roi was given relative to the view start.
Otherwise, assume it was given relative to the on-disk coordinates.
"""
roi = map(TinyVector, roi)
if not use_view_coordinates:
abs_roi = roi
assert (abs_roi[0] >= self.description.view_origin), \
"Roi {} is out-of-bounds: must not span lower than the view origin: ".format( roi, self.description.origin )
view_roi = roi - self.description.view_origin
else:
view_roi = roi
abs_roi = view_roi + self.description.view_origin
# Always name the file according to the absolute roi
roiString = "{}".format((list(abs_roi[0]), list(abs_roi[1])))
datasetPath = self._description.block_file_name_format.format(
roiString=roiString)
fullDatasetPath = os.path.join(exportDirectory, datasetPath)
path_parts = PathComponents(fullDatasetPath)
with h5py.File(path_parts.externalPath, 'w') as f:
self._createDatasetInFile(f, path_parts.internalPath, view_roi)
dataset = f[path_parts.internalPath]
self.readData(view_roi, dataset)
return fullDatasetPath
def handleFinished(result):
# Generate the mapping transforms dataset
mapping = self._opAccumulateFinalImage.Mapping.value
num_labels = mapping.keys()[-1][1]
transform = numpy.zeros(shape=(num_labels, 2), dtype=numpy.uint32)
for (start, stop), body_id in mapping.items():
for supervoxel_label in range(start, stop):
transform[supervoxel_label][0] = supervoxel_label
if body_id == -1:
# Special case: -1 means "identity transform" for this supervoxel
# (Which is really an untouched raveler body)
transform[supervoxel_label][1] = supervoxel_label
else:
transform[supervoxel_label][1] = body_id
# Save the transform before closing the file
f.create_dataset('transforms', data=transform)
# Copy all other datasets from the original segmentation file.
ravelerSegmentationInfo = self.DatasetInfos[2].value
pathComponents = PathComponents(ravelerSegmentationInfo.filePath,
self.WorkingDirectory.value)
with h5py.File(pathComponents.externalPath, 'r') as originalFile:
for k, dset in originalFile.items():
if k not in ['transforms', 'stack']:
f.copy(dset, k)
try:
cleanOps()
logger.info("FINISHED Final Supervoxel Export")
finally:
f.close()
def _export_batch_image(self, lane_index, selection_index, selection_name):
opBatchExport = self.batchExportApplet.topLevelOperator
opBatchExport.InputSelection.setValue(selection_index)
opBatchExportView = opBatchExport.getLane(lane_index)
# Remember this so we can restore it later
default_output_path = opBatchExport.OutputFilenameFormat.value
export_path = opBatchExportView.ExportPath.value
path_comp = PathComponents( export_path, os.getcwd() )
path_comp.filenameBase += '-' + selection_name
opBatchExport.OutputFilenameFormat.setValue( path_comp.externalPath )
logger.info( "Exporting {} for image #{} to {}"
.format(selection_name, lane_index+1, opBatchExportView.ExportPath.value) )
sys.stdout.write( "Result {}/{} Progress: "
.format( lane_index+1, len( self.opBatchClassify ) ) )
sys.stdout.flush()
def print_progress( progress ):
sys.stdout.write( "{} ".format( progress ) )
sys.stdout.flush()
# If the operator provides a progress signal, use it.
slotProgressSignal = opBatchExportView.progressSignal
slotProgressSignal.subscribe( print_progress )
opBatchExportView.run_export()
# Finished.
sys.stdout.write("\n")
# Restore original format
opBatchExport.OutputFilenameFormat.setValue( default_output_path )
def testBasic(self):
graph = Graph()
op = OpCachedTiledVolumeReader(graph=graph)
op.DescriptionFilePath.setValue(
self.data_setup.VOLUME_DESCRIPTION_FILE)
roi = numpy.array([(10, 150, 100), (30, 550, 550)])
cached_result_out = op.CachedOutput(*roi).wait()
uncached_result_out = op.UncachedOutput(*roi).wait()
specified_result_out = op.CachedOutput(*roi).wait()
# We expect a channel dimension to be added automatically...
assert (cached_result_out.shape == roi[1] - roi[0]).all()
ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
with h5py.File(ref_path_comp.externalPath, "r") as f:
ref_data = f[ref_path_comp.internalPath][:]
expected = ref_data[roiToSlice(*roi)]
# numpy.save('/tmp/expected.npy', expected)
# numpy.save('/tmp/result_out.npy', result_out)
# We can't expect the pixels to match exactly because compression was used to create the tiles...
assert (expected == cached_result_out).all()
assert (expected == uncached_result_out).all()
assert (expected == specified_result_out).all()
def _export_h5n5(self, compress=False):
self.progressSignal(0)
# Create and open the hdf5/n5 file
export_components = PathComponents(self.ExportPath.value)
try:
with OpStreamingH5N5Reader.get_h5_n5_file(
export_components.externalPath, mode="a") as h5N5File:
# Create a temporary operator to do the work for us
opH5N5Writer = OpH5N5WriterBigDataset(parent=self)
with contextlib.suppress(KeyError):
del h5N5File[export_components.internalPath]
try:
opH5N5Writer.CompressionEnabled.setValue(compress)
opH5N5Writer.h5N5File.setValue(h5N5File)
opH5N5Writer.h5N5Path.setValue(
export_components.internalPath)
opH5N5Writer.Image.connect(self.Input)
# The H5 Writer provides it's own progress signal, so just connect ours to it.
opH5N5Writer.progressSignal.subscribe(self.progressSignal)
# Perform the export and block for it in THIS THREAD.
opH5N5Writer.WriteImage[:].wait()
finally:
opH5N5Writer.cleanUp()
self.progressSignal(100)
except IOError as ex:
import sys
msg = "\nException raised when attempting to export to {}: {}\n".format(
export_components.externalPath, str(ex))
sys.stderr.write(msg)
raise
def _getDisplayRoleData(self, index):
laneIndex = index.row()
## Dataset info item
roleIndex = (index.column() - LaneColumn.NumColumns) // DatasetInfoColumn.NumColumns
datasetInfoIndex = (index.column() - LaneColumn.NumColumns) % DatasetInfoColumn.NumColumns
datasetSlot = self._op.DatasetGroup[laneIndex][roleIndex]
if not datasetSlot.ready():
return ""
UninitializedDisplayData = {DatasetInfoColumn.Name: "<please select>"}
datasetSlot = self._op.DatasetGroup[laneIndex][roleIndex]
if datasetSlot.ready():
datasetInfo = self._op.DatasetGroup[laneIndex][roleIndex].value
else:
return UninitializedDisplayData[datasetInfoIndex]
if datasetInfoIndex == DatasetInfoColumn.Name:
if datasetInfo.nickname is not None and datasetInfo.nickname != "":
return datasetInfo.nickname
return PathComponents(datasetInfo.filePath).filename
if datasetInfoIndex == DatasetInfoColumn.Location:
LocationNames = {
RelativeFilesystemDatasetInfo: "External File",
FilesystemDatasetInfo: "External File",
UrlDatasetInfo: "Remote Data",
PreloadedArrayDatasetInfo: "Preloaded Array",
ProjectInternalDatasetInfo: "Project File",
}
return LocationNames[datasetInfo.__class__]
assert False, "Unknown column"
def _getDisplayRoleData(self, index):
laneIndex = index.row()
## Dataset info item
roleIndex = (index.column() -
LaneColumn.NumColumns) // DatasetInfoColumn.NumColumns
datasetInfoIndex = (index.column() - LaneColumn.NumColumns
) % DatasetInfoColumn.NumColumns
datasetSlot = self._op.DatasetGroup[laneIndex][roleIndex]
if not datasetSlot.ready():
return ""
UninitializedDisplayData = {DatasetInfoColumn.Name: "<please select>"}
datasetSlot = self._op.DatasetGroup[laneIndex][roleIndex]
if datasetSlot.ready():
datasetInfo = self._op.DatasetGroup[laneIndex][roleIndex].value
else:
return UninitializedDisplayData[datasetInfoIndex]
if datasetInfoIndex == DatasetInfoColumn.Name:
if datasetInfo.nickname is not None and datasetInfo.nickname != "":
return datasetInfo.nickname
return decode_to_qstring(
PathComponents(datasetInfo.filePath).filename)
if datasetInfoIndex == DatasetInfoColumn.Location:
LocationNames = {
DatasetInfo.Location.FileSystem: "External File",
DatasetInfo.Location.ProjectInternal: "Project File"
}
return LocationNames[datasetInfo.location]
assert False, "Unknown column"
def test_special_z_translation(self):
"""
This tests the special
"""
tiled_volume = TiledVolume(
self.data_setup.SPECIAL_Z_VOLUME_DESCRIPTION_FILE)
tiled_volume.TEST_MODE = True
reference_roi = numpy.array([(20, 150, 100), (40, 550, 550)])
result_out = numpy.zeros(reference_roi[1] - reference_roi[0],
dtype=tiled_volume.description.dtype)
roi_translated = reference_roi - [11, 0, 0]
tiled_volume.read(roi_translated, result_out)
ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH)
with h5py.File(ref_path_comp.externalPath, 'r') as f:
ref_data = f[ref_path_comp.internalPath][:]
expected = ref_data[roiToSlice(*reference_roi)]
#numpy.save('/tmp/expected.npy', expected)
#numpy.save('/tmp/result_out.npy', result_out)
assert (expected == result_out).all()
def _initStorageCombo(self):
# If there's only one dataset, show the path in the combo
showpaths = False
relPath = None
if len( self._laneIndexes ) == 1:
op = self.tempOps.values()[0]
info = op.Dataset.value
cwd = op.WorkingDirectory.value
filePath = PathComponents(info.filePath).externalPath
absPath, relPath = getPathVariants(filePath, cwd)
# commented out:
# Show the paths even if the data is from a stack (they are grayed out, but potentially informative)
#showpaths = not info.fromstack
showpaths = True
if showpaths:
self.storageComboBox.addItem( "Copied to Project File", userData=StorageLocation.ProjectFile )
self.storageComboBox.addItem( decode_to_qstring("Absolute Link: " + absPath), userData=StorageLocation.AbsoluteLink )
if relPath is not None:
self.storageComboBox.addItem( decode_to_qstring("Relative Link: " + relPath), userData=StorageLocation.RelativeLink )
else:
self.storageComboBox.addItem( "Copied to Project File", userData=StorageLocation.ProjectFile )
self.storageComboBox.addItem( "Absolute Link", userData=StorageLocation.AbsoluteLink )
self.storageComboBox.addItem( "Relative Link", userData=StorageLocation.RelativeLink )
self.storageComboBox.setCurrentIndex(-1)
def prompt_export_settings_and_export_layer(layer: Layer,
parent_widget=None) -> None:
"""
Prompt the user for layer export settings, and perform the layer export.
"""
opExport = get_export_operator(layer)
export_dir = PreferencesManager().get("layer",
"export-dir",
default=os.path.expanduser("~"))
opExport.OutputFilenameFormat.setValue(os.path.join(
export_dir, layer.name))
# Use this dialog to populate the operator's slot settings
settingsDlg = DataExportOptionsDlg(parent_widget, opExport)
# If user didn't cancel, run the export now.
if settingsDlg.exec_() == DataExportOptionsDlg.Accepted:
export_dir = PathComponents(
opExport.ExportPath.value).externalDirectory
PreferencesManager().set("layer", "export-dir", export_dir)
helper = ExportHelper(parent_widget)
helper.run(opExport)
# Clean up our temporary operators
opExport.cleanUp()
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName):
"""
Compute the predictions for each of the specified batch input files,
and export them to corresponding h5 files.
"""
batchInputPaths = convertStacksToH5(batchInputPaths)
batchInputInfos = []
for p in batchInputPaths:
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
# Convert all paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
comp = PathComponents(p)
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
batchInputInfos.append(info)
# Configure batch input operator
opBatchInputs = workflow.batchInputApplet.topLevelOperator
opBatchInputs.Dataset.setValues( batchInputInfos )
# Configure batch export operator
opBatchResults = workflow.batchResultsApplet.topLevelOperator
opBatchResults.ExportDirectory.setValue(batchExportDir)
opBatchResults.Format.setValue(ExportFormat.H5)
opBatchResults.Suffix.setValue(batchOutputSuffix)
opBatchResults.InternalPath.setValue(exportedDatasetName)
opBatchResults.SelectedSlices.setValue([30])
logger.info( "Exporting data to " + opBatchResults.OutputDataPath[0].value )
# Set up progress display handling (just logging for now)
currentProgress = [None]
def handleProgress(percentComplete):
if currentProgress[0] != percentComplete:
currentProgress[0] = percentComplete
logger.info("Batch job: {}% complete.".format(percentComplete))
progressSignal = opBatchResults.ProgressSignal[0].value
progressSignal.subscribe( handleProgress )
# Make it happen!
result = opBatchResults.ExportResult[0].value
return result
def getPartiallyFormattedName(self, lane_index, path_format_string):
''' Takes the format string for the output file, fills in the most important placeholders, and returns it '''
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
known_keys['result_type'] = self.dataExportTrackingApplet.topLevelOperator.SelectedPlugin._value
# use partial formatting to fill in non-coordinate name fields
partially_formatted_name = format_known_keys(path_format_string, known_keys)
return partially_formatted_name
def getPartiallyFormattedName(self, lane_index, path_format_string):
''' Takes the format string for the output file, fills in the most important placeholders, and returns it '''
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
known_keys['result_type'] = self.dataExportTrackingApplet.topLevelOperator.SelectedPlugin._value
# use partial formatting to fill in non-coordinate name fields
partially_formatted_name = format_known_keys(path_format_string, known_keys)
return partially_formatted_name
def setupOutputs(self):
self.cleanupOnDiskView()
# FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then
# we have no guarantees about which one will trigger setupOutputs() first.
# It is therefore possible for 'RawDatasetInfo' to appear ready() to us,
# even though it's upstream partner is UNready. We are about to get the
# unready() notification, but it will come too late to prevent our
# setupOutputs method from being called.
# Without proper graph setup transaction semantics, we have to use this
# hack as a workaround.
try:
rawInfo = self.RawDatasetInfo.value
except:
for oslot in self.outputs.values():
if oslot.partner is None:
oslot.meta.NOTREADY = True
return
dataset_dir = PathComponents(rawInfo.filePath).externalDirectory
abs_dataset_dir, _ = getPathVariants(dataset_dir,
self.WorkingDirectory.value)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
known_keys['nickname'] = rawInfo.nickname
# Disconnect to open the 'transaction'
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.disconnect()
self._opFormattedExport.TransactionSlot.disconnect()
# Blank the internal path while we manipulate the external path
# to avoid invalid intermediate states of ExportPath
self._opFormattedExport.OutputInternalPath.setValue("")
# use partial formatting to fill in non-coordinate name fields
name_format = self.OutputFilenameFormat.value
partially_formatted_name = format_known_keys(name_format, known_keys)
# Convert to absolute path before configuring the internal op
abs_path, _ = getPathVariants(partially_formatted_name,
self.WorkingDirectory.value)
self._opFormattedExport.OutputFilenameFormat.setValue(abs_path)
# use partial formatting on the internal dataset name, too
internal_dataset_format = self.OutputInternalPath.value
partially_formatted_dataset_name = format_known_keys(
internal_dataset_format, known_keys)
self._opFormattedExport.OutputInternalPath.setValue(
partially_formatted_dataset_name)
# Re-connect to finish the 'transaction'
self._opFormattedExport.TransactionSlot.connect(self.TransactionSlot)
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.connect(
self.TransactionSlot)
self.setupOnDiskView()
def post_process_lane_export(self, lane_index):
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(
lane_index).get_table_export_settings()
if settings:
self.dataExportApplet.progressSignal.emit(0)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[
lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(
raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(
os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys(
name_format, known_keys)
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(
lane_index
).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportApplet.progressSignal.emit(100)
# Restore state of axis ranges
parameters = self.trackingApplet.topLevelOperator.Parameters.value
parameters['time_range'] = self.prev_time_range
parameters['x_range'] = self.prev_x_range
parameters['y_range'] = self.prev_y_range
parameters['z_range'] = self.prev_z_range
def importStackAsLocalDataset(self, info, sequence_axis='t'):
"""
Add the given stack data to the project file as a local dataset.
Does not update the topLevelOperator.
:param info: A DatasetInfo object.
Note: info.filePath must be a str which lists the stack files, delimited with os.path.pathsep
Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset.
"""
self.progressSignal.emit(0)
projectFileHdf5 = self.topLevelOperator.ProjectFile.value
globstring = info.filePath
info.location = DatasetInfo.Location.ProjectInternal
firstPathParts = PathComponents(info.filePath.split(os.path.pathsep)[0])
info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension
info.fromstack = True
# Use absolute path
cwd = self.topLevelOperator.WorkingDirectory
if os.path.pathsep not in globstring and not os.path.isabs(globstring):
globstring = os.path.normpath( os.path.join(cwd, globstring) )
if firstPathParts.extension.lower() in OpTiffReader.TIFF_EXTS:
# Special loader for TIFFs
opLoader = OpTiffSequenceReader( parent=self.topLevelOperator.parent )
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.GlobString.setValue(globstring)
data_slot = opLoader.Output
else:
# All other sequences (e.g. pngs, jpegs, etc.)
opLoader = OpStackLoader( parent=self.topLevelOperator.parent )
opLoader.SequenceAxis.setValue(sequence_axis)
opLoader.globstring.setValue(globstring)
data_slot = opLoader.stack
try:
opWriter = OpH5WriterBigDataset(parent=self.topLevelOperator.parent)
opWriter.hdf5File.setValue(projectFileHdf5)
opWriter.hdf5Path.setValue(self.topGroupName + '/local_data/' + info.datasetId)
opWriter.CompressionEnabled.setValue(False)
# We assume that the main bottleneck is the hard disk,
# so adding lots of threads to access it at once seems like a bad idea.
opWriter.BatchSize.setValue(1)
opWriter.Image.connect( data_slot )
# Forward progress from the writer directly to our applet
opWriter.progressSignal.subscribe( self.progressSignal.emit )
success = opWriter.WriteImage.value
finally:
opWriter.cleanUp()
opLoader.cleanUp()
self.progressSignal.emit(100)
return success
def getDatasetPathComponents(self, block_start):
"""
Return a PathComponents object for the block file that corresponds to the given block start coordinate.
"""
datasetFilename = self._getBlockFileName(block_start)
datasetDir = self.getDatasetDirectory(block_start)
datasetPath = os.path.join(datasetDir, datasetFilename)
return PathComponents(datasetPath)
def getPartiallyFormattedName(self, lane_index: int, path_format_string: str) -> str:
''' Takes the format string for the output file, fills in the most important placeholders, and returns it '''
raw_dataset_info = self.topLevelOperator.RawDatasetInfo[lane_index].value
project_path = self.topLevelOperator.WorkingDirectory.value
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_path)
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys = {
'dataset_dir': abs_dataset_dir,
'nickname': nickname,
'result_type': self.topLevelOperator.SelectedPlugin._value,
}
return format_known_keys(path_format_string, known_keys)
def append_lane(workflow, input_filepath, axisorder=None):
# Sanity checks
assert isinstance(workflow, PixelClassificationWorkflow)
opPixelClassification = workflow.pcApplet.topLevelOperator
assert opPixelClassification.Classifier.ready()
# If the filepath is a globstring, convert the stack to h5
input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], TMP_DIR )[0]
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug( "adding lane: {}".format( info ) )
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len( opDataSelection.DatasetGroup )+1
logger.debug( "num_lanes: {}".format( num_lanes ) )
opDataSelection.DatasetGroup.resize( num_lanes )
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue( info )
# Sanity check
assert len(opPixelClassification.InputImages) == num_lanes
return opPixelClassification
def append_lane(workflow, input_filepath, axisorder=None):
"""
Add a lane to the project file for the given input file.
If axisorder is given, override the default axisorder for
the file and force the project to use the given one.
Globstrings are supported, in which case the files are converted to HDF5 first.
"""
# If the filepath is a globstring, convert the stack to h5
input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], tempfile.mkdtemp() )[0]
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = input_filepath
comp = PathComponents(input_filepath)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(input_filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
if axisorder:
info.axistags = vigra.defaultAxistags(axisorder)
logger.debug( "adding lane: {}".format( info ) )
opDataSelection = workflow.dataSelectionApplet.topLevelOperator
# Add a lane
num_lanes = len( opDataSelection.DatasetGroup )+1
logger.debug( "num_lanes: {}".format( num_lanes ) )
opDataSelection.DatasetGroup.resize( num_lanes )
# Configure it.
role_index = 0 # raw data
opDataSelection.DatasetGroup[-1][role_index].setValue( info )
def post_process_lane_export(self, lane_index):
settings, selected_features = self.trackingApplet.topLevelOperator.getLane(lane_index).get_table_export_settings()
if settings:
self.dataExportApplet.progressSignal.emit(0)
raw_dataset_info = self.dataSelectionApplet.topLevelOperator.DatasetGroup[lane_index][0].value
project_path = self.shell.projectManager.currentProjectPath
project_dir = os.path.dirname(project_path)
dataset_dir = PathComponents(raw_dataset_info.filePath).externalDirectory
abs_dataset_dir = make_absolute(dataset_dir, cwd=project_dir)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = raw_dataset_info.nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
known_keys['nickname'] = nickname
# use partial formatting to fill in non-coordinate name fields
name_format = settings['file path']
partially_formatted_name = format_known_keys( name_format, known_keys )
settings['file path'] = partially_formatted_name
req = self.trackingApplet.topLevelOperator.getLane(lane_index).export_object_data(
lane_index,
# FIXME: Even in non-headless mode, we can't show the gui because we're running in a non-main thread.
# That's not a huge deal, because there's still a progress bar for the overall export.
show_gui=False)
req.wait()
self.dataExportApplet.progressSignal.emit(100)
# Restore state of axis ranges
parameters = self.trackingApplet.topLevelOperator.Parameters.value
parameters['time_range'] = self.prev_time_range
parameters['x_range'] = self.prev_x_range
parameters['y_range'] = self.prev_y_range
parameters['z_range'] = self.prev_z_range
def create_default_headless_dataset_info(cls, filepath):
"""
filepath may be a globstring or a full hdf5 path+dataset
"""
comp = PathComponents(filepath)
nickname = comp.filenameBase
# Remove globstring syntax.
if '*' in nickname:
nickname = nickname.replace('*', '')
if os.path.pathsep in nickname:
nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.nickname = nickname
info.filePath = filepath
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(filepath):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
return info
f.visit(allkeys.append)
dataset_keys = filter(lambda key: isinstance(f[key], h5py.Dataset),
allkeys)
return dataset_keys
if __name__ == "__main__":
import sys
import argparse
# Construct a parser with all the 'normal' export options, and add arg for input_path.
parser = DataExportApplet.make_cmdline_parser( argparse.ArgumentParser() )
parser.add_argument("input_path", help="Path to your exported predictions.")
parsed_args = parser.parse_args()
# As a convenience, auto-determine the internal dataset path if possible.
path_comp = PathComponents(parsed_args.input_path, os.getcwd())
if path_comp.extension in PathComponents.HDF5_EXTS and path_comp.internalDatasetName == "":
with h5py.File(path_comp.externalPath, 'r') as f:
all_internal_paths = all_dataset_internal_paths(f)
if len(all_internal_paths) == 1:
path_comp.internalPath = all_internal_paths[0]
parsed_args.input_path = path_comp.totalPath()
elif len(all_internal_paths) == 0:
sys.stderr.write("Could not find any datasets in your input file.")
sys.exit(1)
else:
sys.stderr.write("Found more than one dataset in your input file.\n"
"Please specify the dataset name, e.g. /path/to/myfile.h5/internal/dataset_name")
sys.exit(1)
def setupOutputs(self):
self.cleanupOnDiskView()
# FIXME: If RawData becomes unready() at the same time as RawDatasetInfo(), then
# we have no guarantees about which one will trigger setupOutputs() first.
# It is therefore possible for 'RawDatasetInfo' to appear ready() to us,
# even though it's upstream partner is UNready. We are about to get the
# unready() notification, but it will come too late to prevent our
# setupOutputs method from being called.
# Without proper graph setup transaction semantics, we have to use this
# hack as a workaround.
try:
rawInfo = self.RawDatasetInfo.value
except:
for oslot in self.outputs.values():
if oslot.partner is None:
oslot.meta.NOTREADY = True
return
selection_index = self.InputSelection.value
if not self.Inputs[selection_index].ready():
for oslot in self.outputs.values():
if oslot.partner is None:
oslot.meta.NOTREADY = True
return
self._opFormattedExport.Input.connect( self.Inputs[selection_index] )
dataset_dir = PathComponents(rawInfo.filePath).externalDirectory
abs_dataset_dir, _ = getPathVariants(dataset_dir, self.WorkingDirectory.value)
known_keys = {}
known_keys['dataset_dir'] = abs_dataset_dir
nickname = rawInfo.nickname.replace('*', '')
if '//' in nickname:
nickname = PathComponents(nickname.split('//')[0]).fileNameBase
known_keys['nickname'] = nickname
# Disconnect to open the 'transaction'
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.disconnect()
self._opFormattedExport.TransactionSlot.disconnect()
# Blank the internal path while we manipulate the external path
# to avoid invalid intermediate states of ExportPath
self._opFormattedExport.OutputInternalPath.setValue( "" )
# use partial formatting to fill in non-coordinate name fields
name_format = self.OutputFilenameFormat.value
partially_formatted_name = format_known_keys( name_format, known_keys )
# Convert to absolute path before configuring the internal op
abs_path, _ = getPathVariants( partially_formatted_name, self.WorkingDirectory.value )
self._opFormattedExport.OutputFilenameFormat.setValue( abs_path )
# use partial formatting on the internal dataset name, too
internal_dataset_format = self.OutputInternalPath.value
partially_formatted_dataset_name = format_known_keys( internal_dataset_format, known_keys )
self._opFormattedExport.OutputInternalPath.setValue( partially_formatted_dataset_name )
# Re-connect to finish the 'transaction'
self._opFormattedExport.TransactionSlot.connect( self.TransactionSlot )
if self._opImageOnDiskProvider is not None:
self._opImageOnDiskProvider.TransactionSlot.connect( self.TransactionSlot )
self.setupOnDiskView()
return dataset_keys
if __name__ == "__main__":
import sys
import argparse
#sys.argv += "/tmp/example_slice.h5/data /tmp/example_slice2.h5/data --export_drange=(0,255) --output_format=png --pipeline_result_drange=(1,2)".split()
# Construct a parser with all the 'normal' export options, and add arg for prediction_image_paths.
parser = DataExportApplet.make_cmdline_parser( argparse.ArgumentParser() )
parser.add_argument("prediction_image_paths", nargs='+', help="Path(s) to your exported predictions.")
parsed_args = parser.parse_args()
parsed_args, unused_args = DataExportApplet.parse_known_cmdline_args( sys.argv[1:], parsed_args )
# As a convenience, auto-determine the internal dataset path if possible.
for index, input_path in enumerate(parsed_args.prediction_image_paths):
path_comp = PathComponents(input_path, os.getcwd())
if not parsed_args.output_internal_path:
parsed_args.output_internal_path = "segmentation"
if path_comp.extension in PathComponents.HDF5_EXTS and path_comp.internalDatasetName == "":
with h5py.File(path_comp.externalPath, 'r') as f:
all_internal_paths = all_dataset_internal_paths(f)
if len(all_internal_paths) == 1:
path_comp.internalPath = all_internal_paths[0]
parsed_args.prediction_image_paths[index] = path_comp.totalPath()
elif len(all_internal_paths) == 0:
sys.stderr.write("Could not find any datasets in your input file:\n"
"{}\n".format(input_path))
sys.exit(1)
else:
sys.stderr.write("Found more than one dataset in your input file:\n"
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName, stackVolumeCacheDir):
"""
Compute the predictions for each of the specified batch input files,
and export them to corresponding h5 files.
"""
originalBatchInputPaths = list(batchInputPaths)
batchInputPaths = convertStacksToH5(batchInputPaths, stackVolumeCacheDir)
batchInputInfos = []
for p in batchInputPaths:
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
# Convert all paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
comp = PathComponents(p)
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
batchInputInfos.append(info)
# Also convert the export dir to absolute (for the same reason)
if batchExportDir != '':
batchExportDir = os.path.abspath( batchExportDir )
# Configure batch input operator
opBatchInputs = workflow.batchInputApplet.topLevelOperator
opBatchInputs.DatasetGroup.resize( len(batchInputInfos) )
for info, multislot in zip(batchInputInfos, opBatchInputs.DatasetGroup):
# FIXME: This assumes that the workflow has exactly one dataset role.
multislot[0].setValue( info )
# Configure batch export operator
opBatchResults = workflow.batchResultsApplet.topLevelOperator
# By default, the output files from the batch export operator
# are named using the input file name.
# If we converted any stacks to hdf5, then the user won't recognize the input file name.
# Let's override the output file name using the *original* input file names.
outputFileNameBases = []
for origPath in originalBatchInputPaths:
outputFileNameBases.append( origPath.replace('*', 'STACKED') )
opBatchResults.OutputFileNameBase.setValues( outputFileNameBases )
opBatchResults.ExportDirectory.setValue(batchExportDir)
opBatchResults.Format.setValue(ExportFormat.H5)
opBatchResults.Suffix.setValue(batchOutputSuffix)
opBatchResults.InternalPath.setValue(exportedDatasetName)
logger.info( "Exporting data to " + opBatchResults.OutputDataPath[0].value )
# Set up progress display handling (just logging for now)
currentProgress = [None]
def handleProgress(percentComplete):
if currentProgress[0] != percentComplete:
currentProgress[0] = percentComplete
logger.info("Batch job: {}% complete.".format(percentComplete))
progressSignal = opBatchResults.ProgressSignal[0].value
progressSignal.subscribe( handleProgress )
# Make it happen!
result = opBatchResults.ExportResult[0].value
return result
def configure_operator_with_parsed_args(self, parsed_args):
"""
Helper function for headless workflows.
Configures this applet's top-level operator according to the settings provided in ``parsed_args``.
:param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`.
"""
role_names = self.topLevelOperator.DatasetRoles.value
role_paths = collections.OrderedDict()
if role_names:
for role_index, role_name in enumerate(role_names):
arg_name = self._role_name_to_arg_name(role_name)
input_paths = getattr(parsed_args, arg_name)
role_paths[role_index] = input_paths
if parsed_args.input_files:
# We allow the file list to go to the 'default' role, but only if no other roles were explicitly configured.
for role_index, input_paths in role_paths.items():
if input_paths:
# FIXME: This error message could be more helpful.
role_args = map( self._role_name_to_arg_name, role_names )
role_args = map( lambda s: '--' + s, role_args )
role_args_str = ", ".join( role_args )
raise Exception("Invalid command line arguments: All roles must be configured explicitly.\n"
"Use the following flags to specify which files are matched with which inputs:\n"
+ role_args_str )
role_paths = { 0 : parsed_args.input_files }
for role_index, input_paths in role_paths.items():
# If the user doesn't want image stacks to be copied into the project file,
# we generate hdf5 volumes in a temporary directory and use those files instead.
if parsed_args.preconvert_stacks:
import tempfile
input_paths = self.convertStacksToH5( input_paths, tempfile.gettempdir() )
input_infos = []
for p in input_paths:
info = DatasetInfo()
info.location = DatasetInfo.Location.FileSystem
info.filePath = p
comp = PathComponents(p)
# Convert all (non-url) paths to absolute
# (otherwise they are relative to the project file, which probably isn't what the user meant)
if not isUrl(p):
comp.externalPath = os.path.abspath(comp.externalPath)
info.filePath = comp.totalPath()
info.nickname = comp.filenameBase
# Remove globstring syntax.
if '*' in info.nickname:
info.nickname = info.nickname.replace('*', '')
if os.path.pathsep in info.nickname:
info.nickname = PathComponents(info.nickname.split(os.path.pathsep)[0]).fileNameBase
input_infos.append(info)
opDataSelection = self.topLevelOperator
existing_lanes = len(opDataSelection.DatasetGroup)
opDataSelection.DatasetGroup.resize( max(len(input_infos), existing_lanes) )
for lane_index, info in enumerate(input_infos):
opDataSelection.DatasetGroup[lane_index][role_index].setValue( info )
need_warning = False
for lane_index in range(len(input_infos)):
output_slot = opDataSelection.ImageGroup[lane_index][role_index]
if output_slot.meta.prefer_2d:
need_warning = True
break
if need_warning:
logger.warn("*******************************************************************************************")
logger.warn("Some of your input data is stored in a format that is not efficient for 3D access patterns.")
logger.warn("Performance may suffer as a result. For best performance, use a chunked HDF5 volume.")
logger.warn("*******************************************************************************************")
