def addFileNames(self, fileNames): """ Add the given filenames to both the GUI table and the top-level operator inputs. """ with Tracer(traceLogger): # Allocate additional subslots in the operator inputs. oldNumFiles = len(self.mainOperator.Dataset) self.mainOperator.Dataset.resize(oldNumFiles + len(fileNames)) # Assign values to the new inputs we just allocated. # The GUI will be updated by callbacks that are listening to slot changes for i, filePath in enumerate(fileNames): datasetInfo = DatasetInfo() cwd = self.mainOperator.WorkingDirectory.value absPath, relPath = getPathVariants(filePath, cwd) # Relative by default, unless the file is in a totally different tree from the working directory. if len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath else: datasetInfo.filePath = absPath # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = (self.guiMode == GuiMode.Normal) self.mainOperator.Dataset[i + oldNumFiles].setValue(datasetInfo)
def addFileNames(self, fileNames): """ Add the given filenames to both the GUI table and the top-level operator inputs. """ with Tracer(traceLogger): # Allocate additional subslots in the operator inputs. oldNumFiles = len(self.mainOperator.Dataset) self.mainOperator.Dataset.resize( oldNumFiles+len(fileNames) ) # Assign values to the new inputs we just allocated. # The GUI will be updated by callbacks that are listening to slot changes for i, filePath in enumerate(fileNames): datasetInfo = DatasetInfo() cwd = self.mainOperator.WorkingDirectory.value absPath, relPath = getPathVariants(filePath, cwd) # Relative by default, unless the file is in a totally different tree from the working directory. if len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath else: datasetInfo.filePath = absPath h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths( absPath ) if len(datasetNames) > 0: datasetInfo.filePath += str(datasetNames[0]) else: raise RuntimeError("HDF5 file has no image datasets") # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal ) self.mainOperator.Dataset[i+oldNumFiles].setValue( datasetInfo )
def configure_operator_with_parsed_args(self, parsed_args): """ Helper function for headless workflows. Configures this applet's top-level operator according to the settings provided in ``parsed_args``. :param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`. """ # TODO: Support image stack inputs by checking for globstrings and converting to hdf5. input_paths = parsed_args.input_files input_infos = [] for p in input_paths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() info.nickname = comp.filenameBase input_infos.append(info) opDataSelection = self.topLevelOperator opDataSelection.DatasetGroup.resize( len(input_infos) ) for lane_index, info in enumerate(input_infos): # Only one dataset role in pixel classification opDataSelection.DatasetGroup[lane_index][0].setValue( info )
def configure_operator_with_parsed_args(self, parsed_args): """ Helper function for headless workflows. Configures this applet's top-level operator according to the settings provided in ``parsed_args``. :param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`. """ input_paths = parsed_args.input_files # If the user doesn't want image stacks to be copied inte the project file, # we generate hdf5 volumes in a temporary directory and use those files instead. if parsed_args.preconvert_stacks: import tempfile input_paths = self.convertStacksToH5( input_paths, tempfile.gettempdir() ) input_infos = [] for p in input_paths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() info.nickname = comp.filenameBase input_infos.append(info) opDataSelection = self.topLevelOperator opDataSelection.DatasetGroup.resize( len(input_infos) ) for lane_index, info in enumerate(input_infos): # Only one dataset role in pixel classification opDataSelection.DatasetGroup[lane_index][0].setValue( info )
def addStack(self, roleIndex, laneIndex): """ The user clicked the "Import Stack Files" button. """ stackDlg = StackFileSelectionWidget(self) stackDlg.exec_() if stackDlg.result() != QDialog.Accepted : return files = stackDlg.selectedFiles sequence_axis = stackDlg.sequence_axis if len(files) == 0: return info = DatasetInfo() info.filePath = os.path.pathsep.join( files ) prefix = os.path.commonprefix(files) info.nickname = PathComponents(prefix).filenameBase # Add an underscore for each wildcard digit num_wildcards = len(files[-1]) - len(prefix) - len( os.path.splitext(files[-1])[1] ) info.nickname += "_"*num_wildcards # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = ( self.guiMode == GuiMode.Normal ) info.fromstack = True originalNumLanes = len(self.topLevelOperator.DatasetGroup) if laneIndex is None or laneIndex == -1: laneIndex = len(self.topLevelOperator.DatasetGroup) if len(self.topLevelOperator.DatasetGroup) < laneIndex+1: self.topLevelOperator.DatasetGroup.resize(laneIndex+1) def importStack(): self.parentApplet.busy = True self.parentApplet.appletStateUpdateRequested.emit() # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset( info, sequence_axis ) try: self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue(info) except DatasetConstraintError as ex: # Give the user a chance to repair the problem. filename = files[0] + "\n...\n" + files[-1] return_val = [False] self.handleDatasetConstraintError( info, filename, ex, roleIndex, laneIndex, return_val ) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. self.topLevelOperator.DatasetGroup.resize(originalNumLanes) finally: self.parentApplet.busy = False self.parentApplet.appletStateUpdateRequested.emit() req = Request( importStack ) req.notify_finished( lambda result: self.showDataset(laneIndex, roleIndex) ) req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes ) ) req.submit()
def addFileNames(self, fileNames): """ Add the given filenames to both the GUI table and the top-level operator inputs. """ with Tracer(traceLogger): infos = [] oldNumFiles = len(self.topLevelOperator.Dataset) # HACK: If the filePath isn't valid, replace it # This is to work around the scenario where two independent data selection applets are coupled, causing mutual resizes. # This will be fixed when a multi-file data selection applet gui replaces this gui. for i in reversed( range( oldNumFiles ) ): if not self.topLevelOperator.Dataset[i].ready(): oldNumFiles -= 1 else: break # Assign values to the new inputs we just allocated. # The GUI will be updated by callbacks that are listening to slot changes for i, filePath in enumerate(fileNames): datasetInfo = DatasetInfo() cwd = self.topLevelOperator.WorkingDirectory.value if not areOnSameDrive(filePath,cwd): QMessageBox.critical(self, "Drive Error","Data must be on same drive as working directory.") return absPath, relPath = getPathVariants(filePath, cwd) # Relative by default, unless the file is in a totally different tree from the working directory. if len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath else: datasetInfo.filePath = absPath h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths( absPath ) if len(datasetNames) > 0: datasetInfo.filePath += str(datasetNames[0]) else: raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath) # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal ) infos.append(datasetInfo) #if no exception was thrown, set up the operator now self.topLevelOperator.Dataset.resize( oldNumFiles+len(fileNames) ) for i in range(len(infos)): self.topLevelOperator.Dataset[i+oldNumFiles].setValue( infos[i] )
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath): with Tracer(traceLogger): self._projectFilePath = projectFilePath self.initWithoutTopGroup(hdf5File, projectFilePath) infoDir = topGroup['infos'] self.mainOperator.Dataset.resize(len(infoDir)) for index, (infoGroupName, infoGroup) in enumerate(sorted(infoDir.items())): datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = { v: k for k, v in self.LocationStrings.items() } datasetInfo.location = LocationLookup[str( infoGroup['location'].value)] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = str(infoGroup['filePath'].value) datasetInfo._datasetId = str(infoGroup['datasetId'].value) # Deserialize the "allow labels" flag try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass # Deserialize the axisorder (if present) try: datasetInfo.axisorder = infoGroup['axisorder'].value except KeyError: if ilastik.utility.globals.ImportOptions.default_axis_order is not None: datasetInfo.axisorder = ilastik.utility.globals.ImportOptions.default_axis_order # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in topGroup[ 'local_data'].keys(): raise RuntimeError( "Corrupt project file. Could not find data for " + infoGroupName) # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem: filePath = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0]).externalPath if not os.path.exists(filePath): raise RuntimeError("Could not find external data: " + filePath) # Give the new info to the operator self.mainOperator.Dataset[index].setValue(datasetInfo) self._dirty = False
def _createDatasetInfo(self, roleIndex, filePath, roi): """ Create a DatasetInfo object for the given filePath and roi. roi may be None, in which case it is ignored. """ cwd = self.topLevelOperator.WorkingDirectory.value datasetInfo = DatasetInfo(filePath, cwd=cwd) datasetInfo.subvolume_roi = roi # (might be None) absPath, relPath = getPathVariants(filePath, cwd) # If the file is in a totally different tree from the cwd, # then leave the path as absolute. Otherwise, override with the relative path. if relPath is not None and len(os.path.commonprefix([cwd, absPath ])) > 1: datasetInfo.filePath = relPath h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths(absPath) if len(datasetNames) == 0: raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath) elif len(datasetNames) == 1: datasetInfo.filePath += str(datasetNames[0]) else: # If exactly one of the file's datasets matches a user's previous choice, use it. if roleIndex not in self._default_h5_volumes: self._default_h5_volumes[roleIndex] = set() previous_selections = self._default_h5_volumes[roleIndex] possible_auto_selections = previous_selections.intersection( datasetNames) if len(possible_auto_selections) == 1: datasetInfo.filePath += str( list(possible_auto_selections)[0]) else: # Ask the user which dataset to choose dlg = H5VolumeSelectionDlg(datasetNames, self) if dlg.exec_() == QDialog.Accepted: selected_index = dlg.combo.currentIndex() selected_dataset = str(datasetNames[selected_index]) datasetInfo.filePath += selected_dataset self._default_h5_volumes[roleIndex].add( selected_dataset) else: raise DataSelectionGui.UserCancelledError() # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = (self.guiMode == GuiMode.Normal) return datasetInfo
def deserializeFromHdf5(self, hdf5File, projectFilePath): with Tracer(traceLogger): # Check the overall file version ilastikVersion = hdf5File["ilastikVersion"].value # This is the v0.5 import deserializer. Don't work with 0.6 projects (or anything else). if ilastikVersion != 0.5: return # The 'working directory' for the purpose of constructing absolute # paths from relative paths is the project file's directory. projectDir = os.path.split(projectFilePath)[0] self.mainOperator.WorkingDirectory.setValue( projectDir ) # These project file inputs are required, but are not used because the data is treated as "external" self.mainOperator.ProjectDataGroup.setValue( 'DataSets' ) self.mainOperator.ProjectFile.setValue(hdf5File) # Access the top group and the info group try: #dataset = hdf5File["DataSets"]["dataItem00"]["data"] dataDir = hdf5File["DataSets"] except KeyError: # If our group (or subgroup) doesn't exist, then make sure the operator is empty self.mainOperator.Dataset.resize( 0 ) return self.mainOperator.Dataset.resize( len(dataDir) ) for index, (datasetDirName, datasetDir) in enumerate( sorted(dataDir.items()) ): datasetInfo = DatasetInfo() # Since we are importing from a 0.5 file, all datasets will be external # to the project (pulled in from the old file as hdf5 datasets) datasetInfo.location = DatasetInfo.Location.FileSystem # Some older versions of ilastik 0.5 stored the data in tzyxc order. # Some power-users can enable a command-line flag that tells us to # transpose the data back to txyzc order when we import the old project. if ilastik.utility.globals.ImportOptions.default_axis_order is not None: datasetInfo.axisorder = ilastik.utility.globals.ImportOptions.default_axis_order # Write to the 'private' members to avoid resetting the dataset id totalDatasetPath = projectFilePath + '/DataSets/' + datasetDirName + '/data' datasetInfo._filePath = str(totalDatasetPath) datasetInfo._datasetId = datasetDirName # Use the old dataset name as the new dataset id # Give the new info to the operator self.mainOperator.Dataset[index].setValue(datasetInfo)
def deserializeFromHdf5(self, hdf5File, projectFilePath, headless = False): # Check the overall file version ilastikVersion = hdf5File["ilastikVersion"].value # This is the v0.5 import deserializer. Don't work with 0.6 projects (or anything else). if ilastikVersion != 0.5: return # The 'working directory' for the purpose of constructing absolute # paths from relative paths is the project file's directory. projectDir = os.path.split(projectFilePath)[0] self.topLevelOperator.WorkingDirectory.setValue( projectDir ) # Access the top group and the info group try: #dataset = hdf5File["DataSets"]["dataItem00"]["data"] dataDir = hdf5File["DataSets"] except KeyError: # If our group (or subgroup) doesn't exist, then make sure the operator is empty self.topLevelOperator.Dataset.resize( 0 ) return self.topLevelOperator.Dataset.resize( len(dataDir) ) for index, (datasetDirName, datasetDir) in enumerate( sorted(dataDir.items()) ): datasetInfo = DatasetInfo() # We'll set up the link to the dataset in the old project file, # but we'll set the location to ProjectInternal so that it will # be copied to the new file when the project is saved. datasetInfo.location = DatasetInfo.Location.ProjectInternal # Some older versions of ilastik 0.5 stored the data in tzyxc order. # Some power-users can enable a command-line flag that tells us to # transpose the data back to txyzc order when we import the old project. default_axis_order = ilastik.utility.globals.ImportOptions.default_axis_order if default_axis_order is not None: import warnings warnings.warn( "Using a strange axis order to import ilastik 0.5 projects: {}".format( default_axis_order ) ) datasetInfo.axisorder = default_axis_order # Write to the 'private' members to avoid resetting the dataset id totalDatasetPath = projectFilePath + '/DataSets/' + datasetDirName + '/data' datasetInfo._filePath = str(totalDatasetPath) datasetInfo._datasetId = datasetDirName # Use the old dataset name as the new dataset id # Give the new info to the operator self.topLevelOperator.Dataset[index].setValue(datasetInfo)
def replaceWithStack(self, roleIndex, laneIndex): """ The user clicked the "Import Stack Files" button. """ stackDlg = StackFileSelectionWidget(self) stackDlg.exec_() if stackDlg.result() != QDialog.Accepted : return files = stackDlg.selectedFiles if len(files) == 0: return info = DatasetInfo() info.filePath = "//".join( files ) prefix = os.path.commonprefix(files) info.nickname = PathComponents(prefix).filenameBase + "..." # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = ( self.guiMode == GuiMode.Normal ) info.fromstack = True originalNumLanes = len(self.topLevelOperator.DatasetGroup) if laneIndex is None: laneIndex = self._findFirstEmptyLane(roleIndex) if len(self.topLevelOperator.DatasetGroup) < laneIndex+1: self.topLevelOperator.DatasetGroup.resize(laneIndex+1) def importStack(): self.guiControlSignal.emit( ControlCommand.DisableAll ) # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset( info ) try: self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue(info) except DatasetConstraintError as ex: # Give the user a chance to repair the problem. filename = files[0] + "\n...\n" + files[-1] if not self.handleDatasetConstraintError( info, filename, ex, roleIndex, laneIndex ): self.topLevelOperator.DatasetGroup.resize(originalNumLanes) finally: self.guiControlSignal.emit( ControlCommand.Pop ) req = Request( importStack ) req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes ) ) req.submit()
def _createDatasetInfo(self, roleIndex, filePath, roi): """ Create a DatasetInfo object for the given filePath and roi. roi may be None, in which case it is ignored. """ cwd = self.topLevelOperator.WorkingDirectory.value datasetInfo = DatasetInfo(filePath, cwd=cwd) datasetInfo.subvolume_roi = roi # (might be None) absPath, relPath = getPathVariants(filePath, cwd) # If the file is in a totally different tree from the cwd, # then leave the path as absolute. Otherwise, override with the relative path. if relPath is not None and len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths( absPath ) if len(datasetNames) == 0: raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath) elif len(datasetNames) == 1: datasetInfo.filePath += str(datasetNames[0]) else: # If exactly one of the file's datasets matches a user's previous choice, use it. if roleIndex not in self._default_h5_volumes: self._default_h5_volumes[roleIndex] = set() previous_selections = self._default_h5_volumes[roleIndex] possible_auto_selections = previous_selections.intersection(datasetNames) if len(possible_auto_selections) == 1: datasetInfo.filePath += str(list(possible_auto_selections)[0]) else: # Ask the user which dataset to choose dlg = H5VolumeSelectionDlg(datasetNames, self) if dlg.exec_() == QDialog.Accepted: selected_index = dlg.combo.currentIndex() selected_dataset = str(datasetNames[selected_index]) datasetInfo.filePath += selected_dataset self._default_h5_volumes[roleIndex].add( selected_dataset ) else: raise DataSelectionGui.UserCancelledError() # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal ) return datasetInfo
def importStackFromGlobString(self, globString): """ The word 'glob' is used loosely here. See the OpStackLoader operator for details. """ info = DatasetInfo() info.filePath = globString # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = ( self.guiMode == GuiMode.Normal ) def importStack(): self.guiControlSignal.emit( ControlCommand.DisableAll ) # Serializer will update the operator for us, which will propagate to the GUI. self.serializer.importStackAsLocalDataset( info ) self.guiControlSignal.emit( ControlCommand.Pop ) importThread = threading.Thread( target=importStack ) importThread.start()
def importStackFromGlobString(self, globString): """ The word 'glob' is used loosely here. See the OpStackLoader operator for details. """ info = DatasetInfo() info.filePath = globString # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = (self.guiMode == GuiMode.Normal) def importStack(): self.guiControlSignal.emit(ControlCommand.DisableAll) # Serializer will update the operator for us, which will propagate to the GUI. self.serializer.importStackAsLocalDataset(info) self.guiControlSignal.emit(ControlCommand.Pop) importThread = threading.Thread(target=importStack) importThread.start()
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath, headless): self._projectFilePath = projectFilePath self.initWithoutTopGroup(hdf5File, projectFilePath) # normally the serializer is not dirty after loading a project file # however, when the file was corrupted, the user has the possibility # to save the fixed file after loading it. dirty = False infoDir = topGroup['infos'] self.topLevelOperator.Dataset.resize( len(infoDir) ) for index, (infoGroupName, infoGroup) in enumerate( sorted(infoDir.items()) ): datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = { v:k for k,v in self.LocationStrings.items() } datasetInfo.location = LocationLookup[ str(infoGroup['location'].value) ] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = str(infoGroup['filePath'].value) datasetInfo._datasetId = str(infoGroup['datasetId'].value) # Deserialize the "allow labels" flag try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass # Deserialize the axisorder (if present) try: datasetInfo.axisorder = infoGroup['axisorder'].value except KeyError: pass # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in topGroup['local_data'].keys(): raise RuntimeError("Corrupt project file. Could not find data for " + infoGroupName) # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem: pathData = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0]) filePath = pathData.externalPath if not os.path.exists(filePath): if headless: raise RuntimeError("Could not find data at " + filePath) filt = "Image files (" + ' '.join('*.' + x for x in OpDataSelection.SupportedExtensions) + ')' newpath = self.repairFile(filePath, filt) newpath = newpath+pathData.internalPath datasetInfo._filePath = getPathVariants(newpath , os.path.split(projectFilePath)[0])[0] dirty = True # Give the new info to the operator self.topLevelOperator.Dataset[index].setValue(datasetInfo) self._dirty = dirty
def configureRoleFromJson(self, lane, role, dataset_info_namespace): opDataSelection = self.topLevelOperator logger.debug( "Configuring dataset for role {}".format( role ) ) logger.debug( "Params: {}".format(dataset_info_namespace) ) datasetInfo = DatasetInfo() datasetInfo.updateFromJson( dataset_info_namespace ) # Check for globstring, which means we need to import the stack first. if '*' in datasetInfo.filePath: totalProgress = [-100] def handleStackImportProgress( progress ): if progress / 10 != totalProgress[0] / 10: totalProgress[0] = progress logger.info( "Importing stack: {}%".format( totalProgress[0] ) ) serializer = self.dataSerializers[0] serializer.progressSignal.connect( handleStackImportProgress ) serializer.importStackAsLocalDataset( datasetInfo ) opDataSelection.DatasetGroup[lane][role].setValue( datasetInfo )
def configure_operator_with_parsed_args(self, parsed_args): """ Helper function for headless workflows. Configures this applet's top-level operator according to the settings provided in ``parsed_args``. :param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`. """ role_names = self.topLevelOperator.DatasetRoles.value role_paths = self.role_paths_from_parsed_args(parsed_args, role_names) for role_index, input_paths in role_paths.items(): # If the user doesn't want image stacks to be copied into the project file, # we generate hdf5 volumes in a temporary directory and use those files instead. if parsed_args.preconvert_stacks: import tempfile input_paths = self.convertStacksToH5(input_paths, tempfile.gettempdir()) input_infos = [DatasetInfo(p) if p else None for p in input_paths] if parsed_args.input_axes: for info in filter(None, input_infos): info.axistags = vigra.defaultAxistags( parsed_args.input_axes) opDataSelection = self.topLevelOperator existing_lanes = len(opDataSelection.DatasetGroup) opDataSelection.DatasetGroup.resize( max(len(input_infos), existing_lanes)) for lane_index, info in enumerate(input_infos): if info: opDataSelection.DatasetGroup[lane_index][ role_index].setValue(info) need_warning = False for lane_index in range(len(input_infos)): output_slot = opDataSelection.ImageGroup[lane_index][ role_index] if output_slot.ready( ) and output_slot.meta.prefer_2d and 'z' in output_slot.meta.axistags: need_warning = True break if need_warning: logger.warn( "*******************************************************************************************" ) logger.warn( "Some of your input data is stored in a format that is not efficient for 3D access patterns." ) logger.warn( "Performance may suffer as a result. For best performance, use a chunked HDF5 volume." ) logger.warn( "*******************************************************************************************" )
def addStack(self, roleIndex, laneIndex): """ The user clicked the "Import Stack Files" button. """ stackDlg = StackFileSelectionWidget(self) stackDlg.exec_() if stackDlg.result() != QDialog.Accepted: return files = stackDlg.selectedFiles sequence_axis = stackDlg.sequence_axis if len(files) == 0: return cwd = self.topLevelOperator.WorkingDirectory.value info = DatasetInfo(os.path.pathsep.join(files), cwd=cwd) originalNumLanes = len(self.topLevelOperator.DatasetGroup) if laneIndex is None or laneIndex == -1: laneIndex = len(self.topLevelOperator.DatasetGroup) if len(self.topLevelOperator.DatasetGroup) < laneIndex + 1: self.topLevelOperator.DatasetGroup.resize(laneIndex + 1) def importStack(): self.parentApplet.busy = True self.parentApplet.appletStateUpdateRequested.emit() # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset(info, sequence_axis) try: self.topLevelOperator.DatasetGroup[laneIndex][ roleIndex].setValue(info) except DatasetConstraintError as ex: # Give the user a chance to repair the problem. filename = files[0] + "\n...\n" + files[-1] return_val = [False] self.handleDatasetConstraintError(info, filename, ex, roleIndex, laneIndex, return_val) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. self.topLevelOperator.DatasetGroup.resize( originalNumLanes) finally: self.parentApplet.busy = False self.parentApplet.appletStateUpdateRequested.emit() req = Request(importStack) req.notify_finished( lambda result: self.showDataset(laneIndex, roleIndex)) req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes)) req.submit()
def configureRoleFromJson(self, lane, role, dataset_info_namespace): assert sys.version_info.major == 2, "Alert! This function has not been tested "\ "under python 3. Please remove this assetion and be wary of any strnage behavior you encounter" opDataSelection = self.topLevelOperator logger.debug( "Configuring dataset for role {}".format( role ) ) logger.debug( "Params: {}".format(dataset_info_namespace) ) datasetInfo = DatasetInfo() datasetInfo.updateFromJson( dataset_info_namespace ) # Check for globstring, which means we need to import the stack first. if '*' in datasetInfo.filePath: totalProgress = [-100] def handleStackImportProgress( progress ): if progress // 10 != totalProgress[0] // 10: totalProgress[0] = progress logger.info( "Importing stack: {}%".format( totalProgress[0] ) ) serializer = self.dataSerializers[0] serializer.progressSignal.connect( handleStackImportProgress ) serializer.importStackAsLocalDataset( datasetInfo ) opDataSelection.DatasetGroup[lane][role].setValue( datasetInfo )
def importStackFromGlobString(self, globString): """ The word 'glob' is used loosely here. See the OpStackLoader operator for details. """ globString = globString.replace("\\","/") info = DatasetInfo() info.filePath = globString # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = ( self.guiMode == GuiMode.Normal ) def importStack(): self.guiControlSignal.emit( ControlCommand.DisableAll ) # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset( info ) finally: self.guiControlSignal.emit( ControlCommand.Pop ) req = Request( importStack ) req.notify_failed( partial(self.handleFailedStackLoad, globString ) ) req.submit()
def deserializeFromHdf5(self, hdf5File, projectFilePath, headless=False): # Check the overall file version ilastikVersion = hdf5File["ilastikVersion"].value # This is the v0.5 import deserializer. Don't work with 0.6 projects (or anything else). if ilastikVersion != 0.5: return # The 'working directory' for the purpose of constructing absolute # paths from relative paths is the project file's directory. projectDir = os.path.split(projectFilePath)[0] self.topLevelOperator.WorkingDirectory.setValue(projectDir) # Access the top group and the info group try: #dataset = hdf5File["DataSets"]["dataItem00"]["data"] dataDir = hdf5File["DataSets"] except KeyError: # If our group (or subgroup) doesn't exist, then make sure the operator is empty self.topLevelOperator.DatasetGroup.resize(0) return self.topLevelOperator.DatasetGroup.resize(len(dataDir)) for index, (datasetDirName, datasetDir) in enumerate(sorted(dataDir.items())): datasetInfo = DatasetInfo() # We'll set up the link to the dataset in the old project file, # but we'll set the location to ProjectInternal so that it will # be copied to the new file when the project is saved. datasetInfo.location = DatasetInfo.Location.ProjectInternal # Some older versions of ilastik 0.5 stored the data in tzyxc order. # Some power-users can enable a command-line flag that tells us to # transpose the data back to txyzc order when we import the old project. default_axis_order = ilastik.utility.globals.ImportOptions.default_axis_order if default_axis_order is not None: import warnings warnings.warn( "Using a strange axis order to import ilastik 0.5 projects: {}" .format(default_axis_order)) datasetInfo.axistags = vigra.defaultAxistags( default_axis_order) # Write to the 'private' members to avoid resetting the dataset id totalDatasetPath = str(projectFilePath + '/DataSets/' + datasetDirName + '/data') datasetInfo._filePath = totalDatasetPath datasetInfo._datasetId = datasetDirName # Use the old dataset name as the new dataset id datasetInfo.nickname = "{} (imported from v0.5)".format( datasetDirName) # Give the new info to the operator self.topLevelOperator.DatasetGroup[index][0].setValue(datasetInfo)
def replaceWithStack(self, roleIndex, laneIndex): """ The user clicked the "Import Stack Files" button. """ stackDlg = StackFileSelectionWidget(self) stackDlg.exec_() if stackDlg.result() != QDialog.Accepted: return files = stackDlg.selectedFiles if len(files) == 0: return info = DatasetInfo() info.filePath = "//".join(files) prefix = os.path.commonprefix(files) info.nickname = PathComponents(prefix).filenameBase # Add an underscore for each wildcard digit num_wildcards = len(files[-1]) - len(prefix) - len( os.path.splitext(files[-1])[1]) info.nickname += "_" * num_wildcards # Allow labels by default if this gui isn't being used for batch data. info.allowLabels = (self.guiMode == GuiMode.Normal) info.fromstack = True originalNumLanes = len(self.topLevelOperator.DatasetGroup) if laneIndex is None: laneIndex = self._findFirstEmptyLane(roleIndex) if len(self.topLevelOperator.DatasetGroup) < laneIndex + 1: self.topLevelOperator.DatasetGroup.resize(laneIndex + 1) def importStack(): self.guiControlSignal.emit(ControlCommand.DisableAll) # Serializer will update the operator for us, which will propagate to the GUI. try: self.serializer.importStackAsLocalDataset(info) try: self.topLevelOperator.DatasetGroup[laneIndex][ roleIndex].setValue(info) except DatasetConstraintError as ex: # Give the user a chance to repair the problem. filename = files[0] + "\n...\n" + files[-1] return_val = [False] self.handleDatasetConstraintError(info, filename, ex, roleIndex, laneIndex, return_val) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. self.topLevelOperator.DatasetGroup.resize( originalNumLanes) finally: self.guiControlSignal.emit(ControlCommand.Pop) req = Request(importStack) req.notify_failed( partial(self.handleFailedStackLoad, files, originalNumLanes)) req.submit()
def configureRoleFromJson(self, lane, role, dataset_info_namespace): assert sys.version_info.major == 2, "Alert! This function has not been tested "\ "under python 3. Please remove this assetion and be wary of any strnage behavior you encounter" opDataSelection = self.topLevelOperator logger.debug("Configuring dataset for role {}".format(role)) logger.debug("Params: {}".format(dataset_info_namespace)) datasetInfo = DatasetInfo() datasetInfo.updateFromJson(dataset_info_namespace) # Check for globstring, which means we need to import the stack first. if '*' in datasetInfo.filePath: totalProgress = [-100] def handleStackImportProgress(progress): if progress // 10 != totalProgress[0] // 10: totalProgress[0] = progress logger.info("Importing stack: {}%".format( totalProgress[0])) serializer = self.dataSerializers[0] serializer.progressSignal.connect(handleStackImportProgress) serializer.importStackAsLocalDataset(datasetInfo) opDataSelection.DatasetGroup[lane][role].setValue(datasetInfo)
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath): with Tracer(traceLogger): self._projectFilePath = projectFilePath self.initWithoutTopGroup(hdf5File, projectFilePath) infoDir = topGroup['infos'] self.mainOperator.Dataset.resize( len(infoDir) ) for index, (infoGroupName, infoGroup) in enumerate( sorted(infoDir.items()) ): datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = { v:k for k,v in self.LocationStrings.items() } datasetInfo.location = LocationLookup[ str(infoGroup['location'].value) ] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = str(infoGroup['filePath'].value) datasetInfo._datasetId = str(infoGroup['datasetId'].value) # Deserialize the "allow labels" flag try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass # Deserialize the axisorder (if present) try: datasetInfo.axisorder = infoGroup['axisorder'].value except KeyError: if ilastik.utility.globals.ImportOptions.default_axis_order is not None: datasetInfo.axisorder = ilastik.utility.globals.ImportOptions.default_axis_order # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in topGroup['local_data'].keys(): raise RuntimeError("Corrupt project file. Could not find data for " + infoGroupName) # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem: filePath = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0] ).externalPath if not os.path.exists(filePath): raise RuntimeError("Could not find external data: " + filePath) # Give the new info to the operator self.mainOperator.Dataset[index].setValue(datasetInfo) self._dirty = False
def deserializeFromHdf5(self, hdf5File, projectFilePath): with Tracer(traceLogger): # Check the overall file version ilastikVersion = hdf5File["ilastikVersion"].value # This is the v0.5 import deserializer. Don't work with 0.6 projects (or anything else). if ilastikVersion != 0.5: return # The 'working directory' for the purpose of constructing absolute # paths from relative paths is the project file's directory. projectDir = os.path.split(projectFilePath)[0] self.mainOperator.WorkingDirectory.setValue(projectDir) # These project file inputs are required, but are not used because the data is treated as "external" self.mainOperator.ProjectDataGroup.setValue('DataSets') self.mainOperator.ProjectFile.setValue(hdf5File) # Access the top group and the info group try: #dataset = hdf5File["DataSets"]["dataItem00"]["data"] dataDir = hdf5File["DataSets"] except KeyError: # If our group (or subgroup) doesn't exist, then make sure the operator is empty self.mainOperator.Dataset.resize(0) return self.mainOperator.Dataset.resize(len(dataDir)) for index, (datasetDirName, datasetDir) in enumerate(sorted(dataDir.items())): datasetInfo = DatasetInfo() # Since we are importing from a 0.5 file, all datasets will be external # to the project (pulled in from the old file as hdf5 datasets) datasetInfo.location = DatasetInfo.Location.FileSystem # Some older versions of ilastik 0.5 stored the data in tzyxc order. # Some power-users can enable a command-line flag that tells us to # transpose the data back to txyzc order when we import the old project. if ilastik.utility.globals.ImportOptions.default_axis_order is not None: datasetInfo.axisorder = ilastik.utility.globals.ImportOptions.default_axis_order # Write to the 'private' members to avoid resetting the dataset id totalDatasetPath = projectFilePath + '/DataSets/' + datasetDirName + '/data' datasetInfo._filePath = str(totalDatasetPath) datasetInfo._datasetId = datasetDirName # Use the old dataset name as the new dataset id # Give the new info to the operator self.mainOperator.Dataset[index].setValue(datasetInfo)
def create_default_headless_dataset_info(cls, filepath): """ filepath may be a globstring or a full hdf5 path+dataset """ comp = PathComponents(filepath) nickname = comp.filenameBase # Remove globstring syntax. if '*' in nickname: nickname = nickname.replace('*', '') if os.path.pathsep in nickname: nickname = PathComponents(nickname.split(os.path.pathsep)[0]).fileNameBase info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.nickname = nickname info.filePath = filepath # Convert all (non-url) paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) if not isUrl(filepath): comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() return info
def _readDatasetInfo(self, infoGroup, localDataGroup, projectFilePath, headless): # Unready datasets are represented with an empty group. if len( infoGroup ) == 0: return None, False datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = { v:k for k,v in self.LocationStrings.items() } datasetInfo.location = LocationLookup[ str(infoGroup['location'].value) ] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = infoGroup['filePath'].value datasetInfo._datasetId = infoGroup['datasetId'].value try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass try: datasetInfo.drange = tuple( infoGroup['drange'].value ) except KeyError: pass try: datasetInfo.nickname = infoGroup['nickname'].value except KeyError: datasetInfo.nickname = PathComponents(datasetInfo.filePath).filenameBase try: tags = vigra.AxisTags.fromJSON( infoGroup['axistags'].value ) datasetInfo.axistags = tags except KeyError: # Old projects just have an 'axisorder' field instead of full axistags try: axisorder = infoGroup['axisorder'].value datasetInfo.axistags = vigra.defaultAxistags(axisorder) except KeyError: pass # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in localDataGroup.keys(): raise RuntimeError("Corrupt project file. Could not find data for " + infoGroup.name) dirty = False # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem and not isUrl(datasetInfo.filePath): pathData = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0]) filePath = pathData.externalPath if not os.path.exists(filePath): if headless: raise RuntimeError("Could not find data at " + filePath) filt = "Image files (" + ' '.join('*.' + x for x in OpDataSelection.SupportedExtensions) + ')' newpath = self.repairFile(filePath, filt) if pathData.internalPath is not None: newpath += pathData.internalPath datasetInfo._filePath = getPathVariants(newpath , os.path.split(projectFilePath)[0])[0] dirty = True return datasetInfo, dirty
def addFileNames(self, fileNames, roleIndex, startingLane=None): """ Add the given filenames to both the GUI table and the top-level operator inputs. If startingLane is None, the filenames will be *appended* to the role's list of files. """ infos = [] if startingLane is None or startingLane == -1: startingLane = len(self.topLevelOperator.DatasetGroup) endingLane = startingLane+len(fileNames)-1 else: assert startingLane < len(self.topLevelOperator.DatasetGroup) max_files = len(self.topLevelOperator.DatasetGroup) - \ startingLane if len(fileNames) > max_files: msg = "You selected {num_selected} files for {num_slots} "\ "slots. To add new files use the 'Add new...' option "\ "in the context menu or the button in the last row."\ .format(num_selected=len(fileNames), num_slots=max_files) QMessageBox.critical( self, "Too many files", msg ) return endingLane = min(startingLane+len(fileNames)-1, len(self.topLevelOperator.DatasetGroup)) if self._max_lanes and endingLane >= self._max_lanes: msg = "You may not add more than {} file(s) to this workflow. Please try again.".format( self._max_lanes ) QMessageBox.critical( self, "Too many files", msg ) return # Assign values to the new inputs we just allocated. # The GUI will be updated by callbacks that are listening to slot changes for i, filePath in enumerate(fileNames): datasetInfo = DatasetInfo() cwd = self.topLevelOperator.WorkingDirectory.value absPath, relPath = getPathVariants(filePath, cwd) # Relative by default, unless the file is in a totally different tree from the working directory. if relPath is not None and len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath else: datasetInfo.filePath = absPath datasetInfo.nickname = PathComponents(absPath).filenameBase h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths( absPath ) if len(datasetNames) > 0: datasetInfo.filePath += str(datasetNames[0]) else: raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath) # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal ) infos.append(datasetInfo) # if no exception was thrown, set up the operator now opTop = self.topLevelOperator originalSize = len(opTop.DatasetGroup) if len( opTop.DatasetGroup ) < endingLane+1: opTop.DatasetGroup.resize( endingLane+1 ) for laneIndex, info in zip(range(startingLane, endingLane+1), infos): try: self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue( info ) except DatasetConstraintError as ex: return_val = [False] # Give the user a chance to fix the problem self.handleDatasetConstraintError(info, info.filePath, ex, roleIndex, laneIndex, return_val) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. opTop.DatasetGroup.resize( originalSize ) break except OpDataSelection.InvalidDimensionalityError as ex: opTop.DatasetGroup.resize( originalSize ) QMessageBox.critical( self, "Dataset has different dimensionality", ex.message ) break except: QMessageBox.critical( self, "Dataset Load Error", "Wasn't able to load your dataset into the workflow. See console for details." ) opTop.DatasetGroup.resize( originalSize ) raise # If we succeeded in adding all images, show the first one. if laneIndex == endingLane: self.showDataset(startingLane, roleIndex) # Notify the workflow that something that could affect applet readyness has occurred. self.parentApplet.appletStateUpdateRequested.emit() self.updateInternalPathVisiblity()
def addFileNames(self, fileNames, roleIndex, startingLane=None): """ Add the given filenames to both the GUI table and the top-level operator inputs. If startingLane is None, the filenames will be *appended* to the role's list of files. """ infos = [] if startingLane is None: startingLane = self._findFirstEmptyLane(roleIndex) endingLane = startingLane + len(fileNames) - 1 else: assert startingLane < len(self.topLevelOperator.DatasetGroup) endingLane = startingLane + len(fileNames) - 1 if self._max_lanes and endingLane >= self._max_lanes: msg = "You may not add more than {} file(s) to this workflow. Please try again.".format( self._max_lanes) QMessageBox.critical(self, "Too many files", msg) return # Assign values to the new inputs we just allocated. # The GUI will be updated by callbacks that are listening to slot changes for i, filePath in enumerate(fileNames): datasetInfo = DatasetInfo() cwd = self.topLevelOperator.WorkingDirectory.value if not areOnSameDrive(filePath, cwd): QMessageBox.critical( self, "Drive Error", "Data must be on same drive as working directory.") return absPath, relPath = getPathVariants(filePath, cwd) # Relative by default, unless the file is in a totally different tree from the working directory. if len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath else: datasetInfo.filePath = absPath datasetInfo.nickname = PathComponents(absPath).filenameBase h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths(absPath) if len(datasetNames) > 0: datasetInfo.filePath += str(datasetNames[0]) else: raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath) # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = (self.guiMode == GuiMode.Normal) infos.append(datasetInfo) # if no exception was thrown, set up the operator now opTop = self.topLevelOperator originalSize = len(opTop.DatasetGroup) if len(opTop.DatasetGroup) < endingLane + 1: opTop.DatasetGroup.resize(endingLane + 1) for laneIndex, info in zip(range(startingLane, endingLane + 1), infos): try: self.topLevelOperator.DatasetGroup[laneIndex][ roleIndex].setValue(info) except DatasetConstraintError as ex: return_val = [False] # Give the user a chance to fix the problem self.handleDatasetConstraintError(info, info.filePath, ex, roleIndex, laneIndex, return_val) if not return_val[0]: # Not successfully repaired. Roll back the changes and give up. opTop.DatasetGroup.resize(originalSize) break except OpDataSelection.InvalidDimensionalityError as ex: opTop.DatasetGroup.resize(originalSize) QMessageBox.critical(self, "Dataset has different dimensionality", ex.message) break except: QMessageBox.critical( self, "Dataset Load Error", "Wasn't able to load your dataset into the workflow. See console for details." ) opTop.DatasetGroup.resize(originalSize) raise self.updateInternalPathVisiblity()
def configure_operator_with_parsed_args(self, parsed_args): """ Helper function for headless workflows. Configures this applet's top-level operator according to the settings provided in ``parsed_args``. :param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`. """ role_names = self.topLevelOperator.DatasetRoles.value role_paths = collections.OrderedDict() if role_names: for role_index, role_name in enumerate(role_names): arg_name = self._role_name_to_arg_name(role_name) input_paths = getattr(parsed_args, arg_name) role_paths[role_index] = input_paths if parsed_args.input_files: # We allow the file list to go to the 'default' role, but only if no other roles were explicitly configured. for role_index, input_paths in role_paths.items(): if input_paths: # FIXME: This error message could be more helpful. role_args = map( self._role_name_to_arg_name, role_names ) role_args = map( lambda s: '--' + s, role_args ) role_args_str = ", ".join( role_args ) raise Exception("Invalid command line arguments: All roles must be configured explicitly.\n" "Use the following flags to specify which files are matched with which inputs:\n" + role_args_str ) role_paths = { 0 : parsed_args.input_files } for role_index, input_paths in role_paths.items(): # If the user doesn't want image stacks to be copied into the project file, # we generate hdf5 volumes in a temporary directory and use those files instead. if parsed_args.preconvert_stacks: import tempfile input_paths = self.convertStacksToH5( input_paths, tempfile.gettempdir() ) input_infos = [] for p in input_paths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.filePath = p comp = PathComponents(p) # Convert all (non-url) paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) if not isUrl(p): comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() info.nickname = comp.filenameBase # Remove globstring syntax. if '*' in info.nickname: info.nickname = info.nickname.replace('*', '') if os.path.pathsep in info.nickname: info.nickname = PathComponents(info.nickname.split(os.path.pathsep)[0]).fileNameBase input_infos.append(info) opDataSelection = self.topLevelOperator existing_lanes = len(opDataSelection.DatasetGroup) opDataSelection.DatasetGroup.resize( max(len(input_infos), existing_lanes) ) for lane_index, info in enumerate(input_infos): opDataSelection.DatasetGroup[lane_index][role_index].setValue( info ) need_warning = False for lane_index in range(len(input_infos)): output_slot = opDataSelection.ImageGroup[lane_index][role_index] if output_slot.meta.prefer_2d: need_warning = True break if need_warning: logger.warn("*******************************************************************************************") logger.warn("Some of your input data is stored in a format that is not efficient for 3D access patterns.") logger.warn("Performance may suffer as a result. For best performance, use a chunked HDF5 volume.") logger.warn("*******************************************************************************************")
def configure_operator_with_parsed_args(self, parsed_args): """ Helper function for headless workflows. Configures this applet's top-level operator according to the settings provided in ``parsed_args``. :param parsed_args: Must be an ``argparse.Namespace`` as returned by :py:meth:`parse_known_cmdline_args()`. """ role_names = self.topLevelOperator.DatasetRoles.value role_paths = collections.OrderedDict() if role_names: for role_index, role_name in enumerate(role_names): arg_name = self._role_name_to_arg_name(role_name) input_paths = getattr(parsed_args, arg_name) role_paths[role_index] = input_paths if parsed_args.input_files: # We allow the file list to go to the 'default' role, but only if no other roles were explicitly configured. for role_index, input_paths in role_paths.items(): if input_paths: # FIXME: This error message could be more helpful. role_args = map(self._role_name_to_arg_name, role_names) role_args = map(lambda s: '--' + s, role_args) role_args_str = ", ".join(role_args) raise Exception( "Invalid command line arguments: All roles must be configured explicitly.\n" "Use the following flags to specify which files are matched with which inputs:\n" + role_args_str) role_paths = {0: parsed_args.input_files} for role_index, input_paths in role_paths.items(): # If the user doesn't want image stacks to be copied into the project file, # we generate hdf5 volumes in a temporary directory and use those files instead. if parsed_args.preconvert_stacks: import tempfile input_paths = self.convertStacksToH5(input_paths, tempfile.gettempdir()) input_infos = [] for p in input_paths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.filePath = p comp = PathComponents(p) # Convert all (non-url) paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) if not isUrl(p): comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() info.nickname = comp.filenameBase # Remove globstring syntax. if '*' in info.nickname: info.nickname = info.nickname.replace('*', '') if os.path.pathsep in info.nickname: info.nickname = PathComponents( info.nickname.split(os.path.pathsep)[0]).fileNameBase input_infos.append(info) opDataSelection = self.topLevelOperator existing_lanes = len(opDataSelection.DatasetGroup) opDataSelection.DatasetGroup.resize( max(len(input_infos), existing_lanes)) for lane_index, info in enumerate(input_infos): opDataSelection.DatasetGroup[lane_index][role_index].setValue( info) need_warning = False for lane_index in range(len(input_infos)): output_slot = opDataSelection.ImageGroup[lane_index][ role_index] if output_slot.meta.prefer_2d: need_warning = True break if need_warning: logger.warn( "*******************************************************************************************" ) logger.warn( "Some of your input data is stored in a format that is not efficient for 3D access patterns." ) logger.warn( "Performance may suffer as a result. For best performance, use a chunked HDF5 volume." ) logger.warn( "*******************************************************************************************" )
def addFileNames(self, fileNames, roleIndex, startingLane=None): """ Add the given filenames to both the GUI table and the top-level operator inputs. If startingLane is None, the filenames will be *appended* to the role's list of files. """ infos = [] if startingLane is None: startingLane = self._findFirstEmptyLane(roleIndex) endingLane = startingLane+len(fileNames)-1 else: assert startingLane < len(self.topLevelOperator.DatasetGroup) endingLane = startingLane+len(fileNames)-1 # Assign values to the new inputs we just allocated. # The GUI will be updated by callbacks that are listening to slot changes for i, filePath in enumerate(fileNames): datasetInfo = DatasetInfo() cwd = self.topLevelOperator.WorkingDirectory.value if not areOnSameDrive(filePath,cwd): QMessageBox.critical(self, "Drive Error","Data must be on same drive as working directory.") return absPath, relPath = getPathVariants(filePath, cwd) # Relative by default, unless the file is in a totally different tree from the working directory. if len(os.path.commonprefix([cwd, absPath])) > 1: datasetInfo.filePath = relPath else: datasetInfo.filePath = absPath datasetInfo.nickname = PathComponents(absPath).filenameBase h5Exts = ['.ilp', '.h5', '.hdf5'] if os.path.splitext(datasetInfo.filePath)[1] in h5Exts: datasetNames = self.getPossibleInternalPaths( absPath ) if len(datasetNames) > 0: datasetInfo.filePath += str(datasetNames[0]) else: raise RuntimeError("HDF5 file %s has no image datasets" % datasetInfo.filePath) # Allow labels by default if this gui isn't being used for batch data. datasetInfo.allowLabels = ( self.guiMode == GuiMode.Normal ) infos.append(datasetInfo) # if no exception was thrown, set up the operator now opTop = self.topLevelOperator originalSize = len(opTop.DatasetGroup) if len( opTop.DatasetGroup ) < endingLane+1: opTop.DatasetGroup.resize( endingLane+1 ) for laneIndex, info in zip(range(startingLane, endingLane+1), infos): try: self.topLevelOperator.DatasetGroup[laneIndex][roleIndex].setValue( info ) except DatasetConstraintError as ex: # Give the user a chance to fix the problem if not self.handleDatasetConstraintError(info, info.filePath, ex, roleIndex, laneIndex): opTop.DatasetGroup.resize( originalSize ) break except: QMessageBox.critical( self, "Dataset Load Error", "Wasn't able to load your dataset into the workflow. See console for details." ) opTop.DatasetGroup.resize( originalSize ) raise self.updateInternalPathVisiblity()
def _readDatasetInfo(self, infoGroup, localDataGroup, projectFilePath, headless): # Unready datasets are represented with an empty group. if len( infoGroup ) == 0: return None, False datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = { v:k for k,v in self.LocationStrings.items() } datasetInfo.location = LocationLookup[ str(infoGroup['location'].value) ] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = infoGroup['filePath'].value datasetInfo._datasetId = infoGroup['datasetId'].value try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass try: datasetInfo.drange = tuple( infoGroup['drange'].value ) except KeyError: pass try: datasetInfo.nickname = infoGroup['nickname'].value except KeyError: datasetInfo.nickname = PathComponents(datasetInfo.filePath).filenameBase try: datasetInfo.fromstack = infoGroup['fromstack'].value except KeyError: # Guess based on the storage setting and original filepath datasetInfo.fromstack = ( datasetInfo.location == DatasetInfo.Location.ProjectInternal and ( ('?' in datasetInfo._filePath) or (os.path.pathsep in datasetInfo._filePath) ) ) try: tags = vigra.AxisTags.fromJSON( infoGroup['axistags'].value ) datasetInfo.axistags = tags except KeyError: # Old projects just have an 'axisorder' field instead of full axistags try: axisorder = infoGroup['axisorder'].value datasetInfo.axistags = vigra.defaultAxistags(axisorder) except KeyError: pass try: start, stop = map( tuple, infoGroup['subvolume_roi'].value ) datasetInfo.subvolume_roi = (start, stop) except KeyError: pass # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in localDataGroup.keys(): raise RuntimeError("Corrupt project file. Could not find data for " + infoGroup.name) dirty = False # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem and not isUrl(datasetInfo.filePath): pathData = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0]) filePath = pathData.externalPath if not os.path.exists(filePath): if headless: raise RuntimeError("Could not find data at " + filePath) filt = "Image files (" + ' '.join('*.' + x for x in OpDataSelection.SupportedExtensions) + ')' newpath = self.repairFile(filePath, filt) if pathData.internalPath is not None: newpath += pathData.internalPath datasetInfo._filePath = getPathVariants(newpath , os.path.split(projectFilePath)[0])[0] dirty = True return datasetInfo, dirty