def _initInternalDatasetNameCombo(self): # If any dataset is either (1) not hdf5 or (2) project-internal, then we can't change the internal path. h5Exts = ['.ilp', '.h5', '.hdf5'] for laneIndex in self._laneIndexes: tmpOp = self.tempOps[laneIndex] datasetInfo = tmpOp.Dataset.value externalPath = PathComponents(datasetInfo.filePath).externalPath if os.path.splitext(externalPath)[1] not in h5Exts \ or datasetInfo.location == DatasetInfo.Location.ProjectInternal: self.internalDatasetNameComboBox.addItem("N/A") self.internalDatasetNameComboBox.setEnabled(False) return # Enable IFF all datasets have at least one common internal dataset, and only show COMMON datasets allInternalPaths = set() commonInternalPaths = None for laneIndex in self._laneIndexes: tmpOp = self.tempOps[laneIndex] datasetInfo = tmpOp.Dataset.value externalPath = PathComponents(datasetInfo.filePath).externalPath absPath, relPath = getPathVariants(externalPath, tmpOp.WorkingDirectory.value) internalPaths = set(self._getPossibleInternalPaths(absPath)) if commonInternalPaths is None: # Init with the first file's set of paths commonInternalPaths = internalPaths # Set operations allInternalPaths |= internalPaths commonInternalPaths &= internalPaths if len(commonInternalPaths) == 0: self.internalDatasetNameComboBox.addItem( "Couldn't find a dataset name common to all selected files." ) self.internalDatasetNameComboBox.setEnabled(False) return uncommonInternalPaths = allInternalPaths - commonInternalPaths # Add all common paths to the combo for path in sorted(commonInternalPaths): self.internalDatasetNameComboBox.addItem(path) # Add the remaining ones, but disable them since they aren't common to all files: for path in sorted(uncommonInternalPaths): self.internalDatasetNameComboBox.addItem(path) # http://theworldwideinternet.blogspot.com/2011/01/disabling-qcombobox-items.html model = self.internalDatasetNameComboBox.model() index = model.index(self.internalDatasetNameComboBox.count() - 1, 0) model.setData(index, 0, Qt.UserRole - 1) # Finally, initialize with NO item selected self.internalDatasetNameComboBox.setCurrentIndex(-1)
def _applyInternalPathToTempOps(self, index): if index == -1: return newInternalPath = str(self.internalDatasetNameComboBox.currentText()) # Save a copy of our settings oldInfos = {} for laneIndex, op in self.tempOps.items(): oldInfos[laneIndex] = copy.copy(op.Dataset.value) # Attempt to apply to all temp operators try: for laneIndex, op in self.tempOps.items(): info = copy.copy(op.Dataset.value) pathComponents = PathComponents(info.filePath) if pathComponents.internalPath != newInternalPath: pathComponents.internalPath = newInternalPath info.filePath = pathComponents.totalPath() op.Dataset.setValue(info) self._error_fields.discard('Internal Dataset Name') return True except Exception as e: # Revert everything back to the previous state for laneIndex, op in self.tempOps.items(): op.Dataset.setValue(oldInfos[laneIndex]) traceback.print_exc() msg = "Could not set new internal path settings due to an exception:\n" msg += "{}".format(e) QMessageBox.warning(self, "Error", msg) self._error_fields.add('Internal Dataset Name') return False
def _deserializeFromHdf5(self, topGroup, groupVersion, hdf5File, projectFilePath): with Tracer(traceLogger): self._projectFilePath = projectFilePath self.initWithoutTopGroup(hdf5File, projectFilePath) infoDir = topGroup['infos'] self.mainOperator.Dataset.resize(len(infoDir)) for index, (infoGroupName, infoGroup) in enumerate(sorted(infoDir.items())): datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = { v: k for k, v in self.LocationStrings.items() } datasetInfo.location = LocationLookup[str( infoGroup['location'].value)] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = str(infoGroup['filePath'].value) datasetInfo._datasetId = str(infoGroup['datasetId'].value) # Deserialize the "allow labels" flag try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass # Deserialize the axisorder (if present) try: datasetInfo.axisorder = infoGroup['axisorder'].value except KeyError: if ilastik.utility.globals.ImportOptions.default_axis_order is not None: datasetInfo.axisorder = ilastik.utility.globals.ImportOptions.default_axis_order # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in topGroup[ 'local_data'].keys(): raise RuntimeError( "Corrupt project file. Could not find data for " + infoGroupName) # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem: filePath = PathComponents( datasetInfo.filePath, os.path.split(projectFilePath)[0]).externalPath if not os.path.exists(filePath): raise RuntimeError("Could not find external data: " + filePath) # Give the new info to the operator self.mainOperator.Dataset[index].setValue(datasetInfo) self._dirty = False
def handleFinished( result ): # Generate the mapping transforms dataset mapping = self._opAccumulateFinalImage.Mapping.value num_labels = mapping.keys()[-1][1] transform = numpy.zeros( shape=(num_labels, 2), dtype=numpy.uint32 ) for (start, stop), body_id in mapping.items(): for supervoxel_label in range(start, stop): transform[supervoxel_label][0] = supervoxel_label if body_id == -1: # Special case: -1 means "identity transform" for this supervoxel # (Which is really an untouched raveler body) transform[supervoxel_label][1] = supervoxel_label else: transform[supervoxel_label][1] = body_id # Save the transform before closing the file f.create_dataset('transforms', data=transform) # Copy all other datasets from the original segmentation file. ravelerSegmentationInfo = self.DatasetInfos[2].value pathComponents = PathComponents(ravelerSegmentationInfo.filePath, self.WorkingDirectory.value) with h5py.File(pathComponents.externalPath, 'r') as originalFile: for k,dset in originalFile.items(): if k not in ['transforms', 'stack']: f.copy(dset, k) try: cleanOps() logger.info("FINISHED Final Supervoxel Export") finally: f.close()
def _initStorageCombo(self): # If there's only one dataset, show the path in the combo showpaths = False if len(self._laneIndexes) == 1: op = self.tempOps.values()[0] info = op.Dataset.value cwd = op.WorkingDirectory.value filePath = PathComponents(info.filePath).externalPath absPath, relPath = getPathVariants(filePath, cwd) showpaths = not info.fromstack if showpaths: self.storageComboBox.addItem("Copied to Project File", userData=StorageLocation.ProjectFile) self.storageComboBox.addItem("Absolute Link: " + absPath, userData=StorageLocation.AbsoluteLink) self.storageComboBox.addItem("Relative Link: " + relPath, userData=StorageLocation.RelativeLink) else: self.storageComboBox.addItem("Copied to Project File", userData=StorageLocation.ProjectFile) self.storageComboBox.addItem("Absolute Link", userData=StorageLocation.AbsoluteLink) self.storageComboBox.addItem("Relative Link", userData=StorageLocation.RelativeLink) self.storageComboBox.setCurrentIndex(-1)
def testCreateExportDirectory(self): """ Test that the batch operator can create the export directory if it doesn't exist yet. """ # Start by writing some test data to disk. self.testData = numpy.random.random((1, 10, 10, 10, 1)) numpy.save(self.testDataFileName, self.testData) cwd = os.getcwd() info = DatasetInfo() info.filePath = os.path.join(cwd, 'NpyTestData.npy') graph = Graph() opBatchIo = OpBatchIo(graph=graph) opInput = OpInputDataReader(graph=graph) opInput.FilePath.setValue(info.filePath) # Our test "processing pipeline" is just a smoothing operator. opSmooth = OpGaussianSmoothing(graph=graph) opSmooth.Input.connect(opInput.Output) opSmooth.sigma.setValue(3.0) exportDir = os.path.join(cwd, 'exported_data') opBatchIo.ExportDirectory.setValue(exportDir) opBatchIo.Suffix.setValue('_smoothed') opBatchIo.Format.setValue(ExportFormat.H5) opBatchIo.DatasetPath.setValue(info.filePath) internalPath = 'path/to/data' opBatchIo.InternalPath.setValue(internalPath) opBatchIo.ImageToExport.connect(opSmooth.Output) dirty = opBatchIo.Dirty.value assert dirty == True outputPath = opBatchIo.OutputDataPath.value assert outputPath == os.path.join(exportDir, 'NpyTestData_smoothed.h5', internalPath) result = opBatchIo.ExportResult.value assert result dirty = opBatchIo.Dirty.value assert dirty == False # Check the file smoothedPath = PathComponents(outputPath).externalPath with h5py.File(smoothedPath, 'r') as f: assert internalPath in f assert f[internalPath].shape == self.testData.shape try: os.remove(smoothedPath) os.rmdir(exportDir) except: pass
def runWorkflow(parsed_args): args = parsed_args # Make sure project file exists. if not os.path.exists(args.project): raise RuntimeError("Project file '" + args.project + "' does not exist.") # Make sure batch inputs exist. for p in args.batch_inputs: error = False p = PathComponents(p).externalPath if not os.path.exists(p): logger.error("Batch input file does not exist: " + p) error = True if error: raise RuntimeError("Could not find one or more batch inputs. See logged errors.") if not args.generate_project_predictions and len(args.batch_inputs) == 0: logger.error("Command-line arguments didn't specify a workload.") return # Instantiate 'shell' shell, workflow = startShellHeadless( PixelClassificationWorkflow ) if args.assume_old_ilp_axes: # Special hack for Janelia: # In some old versions of 0.5, the data was stored in tyxzc order. # We have no way of inspecting the data to determine this, so we allow # users to specify that their ilp is very old using the # assume_old_ilp_axes command-line flag ilastik.utility.globals.ImportOptions.default_axis_order = 'tyxzc' # Load project (auto-import it if necessary) logger.info("Opening project: '" + args.project + "'") shell.openProjectPath(args.project) # Predictions for project input datasets if args.generate_project_predictions: generateProjectPredictions(shell, workflow) # Predictions for other datasets ('batch datasets') result = True if len(args.batch_inputs) > 0: result = generateBatchPredictions(workflow, args.batch_inputs, args.batch_export_dir, args.batch_output_suffix, args.batch_output_dataset_name) logger.info("Closing project...") shell.projectManager.closeCurrentProject() assert result logger.info("FINISHED.")
def hasInternalPaths(self): for mslot in self._op.DatasetGroup: if self._roleIndex < len(mslot): slot = mslot[self._roleIndex] if slot.ready(): datasetInfo = slot.value filePathComponents = PathComponents(datasetInfo.filePath) if (datasetInfo.location == DatasetInfo.Location.FileSystem and filePathComponents.internalPath is not None): return True return False
def deleteAllResults(self): for k in xrange(len(self.topLevelOperator)): operatorView = self.topLevelOperator.getLane(k) operatorView.cleanupPreview() pathComp = PathComponents(operatorView.OutputDataPath.value, operatorView.WorkingDirectory.value) os.remove(pathComp.externalPath) operatorView.setupPreview() # we need to toggle the dirts state in order to enforce a frech dirty signal operatorView.Dirty.setValue(False) operatorView.Dirty.setValue(True)
def deleteAllResults(self): for innerOp in self.topLevelOperator: operatorView = innerOp operatorView.cleanupOnDiskView() pathComp = PathComponents(operatorView.ExportPath.value, operatorView.WorkingDirectory.value) if os.path.exists(pathComp.externalPath): os.remove(pathComp.externalPath) operatorView.setupOnDiskView() # we need to toggle the dirts state in order to enforce a frech dirty signal operatorView.Dirty.setValue( False ) operatorView.Dirty.setValue( True )
def updateWorkingDirectory(self, newpath, oldpath): newdir = PathComponents(newpath).externalDirectory olddir = PathComponents(oldpath).externalDirectory if newdir == olddir: return # Disconnect the working directory while we make these changes. # All the changes will take effect when we set the new working directory. self.topLevelOperator.WorkingDirectory.disconnect() for laneIndex, multislot in enumerate( self.topLevelOperator.DatasetGroup): for roleIndex, slot in enumerate(multislot): if not slot.ready(): # Skip if there is no dataset in this lane/role combination yet. continue datasetInfo = slot.value if datasetInfo.location == DatasetInfo.Location.FileSystem: #construct absolute path and recreate relative to the new path fp = PathComponents(datasetInfo.filePath, olddir).totalPath() abspath, relpath = getPathVariants(fp, newdir) # Same convention as in dataSelectionGui: # Relative by default, unless the file is in a totally different tree from the working directory. if len(os.path.commonprefix([fp, abspath])) > 1: datasetInfo.filePath = relpath else: datasetInfo.filePath = abspath slot.setValue(datasetInfo, check_changed=False) self.topLevelOperator.WorkingDirectory.setValue(newdir) self._projectFilePath = newdir
def _getDisplayRoleData(self, index): # Last row is just buttons if index.row() >= self.rowCount() - 1: return "" laneIndex = index.row() if index.column() < LaneColumn.NumColumns: if index.column() == LaneColumn.LabelsAllowed: firstInfoSlot = self._op.DatasetGroup[laneIndex][0] if not firstInfoSlot.ready(): return "" info = firstInfoSlot.value return {True: "True", False: "False"}[info.allowLabels] else: assert False ## Dataset info item roleIndex = (index.column() - LaneColumn.NumColumns) // DatasetInfoColumn.NumColumns datasetInfoIndex = (index.column() - LaneColumn.NumColumns ) % DatasetInfoColumn.NumColumns datasetSlot = self._op.DatasetGroup[laneIndex][roleIndex] if not datasetSlot.ready(): return "" UninitializedDisplayData = {DatasetInfoColumn.Name: "<please select>"} datasetSlot = self._op.DatasetGroup[laneIndex][roleIndex] if datasetSlot.ready(): datasetInfo = self._op.DatasetGroup[laneIndex][roleIndex].value else: return UninitializedDisplayData[datasetInfoIndex] if datasetInfoIndex == DatasetInfoColumn.Name: if datasetInfo.nickname is not None and datasetInfo.nickname != "": return datasetInfo.nickname return PathComponents(datasetInfo.filePath).filename if datasetInfoIndex == DatasetInfoColumn.Location: LocationNames = { DatasetInfo.Location.FileSystem: "External File", DatasetInfo.Location.ProjectInternal: "Project File" } return LocationNames[datasetInfo.location] assert False, "Unknown column"
def runWorkflow(parsed_args): args = parsed_args # Make sure project file exists. if not os.path.exists(args.project): raise RuntimeError("Project file '" + args.project + "' does not exist.") # Make sure batch inputs exist. for p in args.batch_inputs: print p error = False p = PathComponents(p).externalPath if not os.path.exists(p): logger.error("Batch input file does not exist: " + p) error = True if error: raise RuntimeError("Could not find one or more batch inputs. See logged errors.") if not args.generate_project_predictions and len(args.batch_inputs) == 0: logger.error("Command-line arguments didn't specify a workload.") return # Instantiate 'shell' shell, workflow = startShellHeadless( AutocontextClassificationWorkflow ) # Load project (auto-import it if necessary) logger.info("Opening project: '" + args.project + "'") shell.openProjectPath(args.project) # Predictions for project input datasets if args.generate_project_predictions: generateProjectPredictions(shell, workflow) # Predictions for other datasets ('batch datasets') result = True if len(args.batch_inputs) > 0: result = generateBatchPredictions(workflow, args.batch_inputs, args.batch_export_dir, args.batch_output_suffix, args.batch_output_dataset_name) logger.info("Closing project...") shell.projectManager.closeCurrentProject() assert result logger.info("FINISHED.")
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName): """ Compute the predictions for each of the specified batch input files, and export them to corresponding h5 files. """ batchInputPaths = convertStacksToH5(batchInputPaths) batchInputInfos = [] for p in batchInputPaths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() batchInputInfos.append(info) # Configure batch input operator opBatchInputs = workflow.batchInputApplet.topLevelOperator opBatchInputs.Dataset.setValues( batchInputInfos ) # Configure batch export operator opBatchResults = workflow.batchResultsApplet.topLevelOperator opBatchResults.ExportDirectory.setValue(batchExportDir) opBatchResults.Format.setValue(ExportFormat.H5) opBatchResults.Suffix.setValue(batchOutputSuffix) opBatchResults.InternalPath.setValue(exportedDatasetName) opBatchResults.SelectedSlices.setValue([30]) logger.info( "Exporting data to " + opBatchResults.OutputDataPath[0].value ) # Set up progress display handling (just logging for now) currentProgress = [None] def handleProgress(percentComplete): if currentProgress[0] != percentComplete: currentProgress[0] = percentComplete logger.info("Batch job: {}% complete.".format(percentComplete)) progressSignal = opBatchResults.ProgressSignal[0].value progressSignal.subscribe( handleProgress ) # Make it happen! result = opBatchResults.ExportResult[0].value return result
def importStackAsLocalDataset(self, info): """ Add the given stack data to the project file as a local dataset. Does not update the topLevelOperator. :param info: A DatasetInfo object. Note: info.filePath must be a stack files must be separated by '//' tokens. Note: info will be MODIFIED by this function. Use the modified info when assigning it to a dataset. """ try: self.progressSignal.emit(0) projectFileHdf5 = self.topLevelOperator.ProjectFile.value topGroup = getOrCreateGroup(projectFileHdf5, self.topGroupName) localDataGroup = getOrCreateGroup(topGroup, 'local_data') globstring = info.filePath info.location = DatasetInfo.Location.ProjectInternal firstPathParts = PathComponents(info.filePath.split('//')[0]) info.filePath = firstPathParts.externalDirectory + '/??' + firstPathParts.extension # Use absolute path cwd = self.topLevelOperator.WorkingDirectory if '//' not in globstring and not os.path.isabs(globstring): globstring = os.path.normpath(os.path.join(cwd, globstring)) opWriter = OpStackToH5Writer(parent=self.topLevelOperator.parent, graph=self.topLevelOperator.graph) opWriter.hdf5Group.setValue(localDataGroup) opWriter.hdf5Path.setValue(info.datasetId) opWriter.GlobString.setValue(globstring) # Forward progress from the writer directly to our applet opWriter.progressSignal.subscribe(self.progressSignal.emit) success = opWriter.WriteImage.value finally: opWriter.cleanUp() self.progressSignal.emit(100) return success
def _updateInternalDatasetSelection(self): # If all lanes have the same dataset selected, choose that item. # Otherwise, leave it uninitialized if not self.internalDatasetNameComboBox.isEnabled(): return internalPath = None for laneIndex in self._laneIndexes: tmpOp = self.tempOps[laneIndex] datasetInfo = tmpOp.Dataset.value nextPath = PathComponents(datasetInfo.filePath).internalPath if internalPath is None: internalPath = nextPath # init if internalPath != nextPath: self.internalDatasetNameComboBox.setCurrentIndex(-1) return # Make sure the correct index is selected. index = self.internalDatasetNameComboBox.findText(internalPath) self.internalDatasetNameComboBox.setCurrentIndex(index)
def _serializeToHdf5(self, topGroup, hdf5File, projectFilePath): with Tracer(traceLogger): # Write any missing local datasets to the local_data group localDataGroup = self.getOrCreateGroup(topGroup, 'local_data') wroteInternalData = False for index, slot in enumerate(self.mainOperator.Dataset): info = slot.value # If this dataset should be stored in the project, but it isn't there yet if info.location == DatasetInfo.Location.ProjectInternal \ and info.datasetId not in localDataGroup.keys(): # Obtain the data from the corresponding output and store it to the project. dataSlot = self.mainOperator.Image[index] opWriter = OpH5WriterBigDataset( graph=self.mainOperator.graph) opWriter.hdf5File.setValue(localDataGroup) opWriter.hdf5Path.setValue(info.datasetId) opWriter.Image.connect(dataSlot) # Trigger the copy success = opWriter.WriteImage.value assert success # Add the axistags attribute to the dataset we just created localDataGroup[info.datasetId].attrs[ 'axistags'] = dataSlot.meta.axistags.toJSON() # Update the dataset info with no path, just filename base to remind us what this data is # (operator will react to the change when we call setValue(), below) # Directly set the private member to avoid getting a new datasetid info._filePath = PathComponents(info.filePath).filenameBase wroteInternalData = True # Construct a list of all the local dataset ids we want to keep localDatasetIds = [ slot.value.datasetId for index, slot in enumerate(self.mainOperator.Dataset) if slot.value.location == DatasetInfo.Location.ProjectInternal ] # Delete any datasets in the project that aren't needed any more for datasetName in localDataGroup.keys(): if datasetName not in localDatasetIds: del localDataGroup[datasetName] if wroteInternalData: # We can only re-configure the operator if we're not saving a snapshot # We know we're saving a snapshot if the project file isn't the one we deserialized with. if self._projectFilePath is None or self._projectFilePath == projectFilePath: # Force the operator to setupOutputs() again so it gets data from the project, not external files firstInfo = self.mainOperator.Dataset[0].value self.mainOperator.Dataset[0].setValue(firstInfo, False) # Access the info group infoDir = self.getOrCreateGroup(topGroup, 'infos') # Delete all infos for infoName in infoDir.keys(): del infoDir[infoName] # Rebuild the list of infos for index, slot in enumerate(self.mainOperator.Dataset): infoGroup = infoDir.create_group('info{:04d}'.format(index)) datasetInfo = slot.value locationString = self.LocationStrings[datasetInfo.location] infoGroup.create_dataset('location', data=locationString) infoGroup.create_dataset('filePath', data=datasetInfo.filePath) infoGroup.create_dataset('datasetId', data=datasetInfo.datasetId) infoGroup.create_dataset('allowLabels', data=datasetInfo.allowLabels) if datasetInfo.axisorder is not None: infoGroup.create_dataset('axisorder', data=datasetInfo.axisorder) self._dirty = False
def _readDatasetInfo(self, infoGroup, localDataGroup, projectFilePath, headless): # Unready datasets are represented with an empty group. if len(infoGroup) == 0: return None, False datasetInfo = DatasetInfo() # Make a reverse-lookup of the location storage strings LocationLookup = {v: k for k, v in self.LocationStrings.items()} datasetInfo.location = LocationLookup[str(infoGroup['location'].value)] # Write to the 'private' members to avoid resetting the dataset id datasetInfo._filePath = str(infoGroup['filePath'].value) datasetInfo._datasetId = str(infoGroup['datasetId'].value) try: datasetInfo.allowLabels = infoGroup['allowLabels'].value except KeyError: pass try: datasetInfo.drange = tuple(infoGroup['drange'].value) except KeyError: pass try: datasetInfo.nickname = str(infoGroup['nickname'].value) except KeyError: datasetInfo.nickname = PathComponents( datasetInfo.filePath).filenameBase try: tags = vigra.AxisTags.fromJSON(infoGroup['axistags'].value) datasetInfo.axistags = tags except KeyError: # Old projects just have an 'axisorder' field instead of full axistags try: axisorder = infoGroup['axisorder'].value datasetInfo.axistags = vigra.defaultAxistags(axisorder) except KeyError: pass # If the data is supposed to be in the project, # check for it now. if datasetInfo.location == DatasetInfo.Location.ProjectInternal: if not datasetInfo.datasetId in localDataGroup.keys(): raise RuntimeError( "Corrupt project file. Could not find data for " + infoGroup.name) dirty = False # If the data is supposed to exist outside the project, make sure it really does. if datasetInfo.location == DatasetInfo.Location.FileSystem: pathData = PathComponents(datasetInfo.filePath, os.path.split(projectFilePath)[0]) filePath = pathData.externalPath if not os.path.exists(filePath): if headless: raise RuntimeError("Could not find data at " + filePath) filt = "Image files (" + ' '.join( '*.' + x for x in OpDataSelection.SupportedExtensions) + ')' newpath = self.repairFile(filePath, filt) if pathData.internalPath is not None: newpath += pathData.internalPath datasetInfo._filePath = getPathVariants( newpath, os.path.split(projectFilePath)[0])[0] dirty = True return datasetInfo, dirty
def _getDisplayRoleData(self, index): laneIndex = index.row() UninitializedDisplayData = { DatasetDetailedInfoColumn.Nickname: "<empty>", DatasetDetailedInfoColumn.Location: "", DatasetDetailedInfoColumn.InternalID: "", DatasetDetailedInfoColumn.AxisOrder: "", DatasetDetailedInfoColumn.Shape: "", DatasetDetailedInfoColumn.Range: "" } if len( self._op.DatasetGroup ) <= laneIndex \ or len( self._op.DatasetGroup[laneIndex] ) <= self._roleIndex: return UninitializedDisplayData[index.column()] datasetSlot = self._op.DatasetGroup[laneIndex][self._roleIndex] # Default if not datasetSlot.ready(): return UninitializedDisplayData[index.column()] datasetInfo = self._op.DatasetGroup[laneIndex][self._roleIndex].value filePathComponents = PathComponents(datasetInfo.filePath) ## Input meta-data fields # Name if index.column() == DatasetDetailedInfoColumn.Nickname: return datasetInfo.nickname # Location if index.column() == DatasetDetailedInfoColumn.Location: if datasetInfo.location == DatasetInfo.Location.FileSystem: if os.path.isabs(datasetInfo.filePath): return "Absolute Link: {}".format( filePathComponents.externalPath) else: return "Relative Link: {}".format( filePathComponents.externalPath) else: return "Project File" # Internal ID if index.column() == DatasetDetailedInfoColumn.InternalID: if datasetInfo.location == DatasetInfo.Location.FileSystem: return filePathComponents.internalPath return "" ## Output meta-data fields # Defaults imageSlot = self._op.ImageGroup[laneIndex][self._roleIndex] if not imageSlot.ready(): return UninitializedDisplayData[index.column()] # Axis order if index.column() == DatasetDetailedInfoColumn.AxisOrder: original_axistags = imageSlot.meta.original_axistags axistags = imageSlot.meta.axistags if original_axistags is not None: return "".join(tag.key for tag in original_axistags) if axistags is not None: return "".join(imageSlot.meta.getAxisKeys()) return "" # Shape if index.column() == DatasetDetailedInfoColumn.Shape: original_shape = imageSlot.meta.original_shape shape = imageSlot.meta.shape if original_shape is not None: return str(original_shape) if shape is None: return "" return str(shape) # Range if index.column() == DatasetDetailedInfoColumn.Range: drange = imageSlot.meta.drange if drange is None: return "" return str(drange) assert False, "Unknown column: row={}, column={}".format( index.row(), index.column())
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName, stackVolumeCacheDir): """ Compute the predictions for each of the specified batch input files, and export them to corresponding h5 files. """ originalBatchInputPaths = list(batchInputPaths) batchInputPaths = convertStacksToH5(batchInputPaths, stackVolumeCacheDir) batchInputInfos = [] for p in batchInputPaths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() batchInputInfos.append(info) # Also convert the export dir to absolute (for the same reason) if batchExportDir != '': batchExportDir = os.path.abspath(batchExportDir) # Configure batch input operator opBatchInputs = workflow.batchInputApplet.topLevelOperator opBatchInputs.DatasetGroup.resize(len(batchInputInfos)) for info, multislot in zip(batchInputInfos, opBatchInputs.DatasetGroup): # FIXME: This assumes that the workflow has exactly one dataset role. multislot[0].setValue(info) # Configure batch export operator opBatchResults = workflow.batchResultsApplet.topLevelOperator # By default, the output files from the batch export operator # are named using the input file name. # If we converted any stacks to hdf5, then the user won't recognize the input file name. # Let's override the output file name using the *original* input file names. outputFileNameBases = [] for origPath in originalBatchInputPaths: outputFileNameBases.append(origPath.replace('*', 'STACKED')) opBatchResults.OutputFileNameBase.setValues(outputFileNameBases) opBatchResults.ExportDirectory.setValue(batchExportDir) opBatchResults.Format.setValue(ExportFormat.H5) opBatchResults.Suffix.setValue(batchOutputSuffix) opBatchResults.InternalPath.setValue(exportedDatasetName) logger.info("Exporting data to " + opBatchResults.OutputDataPath[0].value) # Set up progress display handling (just logging for now) currentProgress = [None] def handleProgress(percentComplete): if currentProgress[0] != percentComplete: currentProgress[0] = percentComplete logger.info("Batch job: {}% complete.".format(percentComplete)) progressSignal = opBatchResults.ProgressSignal[0].value progressSignal.subscribe(handleProgress) # Make it happen! result = opBatchResults.ExportResult[0].value return result
def runWorkflow(parsed_args): args = parsed_args # Use a temporary cache dir if args.stack_volume_cache_dir is None: args.stack_volume_cache_dir = tempfile.gettempdir() # Make sure project file exists. if not os.path.exists(args.project): raise RuntimeError("Project file '" + args.project + "' does not exist.") # Make sure batch inputs exist. for p in args.batch_inputs: error = False p = PathComponents(p).externalPath if '*' in p: if len(glob.glob(p)) == 0: logger.error( "Could not find any files for globstring: {}".format(p)) logger.error("Check your quotes!") error = True elif not os.path.exists(p): logger.error("Batch input file does not exist: " + p) error = True if error: raise RuntimeError( "Could not find one or more batch inputs. See logged errors.") # Instantiate 'shell' shell = HeadlessShell( functools.partial(PixelClassificationWorkflow, appendBatchOperators=True)) if args.assume_old_ilp_axes: # Special hack for Janelia: # In some old versions of 0.5, the data was stored in tyxzc order. # We have no way of inspecting the data to determine this, so we allow # users to specify that their ilp is very old using the # assume_old_ilp_axes command-line flag ilastik.utility.globals.ImportOptions.default_axis_order = 'tyxzc' # Load project (auto-import it if necessary) logger.info("Opening project: '" + args.project + "'") shell.openProjectFile(args.project) try: if not args.generate_project_predictions and len( args.batch_inputs) == 0: logger.error( "Command-line arguments didn't specify any classification jobs." ) else: # Predictions for project input datasets if args.generate_project_predictions: generateProjectPredictions(shell) # Predictions for other datasets ('batch datasets') result = True if len(args.batch_inputs) > 0: result = generateBatchPredictions( shell.workflow, args.batch_inputs, args.batch_export_dir, args.batch_output_suffix, args.batch_output_dataset_name, args.stack_volume_cache_dir) assert result finally: logger.info("Closing project...") shell.closeCurrentProject() logger.info("FINISHED.")