def testBasic(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') # Create a list of dataset infos . . . datasetInfos = [] # npy info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = self.testNpyFileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False datasetInfos.append(info) # png info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.filePath = self.testPngFileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False datasetInfos.append(info) reader.Dataset.setValues(datasetInfos) # Read the test files using the data selection operator and verify the contents npyData = reader.Image[0][...].wait() pngData = reader.Image[1][...].wait() # Check the file name output print reader.ImageName[0].value assert reader.ImageName[0].value == self.testNpyFileName assert reader.ImageName[1].value == self.testPngFileName # Check raw images assert npyData.shape == (10, 11, 1) for x in range(npyData.shape[0]): for y in range(npyData.shape[1]): assert npyData[x, y, 0] == x + y assert pngData.shape == (100, 200, 3) for x in range(pngData.shape[0]): for y in range(pngData.shape[1]): for c in range(pngData.shape[2]): assert pngData[x, y, c] == (x + y) % 256
def testBasic(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper( OpDataSelection, graph=graph ) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue( os.getcwd() ) reader.ProjectDataGroup.setValue( 'DataSelection/local_data' ) # Create a list of dataset infos . . . datasetInfos = [] # npy info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = self.testNpyFileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False datasetInfos.append(info) # png info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.filePath = self.testPngFileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False datasetInfos.append(info) reader.Dataset.setValues(datasetInfos) # Read the test files using the data selection operator and verify the contents npyData = reader.Image[0][...].wait() pngData = reader.Image[1][...].wait() # Check the file name output print reader.ImageName[0].value assert reader.ImageName[0].value == self.testNpyFileName assert reader.ImageName[1].value == self.testPngFileName # Check raw images assert npyData.shape == (10,11,1) for x in range(npyData.shape[0]): for y in range(npyData.shape[1]): assert npyData[x,y,0] == x+y assert pngData.shape == (100, 200, 3) for x in range(pngData.shape[0]): for y in range(pngData.shape[1]): for c in range(pngData.shape[2]): assert pngData[x,y,c] == (x+y) % 256
def test_fake_data_source(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = self.testRawDataFileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False # Use *fake* data source info.realDataSource = False info.axistags = vigra.defaultAxistags('tczyx') info.laneShape = self.imgData.shape info.laneDtype = self.imgData.dtype reader.Dataset.setValues([info]) # Verify that now data selection operator returns fake data # with expected shape and type imgData = reader.Image[0][...].wait() assert imgData.shape == self.imgData.shape assert imgData.dtype == self.imgData.dtype expected_fake_data = numpy.zeros(info.laneShape, dtype=info.laneDtype) numpy.testing.assert_array_equal(imgData, expected_fake_data)
def test_real_data_source(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = self.testRawDataFileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False #Use real data source info.realDataSource = True reader.Dataset.setValues([info]) # Read the test file using the data selection operator and verify the contents imgData = reader.Image[0][...].wait() assert imgData.shape == self.imgData.shape numpy.testing.assert_array_equal(imgData, self.imgData)
def test_load_single_file_with_list(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') fileNameString = os.path.pathsep.join(self.file_names) info = DatasetInfo(filepath=fileNameString) # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData = reader.Image[0][...].wait() print('imgData', reader.Image.meta.axistags, reader.Image.meta.original_axistags) # Check raw images assert imgData.shape == self.imgData3Dct.shape, (imgData.shape, self.imgData3Dct.shape) numpy.testing.assert_array_equal(imgData, self.imgData3Dct)
def testBasic3DcStackFromGlobString(self): """Test if stacked 2d 3-channel files are loaded correctly""" # For some reason vigra saves 2D+c data compressed in gifs, so skip! self.compressedExtensions.append('.gif') for fileName in self.imgFileNameGlobs2Dc: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData3Dc = reader.Image[0][...].wait() # Check the file name output assert reader.ImageName[0].value == fileName # Check raw images assert imgData3Dc.shape == self.imgData3Dc.shape, (imgData3Dc.shape, self.imgData3Dc.shape) # skip this if image was saved compressed: if any(x in fileName.lower() for x in self.compressedExtensions): print("Skipping raw comparison for compressed data: {}".format(fileName)) continue numpy.testing.assert_array_equal(imgData3Dc, self.imgData3Dc)
def testBasic3DstacksFromFileList(self): for ext, fileNames in list(self.imgFileLists2D.items()): fileNameString = os.path.pathsep.join(fileNames) graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo(filepath=fileNameString) # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData3D = reader.Image[0][...].wait() # Check raw images assert imgData3D.shape == self.imgData3D.shape, (imgData3D.shape, self.imgData3D.shape) # skip this if image was saved compressed: if any(x.strip('.') in ext.lower() for x in self.compressedExtensions): print("Skipping raw comparison for compressed data: {}".format(ext)) continue numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
def testBasic3DWrongAxes(self): """Test if 3D file with intentionally wrong axes is rejected """ for fileName in self.imgFileNames3D: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False info.axistags = vigra.defaultAxistags('tzyc') try: reader.Dataset.setValues([info]) assert False, "Should have thrown an exception!" except DatasetConstraintError: pass except: assert False, "Should have thrown a DatasetConstraintError!"
def testBasic3D(self): """Test if plane 2d files are loaded correctly""" for fileName in self.imgFileNames3D: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData3D = reader.Image[0][...].wait() # Check the file name output assert reader.ImageName[0].value == fileName # Check raw images assert imgData3D.shape == self.imgData3D.shape, (imgData3D.shape, self.imgData3D.shape) # skip this if image was saved compressed: numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
def testProjectLocalData(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper( OpDataSelection, graph=graph ) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue( os.getcwd() ) reader.ProjectDataGroup.setValue( 'DataSelection/local_data' ) # Create a list of dataset infos . . . datasetInfos = [] # From project info = DatasetInfo() info.location = DatasetInfo.Location.ProjectInternal info.filePath = "This string should be ignored..." info._datasetId = 'dataset1' # (Cheating a bit here...) info.invertColors = False info.convertToGrayscale = False datasetInfos.append(info) reader.Dataset.setValues(datasetInfos) projectInternalData = reader.Image[0][...].wait() assert projectInternalData.shape == self.pngData.shape assert (projectInternalData == self.pngData).all()
def testBasic2D(self): """Test if plane 2d files are loaded correctly""" for fileName in self.imgFileNames2D: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData2D = reader.Image[0][...].wait() # Check the file name output assert reader.ImageName[0].value == fileName # Check raw images assert imgData2D.shape == self.imgData2D.shape # skip this if image was saved compressed: if any(x in fileName.lower() for x in self.compressedExtensions): print("Skipping raw comparison for compressed data: {}".format(fileName)) continue numpy.testing.assert_array_equal(imgData2D, self.imgData2D)
def testBasic3DWrongAxes(self): """Test if 3D file with intentionally wrong axes is rejected """ for fileName in self.imgFileNames3D: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False info.axistags = vigra.defaultAxistags('tzyc') try: reader.Dataset.setValues([info]) assert False, "Should have thrown an exception!" except DatasetConstraintError: pass except: assert False, "Should have thrown a DatasetConstraintError!"
def test_load_single_file_with_list(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') fileNameString = os.path.pathsep.join(self.file_names) info = DatasetInfo(filepath=fileNameString) # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData = reader.Image[0][...].wait() print('imgData', reader.Image.meta.axistags, reader.Image.meta.original_axistags) # Check raw images assert imgData.shape == self.imgData3Dct.shape, ( imgData.shape, self.imgData3Dct.shape) numpy.testing.assert_array_equal(imgData, self.imgData3Dct)
def testProjectLocalData(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') # Create a list of dataset infos . . . datasetInfos = [] # From project info = DatasetInfo() info.location = DatasetInfo.Location.ProjectInternal info.filePath = "This string should be ignored..." info._datasetId = 'dataset1' # (Cheating a bit here...) info.invertColors = False info.convertToGrayscale = False datasetInfos.append(info) reader.Dataset.setValues(datasetInfos) projectInternalData = reader.Image[0][...].wait() assert projectInternalData.shape == self.imgData3Dc.shape assert (projectInternalData == self.imgData3Dc).all()
def testBasic3DstacksFromFileList(self): for ext, fileNames in self.imgFileLists2D.items(): fileNameString = os.path.pathsep.join(fileNames) graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo(filepath=fileNameString) # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData3D = reader.Image[0][...].wait() # Check raw images assert imgData3D.shape == self.imgData3D.shape # skip this if image was saved compressed: if any( x.strip('.') in ext.lower() for x in self.compressedExtensions): print("Skipping raw comparison for compressed data: {}".format( ext)) continue numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
def testBasic3DcStackFromGlobString(self): """Test if stacked 2d 3-channel files are loaded correctly""" # For some reason vigra saves 2D+c data compressed in gifs, so skip! self.compressedExtensions.append('.gif') for fileName in self.imgFileNameGlobs2Dc: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData3Dc = reader.Image[0][...].wait() # Check the file name output assert reader.ImageName[0].value == fileName # Check raw images assert imgData3Dc.shape == self.imgData3Dc.shape # skip this if image was saved compressed: if any(x in fileName.lower() for x in self.compressedExtensions): print("Skipping raw comparison for compressed data: {}".format( fileName)) continue numpy.testing.assert_array_equal(imgData3Dc, self.imgData3Dc)
def testBasic2D(self): """Test if plane 2d files are loaded correctly""" for fileName in self.imgFileNames2D: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph, operator_kwargs={'forceAxisOrder': False}) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData2D = reader.Image[0][...].wait() # Check the file name output assert reader.ImageName[0].value == fileName # Check raw images assert imgData2D.shape == self.imgData2D.shape # skip this if image was saved compressed: if any(x in fileName.lower() for x in self.compressedExtensions): print("Skipping raw comparison for compressed data: {}".format( fileName)) continue numpy.testing.assert_array_equal(imgData2D, self.imgData2D)
def testBasic3D(self): """Test if plane 2d files are loaded correctly""" for fileName in self.imgFileNames3D: graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo() # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.filePath = fileName info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData3D = reader.Image[0][...].wait() # Check the file name output assert reader.ImageName[0].value == fileName # Check raw images assert imgData3D.shape == self.imgData3D.shape # skip this if image was saved compressed: numpy.testing.assert_array_equal(imgData3D, self.imgData3D)
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName): """ Compute the predictions for each of the specified batch input files, and export them to corresponding h5 files. """ batchInputPaths = convertStacksToH5(batchInputPaths) batchInputInfos = [] for p in batchInputPaths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() batchInputInfos.append(info) # Configure batch input operator opBatchInputs = workflow.batchInputApplet.topLevelOperator opBatchInputs.Dataset.setValues( batchInputInfos ) # Configure batch export operator opBatchResults = workflow.batchResultsApplet.topLevelOperator opBatchResults.ExportDirectory.setValue(batchExportDir) opBatchResults.Format.setValue(ExportFormat.H5) opBatchResults.Suffix.setValue(batchOutputSuffix) opBatchResults.InternalPath.setValue(exportedDatasetName) opBatchResults.SelectedSlices.setValue([30]) logger.info( "Exporting data to " + opBatchResults.OutputDataPath[0].value ) # Set up progress display handling (just logging for now) currentProgress = [None] def handleProgress(percentComplete): if currentProgress[0] != percentComplete: currentProgress[0] = percentComplete logger.info("Batch job: {}% complete.".format(percentComplete)) progressSignal = opBatchResults.ProgressSignal[0].value progressSignal.subscribe( handleProgress ) # Make it happen! result = opBatchResults.ExportResult[0].value return result
def _run_export_with_empty_batch_lane(self, role_input_paths, batch_lane_index, template_infos, progress_callback): """ Configure the fresh batch lane with the given input files, and export the results. """ assert role_input_paths[0], "At least one file must be provided for each dataset (the first role)." opDataSelectionBatchLaneView = self.dataSelectionApplet.topLevelOperator.getLane( batch_lane_index ) # Apply new settings for each role for role_index, path_for_role in enumerate(role_input_paths): if not path_for_role: continue if template_infos[role_index]: info = copy.copy(template_infos[role_index]) else: info = DatasetInfo() # Override the template settings with the current filepath. default_info = DataSelectionApplet.create_default_headless_dataset_info(path_for_role) info.filePath = default_info.filePath info.location = default_info.location info.nickname = default_info.nickname # Apply to the data selection operator opDataSelectionBatchLaneView.DatasetGroup[role_index].setValue(info) # Make sure nothing went wrong opDataExportBatchlaneView = self.dataExportApplet.topLevelOperator.getLane( batch_lane_index ) assert opDataExportBatchlaneView.ImageToExport.ready() assert opDataExportBatchlaneView.ExportPath.ready() # New lanes were added. # Give the workflow a chance to restore anything that was unecessarily invalidated (e.g. classifiers) self.workflow.handleNewLanesAdded() # Call customization hook self.dataExportApplet.prepare_lane_for_export(batch_lane_index) # Finally, run the export logger.info("Exporting to {}".format( opDataExportBatchlaneView.ExportPath.value )) opDataExportBatchlaneView.progressSignal.subscribe(progress_callback) opDataExportBatchlaneView.run_export() # Call customization hook self.dataExportApplet.post_process_lane_export(batch_lane_index)
def append_lane(workflow, input_filepath, axisorder=None): # Sanity checks assert isinstance(workflow, PixelClassificationWorkflow) opPixelClassification = workflow.pcApplet.topLevelOperator assert opPixelClassification.Classifier.ready() # If the filepath is a globstring, convert the stack to h5 input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], TMP_DIR )[0] info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.filePath = input_filepath comp = PathComponents(input_filepath) # Convert all (non-url) paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) if not isUrl(input_filepath): comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() info.nickname = comp.filenameBase if axisorder: info.axistags = vigra.defaultAxistags(axisorder) logger.debug( "adding lane: {}".format( info ) ) opDataSelection = workflow.dataSelectionApplet.topLevelOperator # Add a lane num_lanes = len( opDataSelection.DatasetGroup )+1 logger.debug( "num_lanes: {}".format( num_lanes ) ) opDataSelection.DatasetGroup.resize( num_lanes ) # Configure it. role_index = 0 # raw data opDataSelection.DatasetGroup[-1][role_index].setValue( info ) # Sanity check assert len(opPixelClassification.InputImages) == num_lanes return opPixelClassification
def append_lane(workflow, input_filepath, axisorder=None): """ Add a lane to the project file for the given input file. If axisorder is given, override the default axisorder for the file and force the project to use the given one. Globstrings are supported, in which case the files are converted to HDF5 first. """ # If the filepath is a globstring, convert the stack to h5 input_filepath = DataSelectionApplet.convertStacksToH5( [input_filepath], tempfile.mkdtemp() )[0] info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem info.filePath = input_filepath comp = PathComponents(input_filepath) # Convert all (non-url) paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) if not isUrl(input_filepath): comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() info.nickname = comp.filenameBase if axisorder: info.axistags = vigra.defaultAxistags(axisorder) logger.debug( "adding lane: {}".format( info ) ) opDataSelection = workflow.dataSelectionApplet.topLevelOperator # Add a lane num_lanes = len( opDataSelection.DatasetGroup )+1 logger.debug( "num_lanes: {}".format( num_lanes ) ) opDataSelection.DatasetGroup.resize( num_lanes ) # Configure it. role_index = 0 # raw data opDataSelection.DatasetGroup[-1][role_index].setValue( info )
def test_load_single_file_with_glob(self): graph = lazyflow.graph.Graph() reader = OperatorWrapper(OpDataSelection, graph=graph) reader.ProjectFile.setValue(self.projectFile) reader.WorkingDirectory.setValue(os.getcwd()) reader.ProjectDataGroup.setValue('DataSelection/local_data') info = DatasetInfo(filepath=self.glob_string) # Will be read from the filesystem since the data won't be found in the project file. info.location = DatasetInfo.Location.ProjectInternal info.internalPath = "" info.invertColors = False info.convertToGrayscale = False reader.Dataset.setValues([info]) # Read the test files using the data selection operator and verify the contents imgData = reader.Image[0][...].wait() # Check raw images assert imgData.shape == self.imgData3Dct.shape numpy.testing.assert_array_equal(imgData, self.imgData3Dct)
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName, stackVolumeCacheDir): """ Compute the predictions for each of the specified batch input files, and export them to corresponding h5 files. """ originalBatchInputPaths = list(batchInputPaths) batchInputPaths = convertStacksToH5(batchInputPaths, stackVolumeCacheDir) batchInputInfos = [] for p in batchInputPaths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() batchInputInfos.append(info) # Also convert the export dir to absolute (for the same reason) if batchExportDir != '': batchExportDir = os.path.abspath( batchExportDir ) # Configure batch input operator opBatchInputs = workflow.batchInputApplet.topLevelOperator opBatchInputs.Dataset.setValues( batchInputInfos ) # Configure batch export operator opBatchResults = workflow.batchResultsApplet.topLevelOperator # By default, the output files from the batch export operator # are named using the input file name. # If we converted any stacks to hdf5, then the user won't recognize the input file name. # Let's override the output file name using the *original* input file names. outputFileNameBases = [] for origPath in originalBatchInputPaths: outputFileNameBases.append( origPath.replace('*', 'STACKED') ) opBatchResults.OutputFileNameBase.setValues( outputFileNameBases ) opBatchResults.ExportDirectory.setValue(batchExportDir) opBatchResults.Format.setValue(ExportFormat.H5) opBatchResults.Suffix.setValue(batchOutputSuffix) opBatchResults.InternalPath.setValue(exportedDatasetName) logger.info( "Exporting data to " + opBatchResults.OutputDataPath[0].value ) # Set up progress display handling (just logging for now) currentProgress = [None] def handleProgress(percentComplete): if currentProgress[0] != percentComplete: currentProgress[0] = percentComplete logger.info("Batch job: {}% complete.".format(percentComplete)) progressSignal = opBatchResults.ProgressSignal[0].value progressSignal.subscribe( handleProgress ) # Make it happen! result = opBatchResults.ExportResult[0].value return result
def _createDatasetInfo(self): info = DatasetInfo() info.filePath = self.tmpFilePath info.location = DatasetInfo.Location.ProjectInternal return info
def generateBatchPredictions(workflow, batchInputPaths, batchExportDir, batchOutputSuffix, exportedDatasetName, stackVolumeCacheDir): """ Compute the predictions for each of the specified batch input files, and export them to corresponding h5 files. """ originalBatchInputPaths = list(batchInputPaths) batchInputPaths = convertStacksToH5(batchInputPaths, stackVolumeCacheDir) batchInputInfos = [] for p in batchInputPaths: info = DatasetInfo() info.location = DatasetInfo.Location.FileSystem # Convert all paths to absolute # (otherwise they are relative to the project file, which probably isn't what the user meant) comp = PathComponents(p) comp.externalPath = os.path.abspath(comp.externalPath) info.filePath = comp.totalPath() batchInputInfos.append(info) # Also convert the export dir to absolute (for the same reason) if batchExportDir != '': batchExportDir = os.path.abspath(batchExportDir) # Configure batch input operator opBatchInputs = workflow.batchInputApplet.topLevelOperator opBatchInputs.DatasetGroup.resize(len(batchInputInfos)) for info, multislot in zip(batchInputInfos, opBatchInputs.DatasetGroup): # FIXME: This assumes that the workflow has exactly one dataset role. multislot[0].setValue(info) # Configure batch export operator opBatchResults = workflow.batchResultsApplet.topLevelOperator # By default, the output files from the batch export operator # are named using the input file name. # If we converted any stacks to hdf5, then the user won't recognize the input file name. # Let's override the output file name using the *original* input file names. outputFileNameBases = [] for origPath in originalBatchInputPaths: outputFileNameBases.append(origPath.replace('*', 'STACKED')) opBatchResults.OutputFileNameBase.setValues(outputFileNameBases) opBatchResults.ExportDirectory.setValue(batchExportDir) opBatchResults.Format.setValue(ExportFormat.H5) opBatchResults.Suffix.setValue(batchOutputSuffix) opBatchResults.InternalPath.setValue(exportedDatasetName) logger.info("Exporting data to " + opBatchResults.OutputDataPath[0].value) # Set up progress display handling (just logging for now) currentProgress = [None] def handleProgress(percentComplete): if currentProgress[0] != percentComplete: currentProgress[0] = percentComplete logger.info("Batch job: {}% complete.".format(percentComplete)) progressSignal = opBatchResults.ProgressSignal[0].value progressSignal.subscribe(handleProgress) # Make it happen! result = opBatchResults.ExportResult[0].value return result
def test06(self): """ Test the basic functionality of the v0.6 project format serializer. """ # Create an empty project with h5py.File(self.testProjectName) as testProject: testProject.create_dataset("ilastikVersion", data="1.0.0") ## ## Serialization ## # Create an operator to work with and give it some input graph = Graph() operatorToSave = OpMultiLaneDataSelectionGroup( graph=graph ) serializer = DataSelectionSerializer(operatorToSave, 'DataSelectionTest') assert serializer.base_initialized operatorToSave.ProjectFile.setValue(testProject) operatorToSave.WorkingDirectory.setValue( os.path.split(__file__)[0] ) operatorToSave.ProjectDataGroup.setValue( serializer.topGroupName + '/local_data' ) info = DatasetInfo() info.filePath = self.tmpFilePath info.location = DatasetInfo.Location.ProjectInternal operatorToSave.DatasetRoles.setValue( ['Raw Data'] ) operatorToSave.DatasetGroup.resize(1) operatorToSave.DatasetGroup[0][0].setValue(info) # Now serialize! serializer.serializeToHdf5(testProject, self.testProjectName) # Check for dataset existence datasetInternalPath = serializer.topGroupName + '/local_data/' + info.datasetId dataset = testProject[datasetInternalPath][...] # Check axistags attribute assert 'axistags' in testProject[datasetInternalPath].attrs axistags_json = testProject[datasetInternalPath].attrs['axistags'] axistags = vigra.AxisTags.fromJSON(axistags_json) # Debug info... #logging.basicConfig(level=logging.DEBUG) logger.debug('dataset.shape = ' + str(dataset.shape)) logger.debug('should be ' + str(operatorToSave.Image[0].meta.shape)) logger.debug('dataset axistags:') logger.debug(axistags) logger.debug('should be:') logger.debug(operatorToSave.Image[0].meta.axistags) originalShape = operatorToSave.Image[0].meta.shape originalAxisTags = operatorToSave.Image[0].meta.axistags # Now we can directly compare the shape and axis ordering assert dataset.shape == originalShape assert axistags == originalAxisTags ## ## Deserialization ## # Create an empty operator graph = Graph() operatorToLoad = OpMultiLaneDataSelectionGroup( graph=graph ) operatorToLoad.DatasetRoles.setValue( ['Raw Data'] ) deserializer = DataSelectionSerializer(operatorToLoad, serializer.topGroupName) # Copy the group name from the serializer we used. assert deserializer.base_initialized deserializer.deserializeFromHdf5(testProject, self.testProjectName) assert len(operatorToLoad.DatasetGroup) == len(operatorToSave.DatasetGroup) assert len(operatorToLoad.Image) == len(operatorToSave.Image) assert operatorToLoad.Image[0].meta.shape == operatorToSave.Image[0].meta.shape assert operatorToLoad.Image[0].meta.axistags == operatorToSave.Image[0].meta.axistags os.remove(self.testProjectName)
def test06(self): """ Test the basic functionality of the v0.6 project format serializer. """ # Create an empty project with h5py.File(self.testProjectName) as testProject: testProject.create_dataset("ilastikVersion", data=0.6) ## ## Serialization ## # Create an operator to work with and give it some input graph = Graph() operatorToSave = OpMultiLaneDataSelectionGroup(graph=graph) serializer = DataSelectionSerializer(operatorToSave, 'DataSelectionTest') assert serializer.base_initialized operatorToSave.ProjectFile.setValue(testProject) operatorToSave.WorkingDirectory.setValue( os.path.split(__file__)[0]) operatorToSave.ProjectDataGroup.setValue(serializer.topGroupName + '/local_data') info = DatasetInfo() info.filePath = self.tmpFilePath info.location = DatasetInfo.Location.ProjectInternal operatorToSave.DatasetRoles.setValue(['Raw Data']) operatorToSave.DatasetGroup.resize(1) operatorToSave.DatasetGroup[0][0].setValue(info) # Now serialize! serializer.serializeToHdf5(testProject, self.testProjectName) # Check for dataset existence datasetInternalPath = serializer.topGroupName + '/local_data/' + info.datasetId dataset = testProject[datasetInternalPath][...] # Check axistags attribute assert 'axistags' in testProject[datasetInternalPath].attrs axistags_json = testProject[datasetInternalPath].attrs['axistags'] axistags = vigra.AxisTags.fromJSON(axistags_json) # Debug info... #logging.basicConfig(level=logging.DEBUG) logger.debug('dataset.shape = ' + str(dataset.shape)) logger.debug('should be ' + str(operatorToSave.Image[0].meta.shape)) logger.debug('dataset axistags:') logger.debug(axistags) logger.debug('should be:') logger.debug(operatorToSave.Image[0].meta.axistags) originalShape = operatorToSave.Image[0].meta.shape originalAxisTags = operatorToSave.Image[0].meta.axistags # Now we can directly compare the shape and axis ordering assert dataset.shape == originalShape assert axistags == originalAxisTags ## ## Deserialization ## # Create an empty operator graph = Graph() operatorToLoad = OpMultiLaneDataSelectionGroup(graph=graph) operatorToLoad.DatasetRoles.setValue(['Raw Data']) deserializer = DataSelectionSerializer( operatorToLoad, serializer.topGroupName ) # Copy the group name from the serializer we used. assert deserializer.base_initialized deserializer.deserializeFromHdf5(testProject, self.testProjectName) assert len(operatorToLoad.DatasetGroup) == len( operatorToSave.DatasetGroup) assert len(operatorToLoad.Image) == len(operatorToSave.Image) assert operatorToLoad.Image[0].meta.shape == operatorToSave.Image[ 0].meta.shape assert operatorToLoad.Image[ 0].meta.axistags == operatorToSave.Image[0].meta.axistags os.remove(self.testProjectName)