Exemple #1
0
    def testBasic_2d(self):
        data = 255 * numpy.random.random((50, 100))
        data = data.astype(numpy.uint8)
        data = vigra.taggedView(data, vigra.defaultAxistags("yx"))

        graph = Graph()

        opPiper = OpArrayPiper(graph=graph)
        opPiper.Input.setValue(data)

        opExport = OpExportSlot(graph=graph)
        opExport.Input.connect(opPiper.Output)
        opExport.OutputFormat.setValue("png")
        opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
        opExport.CoordinateOffset.setValue((10, 20))

        assert opExport.ExportPath.ready()
        assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x20-120_y10-60.png"
        opExport.run_export()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(opExport.ExportPath.value)
            expected_data = data.view(numpy.ndarray)
            read_data = opRead.Output[:].wait()

            # Note: vigra inserts a channel axis, so read_data is xyc
            assert (read_data[..., 0] == expected_data).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
    def test_via_OpExportSlot(self):
        data = 255 * numpy.random.random( (64, 128, 128, 1) )
        data = data.astype( numpy.uint8 )
        data = vigra.taggedView( data, vigra.defaultAxistags('zyxc') )
         
        graph = Graph()
         
        opPiper = OpArrayPiper(graph=graph)
        opPiper.Input.setValue( data )
         
        opExport = OpExportSlot(graph=graph)
        opExport.Input.connect( opPiper.Output )
        opExport.OutputFormat.setValue( 'dvid' )
        url = 'http://{hostname}/api/node/{data_uuid}/{data_name}'.format( **self.__dict__ )
        opExport.OutputFilenameFormat.setValue( url )
         
        assert opExport.ExportPath.ready()
        assert opExport.ExportPath.value == url
 
        opExport.run_export()
 
        opRead = OpInputDataReader( graph=graph )
        try:
            opRead.FilePath.setValue( opExport.ExportPath.value )
            expected_data = data.view(numpy.ndarray)
            read_data = opRead.Output( *roiFromShape(data.shape) ).wait()
            assert (read_data == expected_data).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
    def testBasic_Hdf5(self):
        data = numpy.random.random((100, 100)).astype(numpy.float32)
        data = vigra.taggedView(data, vigra.defaultAxistags('xy'))

        graph = Graph()
        opPiper = OpArrayPiper(graph=graph)
        opPiper.Input.setValue(data)

        opExport = OpExportSlot(graph=graph)
        opExport.Input.connect(opPiper.Output)
        opExport.OutputFormat.setValue('hdf5')
        opExport.OutputFilenameFormat.setValue(
            self._tmpdir +
            '/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
        opExport.OutputInternalPath.setValue('volume/data')
        opExport.CoordinateOffset.setValue((10, 20))

        assert opExport.ExportPath.ready()
        export_file = PathComponents(opExport.ExportPath.value).externalPath
        assert os.path.split(
            export_file)[1] == 'test_export_x10-110_y20-120.h5'
        #print "exporting data to: {}".format( opExport.ExportPath.value )
        opExport.run_export()

        opRead = OpInputDataReader(graph=graph)
        opRead.FilePath.setValue(opExport.ExportPath.value)
        expected_data = data.view(numpy.ndarray)
        read_data = opRead.Output[:].wait()
        assert (read_data == expected_data
                ).all(), "Read data didn't match exported data!"

        opRead.cleanUp()
 def testBasic_Hdf5(self):
     data = numpy.random.random( (100,100) ).astype( numpy.float32 )
     data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
     
     graph = Graph()
     opExport = OpExportSlot(graph=graph)
     opExport.Input.setValue(data)
     opExport.OutputFormat.setValue( 'hdf5' )
     opExport.OutputFilenameFormat.setValue( self._tmpdir + '/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
     opExport.OutputInternalPath.setValue('volume/data')
     opExport.CoordinateOffset.setValue( (10, 20) )
     
     assert opExport.ExportPath.ready()
     export_file = PathComponents( opExport.ExportPath.value ).externalPath
     assert os.path.split(export_file)[1] == 'test_export_x10-110_y20-120.h5'
     #print "exporting data to: {}".format( opExport.ExportPath.value )
     opExport.run_export()
     
     opRead = OpInputDataReader( graph=graph )
     opRead.FilePath.setValue( opExport.ExportPath.value )
     expected_data = data.view(numpy.ndarray)
     read_data = opRead.Output[:].wait()
     assert (read_data == expected_data).all(), "Read data didn't match exported data!"
     
     opRead.cleanUp()
    def test_h5_stack_multi_file(self, sequence_axis):
        """Test stack/sequence reading in hdf5-files"""
        shape = (4, 8, 16, 32, 3)
        data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
        for index, t_slice in enumerate(data):
            fname = self.testmultiH5FileName.format(index=index)
            with h5py.File(fname) as f:
                data_group = f.create_group("volume")
                data_group.create_dataset("data", data=t_slice)

        if sequence_axis == "z":
            data = numpy.concatenate(data, axis=0)
        elif sequence_axis == "c":
            data = numpy.concatenate(data, axis=-1)

        h5SequenceReader = OpInputDataReader(graph=self.graph)
        h5SequenceReader.SequenceAxis.setValue(sequence_axis)
        globString = self.testmultiH5FileName.replace("02d}", "s}").format(index="*")
        filenamePlusGlob = "{}/volume/data".format(globString)
        try:
            h5SequenceReader.FilePath.setValue(filenamePlusGlob)
            h5data = h5SequenceReader.Output[:].wait()
            assert h5data.shape == data.shape
            numpy.testing.assert_array_equal(h5data, data)
        finally:
            # Call cleanUp() to close the file that this operator opened
            h5SequenceReader.cleanUp()
    def test_npz(self):
        # Create two Numpy test arrays
        a = numpy.zeros((10, 11))
        for x in range(0, 10):
            for y in range(0, 11):
                a[x, y] = x + y

        b = numpy.arange((3 * 9)).reshape((3, 9))
        numpy.savez(self.testNpzDataFileName, a=a, b=b)

        # Now read back our test data using an OpInputDataReader operator
        npyReader = OpInputDataReader(graph=self.graph)

        try:
            for internalPath, referenceArray in zip(["a", "b"], [a, b]):
                npyReader.FilePath.setValue("{}/{}".format(
                    self.testNpzDataFileName, internalPath))
                cwd = os.path.split(__file__)[0]
                npyReader.WorkingDirectory.setValue(cwd)

                npzData = npyReader.Output[:].wait()
                assert npzData.shape == referenceArray.shape
                numpy.testing.assert_array_equal(npzData, referenceArray)
        finally:
            npyReader.cleanUp()
    def test_h5(self):
        # Create HDF5 test data
        with h5py.File(self.testH5FileName) as f:
            f.create_group("volume")
            shape = (1, 2, 3, 4, 5)
            f["volume"].create_dataset(
                "data", data=numpy.indices(shape).sum(0).astype(numpy.float32), chunks=True, compression="gzip"
            )

        # Read the entire HDF5 file and verify the contents
        h5Reader = OpInputDataReader(graph=self.graph)
        try:
            h5Reader.FilePath.setValue(self.testH5FileName + "/volume/data")  # Append internal path
            cwd = os.path.split(__file__)[0]
            h5Reader.WorkingDirectory.setValue(cwd)

            # Grab a section of the h5 data
            h5Data = h5Reader.Output[0, 0, :, :, :].wait()
            assert h5Data.shape == (1, 1, 3, 4, 5)
            # (Just check part of the data)
            for k in range(0, shape[2]):
                for l in range(0, shape[3]):
                    for m in range(0, shape[4]):
                        assert h5Data[0, 0, k, l, m] == k + l + m

        finally:
            # Call cleanUp() to close the file that this operator opened
            h5Reader.cleanUp()
            assert not h5Reader._file  # Whitebox assertion...
Exemple #8
0
    def testBasic_Npy(self):
        data = numpy.random.random((100, 100)).astype(numpy.float32)
        data = vigra.taggedView(data, vigra.defaultAxistags("xy"))

        graph = Graph()
        opPiper = OpArrayPiper(graph=graph)
        opPiper.Input.setValue(data)

        opExport = OpExportSlot(graph=graph)
        opExport.Input.connect(opPiper.Output)
        opExport.OutputFormat.setValue("numpy")
        opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
        opExport.CoordinateOffset.setValue((10, 20))

        assert opExport.ExportPath.ready()
        assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x10-110_y20-120.npy"
        # print "exporting data to: {}".format( opExport.ExportPath.value )
        opExport.run_export()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(opExport.ExportPath.value)
            expected_data = data.view(numpy.ndarray)
            read_data = opRead.Output[:].wait()
            assert (read_data == expected_data).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
Exemple #9
0
    def test_h5(self):
        # Create HDF5 test data
        with h5py.File(self.testH5FileName, "w") as f:
            f.create_group("volume")
            shape = (1, 2, 3, 4, 5)
            f["volume"].create_dataset(
                "data", data=numpy.indices(shape).sum(0).astype(numpy.float32), chunks=True, compression="gzip"
            )

        # Read the entire HDF5 file and verify the contents
        h5Reader = OpInputDataReader(graph=self.graph)
        try:
            h5Reader.FilePath.setValue(self.testH5FileName + "/volume/data")  # Append internal path
            cwd = os.path.split(__file__)[0]
            h5Reader.WorkingDirectory.setValue(cwd)

            # Grab a section of the h5 data
            h5Data = h5Reader.Output[0, 0, :, :, :].wait()
            assert h5Data.shape == (1, 1, 3, 4, 5)
            # (Just check part of the data)
            for k in range(0, shape[2]):
                for l in range(0, shape[3]):
                    for m in range(0, shape[4]):
                        assert h5Data[0, 0, k, l, m] == k + l + m

        finally:
            # Call cleanUp() to close the file that this operator opened
            h5Reader.cleanUp()
            assert not h5Reader._file  # Whitebox assertion...
Exemple #10
0
    def test_basic(self):
        opData = OpArrayCache(graph=self.graph)
        opData.blockShape.setValue(self.testData.shape)
        opData.Input.setValue(self.testData)

        filepath = os.path.join(self._tmpdir, 'multipage.tiff')
        logger.debug("writing to: {}".format(filepath))

        opExport = OpExportMultipageTiff(graph=self.graph)
        opExport.Filepath.setValue(filepath)
        opExport.Input.connect(opData.Output)

        # Run the export
        opExport.run_export()

        opReader = OpInputDataReader(graph=self.graph)
        opReader.FilePath.setValue(filepath)

        # The reader assumes xyzc order.
        # We have to transpose the data before we compare.
        opReorderAxes = OpReorderAxes(graph=self.graph)
        opReorderAxes.AxisOrder.setValue(self._axisorder)
        opReorderAxes.Input.connect(opReader.Output)

        readData = opReorderAxes.Output[:].wait()
        logger.debug("Expected shape={}".format(self.testData.shape))
        logger.debug("Read shape={}".format(readData.shape))

        assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
        assert (opReorderAxes.Output[:].wait() == self.testData.view(
            numpy.ndarray)).all(), "Exported data was not correct"

        # Cleanup
        opReorderAxes.cleanUp()
        opReader.cleanUp()
Exemple #11
0
    def test_h5_stack_single_file(self, sequence_axis):
        """Test stack/sequence reading in hdf5-files for given 'sequence_axis'"""
        shape = (4, 8, 16, 32, 3)  # assuming axis guess order is 'tzyxc'
        data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
        with h5py.File(self.testH5FileName, "w") as f:
            data_group = f.create_group("volumes")
            for index, t_slice in enumerate(data):
                data_group.create_dataset("timepoint-{index:02d}".format(index=index), data=t_slice)

        if sequence_axis == "z":
            data = numpy.concatenate(data, axis=0)
        elif sequence_axis == "c":
            data = numpy.concatenate(data, axis=-1)

        h5SequenceReader = OpInputDataReader(graph=self.graph)
        h5SequenceReader.SequenceAxis.setValue(sequence_axis)
        filenamePlusGlob = "{}/volumes/timepoint-*".format(self.testH5FileName)
        try:
            h5SequenceReader.FilePath.setValue(filenamePlusGlob)

            h5data = h5SequenceReader.Output[:].wait()
            assert h5data.shape == data.shape, f"{h5data.shape}, {data.shape}"
            numpy.testing.assert_array_equal(h5data, data)
        finally:
            # Call cleanUp() to close the file that this operator opened
            h5SequenceReader.cleanUp()
Exemple #12
0
    def test_h5_stack_multi_file(self, sequence_axis):
        """Test stack/sequence reading in hdf5-files"""
        shape = (4, 8, 16, 32, 3)
        data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
        for index, t_slice in enumerate(data):
            fname = self.testmultiH5FileName.format(index=index)
            with h5py.File(fname, "w") as f:
                data_group = f.create_group("volume")
                data_group.create_dataset("data", data=t_slice)

        if sequence_axis == "z":
            data = numpy.concatenate(data, axis=0)
        elif sequence_axis == "c":
            data = numpy.concatenate(data, axis=-1)

        h5SequenceReader = OpInputDataReader(graph=self.graph)
        h5SequenceReader.SequenceAxis.setValue(sequence_axis)
        globString = self.testmultiH5FileName.replace("02d}", "s}").format(index="*")
        filenamePlusGlob = "{}/volume/data".format(globString)
        try:
            h5SequenceReader.FilePath.setValue(filenamePlusGlob)
            h5data = h5SequenceReader.Output[:].wait()
            assert h5data.shape == data.shape
            numpy.testing.assert_array_equal(h5data, data)
        finally:
            # Call cleanUp() to close the file that this operator opened
            h5SequenceReader.cleanUp()
            def handleImportLabelsAction():
                # Find the directory of the most recently opened image file
                mostRecentImageFile = PreferencesManager().get(
                    'DataSelection', 'recent image')
                if mostRecentImageFile is not None:
                    defaultDirectory = os.path.split(mostRecentImageFile)[0]
                else:
                    defaultDirectory = os.path.expanduser('~')
                fileNames = DataSelectionGui.getImageFileNamesToOpen(
                    self, defaultDirectory)
                fileNames = map(str, fileNames)

                # For now, we require a single hdf5 file
                if len(fileNames) > 1:
                    QMessageBox.critical(
                        self, "Too many files",
                        "Labels must be contained in a single hdf5 volume.")
                    return
                if len(fileNames) == 0:
                    # user cancelled
                    return

                file_path = fileNames[0]
                internal_paths = DataSelectionGui.getPossibleInternalPaths(
                    file_path)
                if len(internal_paths) == 0:
                    QMessageBox.critical(
                        self, "No volumes in file",
                        "Couldn't find a suitable dataset in your hdf5 file.")
                    return
                if len(internal_paths) == 1:
                    internal_path = internal_paths[0]
                else:
                    dlg = H5VolumeSelectionDlg(internal_paths, self)
                    if dlg.exec_() == QDialog.Rejected:
                        return
                    selected_index = dlg.combo.currentIndex()
                    internal_path = str(internal_paths[selected_index])

                path_components = PathComponents(file_path)
                path_components.internalPath = str(internal_path)

                try:
                    top_op = self.topLevelOperatorView
                    opReader = OpInputDataReader(parent=top_op.parent)
                    opReader.FilePath.setValue(path_components.totalPath())

                    # Reorder the axes
                    op5 = OpReorderAxes(parent=top_op.parent)
                    op5.AxisOrder.setValue(
                        top_op.LabelInputs.meta.getAxisKeys())
                    op5.Input.connect(opReader.Output)

                    # Finally, import the labels
                    top_op.importLabels(top_op.current_view_index(),
                                        op5.Output)

                finally:
                    op5.cleanUp()
                    opReader.cleanUp()
Exemple #14
0
    def testBasic(self):
        data = numpy.random.random((100, 100)).astype(numpy.float32)
        data = vigra.taggedView(data, vigra.defaultAxistags('xy'))

        graph = Graph()

        opPiper = OpArrayPiper(graph=graph)
        opPiper.Input.setValue(data)

        opWriter = OpNpyWriter(graph=graph)
        opWriter.Input.connect(opPiper.Output)
        opWriter.Filepath.setValue(self._tmpdir +
                                   '/npy_writer_test_output.npy')

        # Write it...
        opWriter.write()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(opWriter.Filepath.value)
            expected_data = data.view(numpy.ndarray)
            read_data = opRead.Output[:].wait()
            assert (read_data == expected_data
                    ).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
    def test_h5_stack_single_file(self, sequence_axis):
        """Test stack/sequence reading in hdf5-files for given 'sequence_axis'"""
        shape = (4, 8, 16, 32, 3)  # assuming axis guess order is 'tzyxc'
        data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
        with h5py.File(self.testH5FileName) as f:
            data_group = f.create_group("volumes")
            for index, t_slice in enumerate(data):
                data_group.create_dataset("timepoint-{index:02d}".format(index=index), data=t_slice)

        if sequence_axis == "z":
            data = numpy.concatenate(data, axis=0)
        elif sequence_axis == "c":
            data = numpy.concatenate(data, axis=-1)

        h5SequenceReader = OpInputDataReader(graph=self.graph)
        h5SequenceReader.SequenceAxis.setValue(sequence_axis)
        filenamePlusGlob = "{}/volumes/timepoint-*".format(self.testH5FileName)
        try:
            h5SequenceReader.FilePath.setValue(filenamePlusGlob)

            h5data = h5SequenceReader.Output[:].wait()
            assert h5data.shape == data.shape, f"{h5data.shape}, {data.shape}"
            numpy.testing.assert_array_equal(h5data, data)
        finally:
            # Call cleanUp() to close the file that this operator opened
            h5SequenceReader.cleanUp()
    def test_via_OpExportSlot(self):
        data = 255 * numpy.random.random((64, 128, 128, 1))
        data = data.astype(numpy.uint8)
        data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))

        graph = Graph()

        opPiper = OpArrayPiper(graph=graph)
        opPiper.Input.setValue(data)

        opExport = OpExportSlot(graph=graph)
        opExport.Input.connect(opPiper.Output)
        opExport.OutputFormat.setValue("dvid")
        url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format(
            **self.__dict__)
        opExport.OutputFilenameFormat.setValue(url)

        assert opExport.ExportPath.ready()
        assert opExport.ExportPath.value == url

        opExport.run_export()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(opExport.ExportPath.value)
            expected_data = data.view(numpy.ndarray)
            read_data = opRead.Output(*roiFromShape(data.shape)).wait()
            assert (read_data == expected_data
                    ).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
    def test_basic(self):
        opData = OpArrayCache( graph=self.graph )
        opData.blockShape.setValue( self.testData.shape )
        opData.Input.setValue( self.testData )
        
        filepath = os.path.join( self._tmpdir, 'multipage.tiff' )
        logger.debug( "writing to: {}".format(filepath) )
        
        opExport = OpExportMultipageTiff(graph=self.graph)
        opExport.Filepath.setValue( filepath )
        opExport.Input.connect( opData.Output )

        # Run the export
        opExport.run_export()

        opReader = OpInputDataReader( graph=self.graph )
        opReader.FilePath.setValue( filepath )

        # The reader assumes xyzc order.
        # We have to transpose the data before we compare.
        opReorderAxes = OpReorderAxes( graph=self.graph )
        opReorderAxes.AxisOrder.setValue( self._axisorder )
        opReorderAxes.Input.connect( opReader.Output )
        
        readData = opReorderAxes.Output[:].wait()
        logger.debug("Expected shape={}".format( self.testData.shape ) )
        logger.debug("Read shape={}".format( readData.shape ) )
        
        assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
        assert (opReorderAxes.Output[:].wait() == self.testData.view( numpy.ndarray )).all(), "Exported data was not correct"
        
        # Cleanup
        opReorderAxes.cleanUp()
        opReader.cleanUp()
    def testBasic(self):
        graph = Graph()
        opExport = OpFormattedDataExport(graph=graph)

        data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
        data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
        opExport.Input.setValue(data)

        sub_roi = [(10, 0), (None, 80)]
        opExport.RegionStart.setValue(sub_roi[0])
        opExport.RegionStop.setValue(sub_roi[1])

        opExport.ExportDtype.setValue(numpy.uint8)

        opExport.InputMin.setValue(0.0)
        opExport.InputMax.setValue(100.0)
        opExport.ExportMin.setValue(100)
        opExport.ExportMax.setValue(200)

        opExport.OutputFormat.setValue('hdf5')
        opExport.OutputFilenameFormat.setValue(
            self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
        opExport.OutputInternalPath.setValue('volume/data')

        opExport.TransactionSlot.setValue(True)

        assert opExport.ImageToExport.ready()
        assert opExport.ExportPath.ready()
        assert opExport.ImageToExport.meta.drange == (100, 200)

        #print "exporting data to: {}".format( opExport.ExportPath.value )
        assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
        opExport.run_export()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(opExport.ExportPath.value)

            # Compare with the correct subregion and convert dtype.
            sub_roi[1] = (100, 80)  # Replace 'None' with full extent
            expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
            expected_data = expected_data.astype(numpy.uint8)
            expected_data += 100  # see renormalization settings

            assert opRead.Output.meta.shape == expected_data.shape
            assert opRead.Output.meta.dtype == expected_data.dtype
            read_data = opRead.Output[:].wait()

            # Due to rounding errors, the actual result and the expected result may differ by 1
            #  e.g. if the original pixel value was 32.99999999
            # Also, must promote to signed values to avoid unsigned rollover
            # See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
            expected_data_signed = expected_data.astype(numpy.int16)
            read_data_signed = expected_data.astype(numpy.int16)
            difference_from_expected = expected_data_signed - read_data_signed
            assert (numpy.abs(difference_from_expected) <=
                    1).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
Exemple #19
0
    def testBasic(self):
        graph = Graph()
        opExport = OpDataExport(graph=graph)
        try:
            opExport.TransactionSlot.setValue(True)
            opExport.WorkingDirectory.setValue(self._tmpdir)

            # Simulate the important fields of a DatasetInfo object
            class MockDatasetInfo(object):
                pass

            rawInfo = MockDatasetInfo()
            rawInfo.nickname = 'test_nickname'
            rawInfo.filePath = './somefile.h5'
            opExport.RawDatasetInfo.setValue(rawInfo)

            opExport.SelectionNames.setValue(['Mock Export Data'])

            data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
            data = vigra.taggedView(data, vigra.defaultAxistags('xy'))

            opExport.Inputs.resize(1)
            opExport.Inputs[0].setValue(data)

            sub_roi = [(10, 20), (90, 80)]
            opExport.RegionStart.setValue(sub_roi[0])
            opExport.RegionStop.setValue(sub_roi[1])

            opExport.ExportDtype.setValue(numpy.uint8)

            opExport.OutputFormat.setValue('hdf5')
            opExport.OutputFilenameFormat.setValue(
                '{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}'
            )
            opExport.OutputInternalPath.setValue('volume/data')

            assert opExport.ImageToExport.ready()
            assert opExport.ExportPath.ready()

            expected_path = self._tmpdir + '/' + rawInfo.nickname + '_export_x10-90_y20-80.h5/volume/data'
            computed_path = opExport.ExportPath.value
            assert os.path.normpath(computed_path) == os.path.normpath(expected_path), \
                "Expected {}\nGot: {}".format( expected_path, computed_path )
            opExport.run_export()
        finally:
            opExport.cleanUp()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(computed_path)

            # Compare with the correct subregion and convert dtype.
            expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
            expected_data = expected_data.astype(numpy.uint8)
            read_data = opRead.Output[:].wait()
            assert (read_data == expected_data
                    ).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
Exemple #20
0
    def testBasic(self, tmp_h5_single_dataset: Path):
        graph = Graph()
        opExport = OpDataExport(graph=graph)
        try:
            opExport.TransactionSlot.setValue(True)
            opExport.WorkingDirectory.setValue(self._tmpdir)

            rawInfo = FilesystemDatasetInfo(filePath=str(
                tmp_h5_single_dataset / "test_group/test_data"),
                                            nickname="test_nickname")

            opExport.RawDatasetInfo.setValue(rawInfo)

            opExport.SelectionNames.setValue(["Mock Export Data"])

            data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
            data = vigra.taggedView(data, vigra.defaultAxistags("xy"))

            opExport.Inputs.resize(1)
            opExport.Inputs[0].setValue(data)

            sub_roi = [(10, 20), (90, 80)]
            opExport.RegionStart.setValue(sub_roi[0])
            opExport.RegionStop.setValue(sub_roi[1])

            opExport.ExportDtype.setValue(numpy.uint8)

            opExport.OutputFormat.setValue("hdf5")
            opExport.OutputFilenameFormat.setValue(
                "{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}"
            )
            opExport.OutputInternalPath.setValue("volume/data")

            assert opExport.ImageToExport.ready()
            assert opExport.ExportPath.ready()

            expected_path = tmp_h5_single_dataset.parent.joinpath(
                rawInfo.nickname +
                "_export_x10-90_y20-80.h5/volume/data").as_posix()
            computed_path = opExport.ExportPath.value
            assert os.path.normpath(computed_path) == os.path.normpath(
                expected_path)
            opExport.run_export()
        finally:
            opExport.cleanUp()

        opRead = OpInputDataReader(graph=graph)
        try:
            opRead.FilePath.setValue(computed_path)

            # Compare with the correct subregion and convert dtype.
            expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
            expected_data = expected_data.astype(numpy.uint8)
            read_data = opRead.Output[:].wait()
            assert (read_data == expected_data
                    ).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
    def testBasic(self):
        graph = Graph()
        opExport = OpFormattedDataExport(graph=graph)
        
        data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100
        data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
        opExport.Input.setValue(data)

        sub_roi = [(10, 0), (None, 80)]
        opExport.RegionStart.setValue( sub_roi[0] )
        opExport.RegionStop.setValue( sub_roi[1] )
        
        opExport.ExportDtype.setValue( numpy.uint8 )

        opExport.InputMin.setValue( 0.0 )
        opExport.InputMax.setValue( 100.0 )
        opExport.ExportMin.setValue( 100 )
        opExport.ExportMax.setValue( 200 )
        
        opExport.OutputFormat.setValue( 'hdf5' )
        opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
        opExport.OutputInternalPath.setValue('volume/data')
        
        opExport.TransactionSlot.setValue( True )

        assert opExport.ImageToExport.ready()
        assert opExport.ExportPath.ready()
        assert opExport.ImageToExport.meta.drange == (100,200)
        
        #print "exporting data to: {}".format( opExport.ExportPath.value )
        assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
        opExport.run_export()
        
        opRead = OpInputDataReader( graph=graph )
        try:
            opRead.FilePath.setValue( opExport.ExportPath.value )
    
            # Compare with the correct subregion and convert dtype.
            sub_roi[1] = (100, 80) # Replace 'None' with full extent
            expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
            expected_data = expected_data.astype(numpy.uint8)
            expected_data += 100 # see renormalization settings
    
            assert opRead.Output.meta.shape == expected_data.shape
            assert opRead.Output.meta.dtype == expected_data.dtype
            read_data = opRead.Output[:].wait()
    
            # Due to rounding errors, the actual result and the expected result may differ by 1
            #  e.g. if the original pixel value was 32.99999999
            # Also, must promote to signed values to avoid unsigned rollover
            # See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
            expected_data_signed = expected_data.astype(numpy.int16)
            read_data_signed = expected_data.astype(numpy.int16)
            difference_from_expected = expected_data_signed - read_data_signed
            assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!"
        finally:
            opRead.cleanUp()
Exemple #22
0
 def testBasic(self):
     graph = Graph()
     opExport = OpDataExport(graph=graph)
     try:
         opExport.TransactionSlot.setValue(True)        
         opExport.WorkingDirectory.setValue( self._tmpdir )
         
         # Simulate the important fields of a DatasetInfo object
         class MockDatasetInfo(object): pass
         rawInfo = MockDatasetInfo()
         rawInfo.nickname = 'test_nickname'
         rawInfo.filePath = './somefile.h5'
         opExport.RawDatasetInfo.setValue( rawInfo )
 
         opExport.SelectionNames.setValue(['Mock Export Data'])
 
         data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100
         data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
         
         opExport.Inputs.resize(1)
         opExport.Inputs[0].setValue(data)
 
         sub_roi = [(10, 20), (90, 80)]
         opExport.RegionStart.setValue( sub_roi[0] )
         opExport.RegionStop.setValue( sub_roi[1] )
         
         opExport.ExportDtype.setValue( numpy.uint8 )
         
         opExport.OutputFormat.setValue( 'hdf5' )
         opExport.OutputFilenameFormat.setValue( '{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
         opExport.OutputInternalPath.setValue('volume/data')
 
         assert opExport.ImageToExport.ready()
         assert opExport.ExportPath.ready()
         
         expected_path = self._tmpdir + '/' + rawInfo.nickname + '_export_x10-90_y20-80.h5/volume/data'
         computed_path = opExport.ExportPath.value
         assert os.path.normpath(computed_path) == os.path.normpath(expected_path), \
             "Expected {}\nGot: {}".format( expected_path, computed_path )
         opExport.run_export()
     finally:
         opExport.cleanUp()
     
     opRead = OpInputDataReader( graph=graph )
     try:
         opRead.FilePath.setValue( computed_path )
 
         # Compare with the correct subregion and convert dtype.
         expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
         expected_data = expected_data.astype(numpy.uint8)
         read_data = opRead.Output[:].wait()
         assert (read_data == expected_data).all(), "Read data didn't match exported data!"
     finally:
         opRead.cleanUp()
        def handleImportLabelsAction():
            # Find the directory of the most recently opened image file
            mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
            if mostRecentImageFile is not None:
                defaultDirectory = os.path.split(mostRecentImageFile)[0]
            else:
                defaultDirectory = os.path.expanduser('~')
            fileNames = DataSelectionGui.getImageFileNamesToOpen(self, defaultDirectory)
            fileNames = list(map(str, fileNames))
            
            # For now, we require a single hdf5 file
            if len(fileNames) > 1:
                QMessageBox.critical(self, "Too many files", 
                                     "Labels must be contained in a single hdf5 volume.")
                return
            if len(fileNames) == 0:
                # user cancelled
                return
            
            file_path = fileNames[0]
            internal_paths = DataSelectionGui.getPossibleInternalPaths(file_path)
            if len(internal_paths) == 0:
                QMessageBox.critical(self, "No volumes in file", 
                                     "Couldn't find a suitable dataset in your hdf5 file.")
                return
            if len(internal_paths) == 1:
                internal_path = internal_paths[0]
            else:
                dlg = H5VolumeSelectionDlg(internal_paths, self)
                if dlg.exec_() == QDialog.Rejected:
                    return
                selected_index = dlg.combo.currentIndex()
                internal_path = str(internal_paths[selected_index])

            path_components = PathComponents(file_path)
            path_components.internalPath = str(internal_path)
            
            try:
                top_op = self.topLevelOperatorView
                opReader = OpInputDataReader(parent=top_op.parent)
                opReader.FilePath.setValue( path_components.totalPath() )
                
                # Reorder the axes
                op5 = OpReorderAxes(parent=top_op.parent)
                op5.AxisOrder.setValue( top_op.LabelInputs.meta.getAxisKeys() )
                op5.Input.connect( opReader.Output )
            
                # Finally, import the labels
                top_op.importLabels( top_op.current_view_index(), op5.Output )
                    
            finally:
                op5.cleanUp()
                opReader.cleanUp()
Exemple #24
0
        def handleImportLabelsAction():
            fileNames = ImageFileDialog(
                self,
                preferences_group="DataSelection",
                preferences_setting="recent image").getSelectedPaths()
            fileNames = list(map(str, fileNames))

            # For now, we require a single hdf5 file
            if len(fileNames) > 1:
                QMessageBox.critical(
                    self, "Too many files",
                    "Labels must be contained in a single hdf5 volume.")
                return
            if len(fileNames) == 0:
                # user cancelled
                return

            file_path = fileNames[0]
            internal_paths = DatasetInfo.getPossibleInternalPathsFor(file_path)
            if len(internal_paths) == 0:
                QMessageBox.critical(
                    self, "No volumes in file",
                    "Couldn't find a suitable dataset in your hdf5 file.")
                return
            if len(internal_paths) == 1:
                internal_path = internal_paths[0]
            else:
                dlg = SubvolumeSelectionDlg(internal_paths, self)
                if dlg.exec_() == QDialog.Rejected:
                    return
                selected_index = dlg.combo.currentIndex()
                internal_path = str(internal_paths[selected_index])

            path_components = PathComponents(file_path)
            path_components.internalPath = str(internal_path)

            try:
                top_op = self.topLevelOperatorView
                opReader = OpInputDataReader(parent=top_op.parent)
                opReader.FilePath.setValue(path_components.totalPath())

                # Reorder the axes
                op5 = OpReorderAxes(parent=top_op.parent)
                op5.AxisOrder.setValue(top_op.LabelInputs.meta.getAxisKeys())
                op5.Input.connect(opReader.Output)

                # Finally, import the labels
                top_op.importLabels(top_op.current_view_index(), op5.Output)

            finally:
                op5.cleanUp()
                opReader.cleanUp()
    def test_npy_with_roi(self):
        a = numpy.indices((100, 100, 200)).astype(numpy.uint8).sum(0)
        assert a.shape == (100, 100, 200)
        numpy.save(self.testNpyDataFileName, a)
        opReader = OpInputDataReader(graph=lazyflow.graph.Graph())
        try:
            opReader.FilePath.setValue(self.testNpyDataFileName)
            opReader.SubVolumeRoi.setValue(((10, 20, 30), (50, 70, 90)))

            all_data = opReader.Output[:].wait()
            assert all_data.shape == (40, 50, 60)
            assert (all_data == a[10:50, 20:70, 30:90]).all()
        finally:
            opReader.cleanUp()
Exemple #26
0
    def test_npy_with_roi(self):
        a = numpy.indices((100, 100, 200)).astype(numpy.uint8).sum(0)
        assert a.shape == (100, 100, 200)
        numpy.save(self.testNpyDataFileName, a)
        opReader = OpInputDataReader(graph=lazyflow.graph.Graph())
        try:
            opReader.FilePath.setValue(self.testNpyDataFileName)
            opReader.SubVolumeRoi.setValue(((10, 20, 30), (50, 70, 90)))

            all_data = opReader.Output[:].wait()
            assert all_data.shape == (40, 50, 60)
            assert (all_data == a[10:50, 20:70, 30:90]).all()
        finally:
            opReader.cleanUp()
    def testBasic_Dvid(self):
        if _skip_dvid:
            raise nose.SkipTest

        # Spin up a mock dvid server to test with.
        dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
        mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
        with H5MockServerDataFile(mockserver_data_file) as test_h5file:
            test_h5file.add_node(dvid_dataset, data_uuid)
        server_proc, shutdown_event = H5MockServer.create_and_start(
            mockserver_data_file,
            "localhost",
            8000,
            same_process=False,
            disable_server_logging=True)

        try:
            data = 255 * numpy.random.random((100, 100, 4))
            data = data.astype(numpy.uint8)
            data = vigra.taggedView(data, vigra.defaultAxistags('xyc'))

            graph = Graph()

            opPiper = OpArrayPiper(graph=graph)
            opPiper.Input.setValue(data)

            opExport = OpExportSlot(graph=graph)
            opExport.Input.connect(opPiper.Output)
            opExport.OutputFormat.setValue('dvid')
            url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format(
                **locals())
            opExport.OutputFilenameFormat.setValue(url)

            assert opExport.ExportPath.ready()
            assert opExport.ExportPath.value == url
            opExport.run_export()

            try:
                opRead = OpInputDataReader(graph=graph)
                opRead.FilePath.setValue(opExport.ExportPath.value)
                expected_data = data.view(numpy.ndarray)
                read_data = opRead.Output[:].wait()
                assert (read_data == expected_data
                        ).all(), "Read data didn't match exported data!"
            finally:
                opRead.cleanUp()
        finally:
            shutdown_event.set()
            server_proc.join()
Exemple #28
0
    def testBasic_Dvid(self):
        if _skip_dvid:
            raise nose.SkipTest
        
        # Spin up a mock dvid server to test with.
        dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
        mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
        with H5MockServerDataFile( mockserver_data_file ) as test_h5file:
            test_h5file.add_node( dvid_dataset, data_uuid )
        server_proc, shutdown_event = H5MockServer.create_and_start( mockserver_data_file, "localhost", 8000,
                                                                     same_process=False, disable_server_logging=True )

        try:            
            data = 255 * numpy.random.random( (100,100, 4) )
            data = data.astype( numpy.uint8 )
            data = vigra.taggedView( data, vigra.defaultAxistags('xyc') )
            
            graph = Graph()
            
            opPiper = OpArrayPiper(graph=graph)
            opPiper.Input.setValue( data )
            
            opExport = OpExportSlot(graph=graph)
            opExport.Input.connect( opPiper.Output )
            opExport.OutputFormat.setValue( 'dvid' )
            url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format( **locals() )
            opExport.OutputFilenameFormat.setValue( url )
            
            assert opExport.ExportPath.ready()
            assert opExport.ExportPath.value == url
            opExport.run_export()

            try:
                opRead = OpInputDataReader( graph=graph )
                opRead.FilePath.setValue( opExport.ExportPath.value )
                expected_data = data.view(numpy.ndarray)
                read_data = opRead.Output[:].wait()
                assert (read_data == expected_data).all(), "Read data didn't match exported data!"
            finally:
                opRead.cleanUp()
        finally:
            shutdown_event.set()
            server_proc.join()
Exemple #29
0
    def __init__(
        self,
        *,
        filePath: str,
        project_file: h5py.File = None,
        sequence_axis: str = None,
        nickname: str = "",
        drange: Tuple[Number, Number] = None,
        **info_kwargs,
    ):
        """
        sequence_axis: Axis along which to stack (only applicable for stacks).
        """
        self.sequence_axis = sequence_axis
        self.base_dir = str(Path(project_file.filename).absolute().parent
                            ) if project_file else os.getcwd()
        assert os.path.isabs(
            self.base_dir
        )  # FIXME: if file_project was opened as a relative path, this would break
        self.expanded_paths = self.expand_path(filePath, cwd=self.base_dir)
        assert len(self.expanded_paths) == 1 or self.sequence_axis
        if len({PathComponents(ep).extension
                for ep in self.expanded_paths}) > 1:
            raise Exception(
                f"Multiple extensions unsupported as a single data source: {self.expanded_paths}"
            )
        self.filePath = os.path.pathsep.join(self.expanded_paths)

        op_reader = OpInputDataReader(graph=Graph(),
                                      WorkingDirectory=self.base_dir,
                                      FilePath=self.filePath,
                                      SequenceAxis=self.sequence_axis)
        meta = op_reader.Output.meta.copy()
        op_reader.cleanUp()

        super().__init__(
            default_tags=meta.axistags,
            nickname=nickname or self.create_nickname(self.expanded_paths),
            laneShape=meta.shape,
            laneDtype=meta.dtype,
            drange=drange or meta.get("drange"),
            **info_kwargs,
        )
    def testBasic(self):
        data = numpy.random.random( (100,100) ).astype( numpy.float32 )
        data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
        
        graph = Graph()
        opWriter = OpNpyWriter(graph=graph)
        opWriter.Input.setValue(data)
        opWriter.Filepath.setValue( self._tmpdir + '/npy_writer_test_output.npy' )

        # Write it...
        opWriter.write()
        
        opRead = OpInputDataReader( graph=graph )
        opRead.FilePath.setValue( opWriter.Filepath.value )
        expected_data = data.view(numpy.ndarray)
        read_data = opRead.Output[:].wait()
        assert (read_data == expected_data).all(), "Read data didn't match exported data!"
        
        opRead.cleanUp()
 def importStackAsLocalDataset(self,
                               abs_paths: List[str],
                               sequence_axis: str = "z",
                               progress_signal: Callable[[int],
                                                         None] = None):
     progress_signal = progress_signal or self.progressSignal
     progress_signal(0)
     op_reader = None
     op_writer = None
     try:
         colon_paths = os.path.pathsep.join(abs_paths)
         op_reader = OpInputDataReader(graph=self.topLevelOperator.graph,
                                       FilePath=colon_paths,
                                       SequenceAxis=sequence_axis)
         axistags = op_reader.Output.meta.axistags
         inner_path = self.local_data_path.joinpath(
             DatasetInfo.generate_id()).as_posix()
         project_file = self.topLevelOperator.ProjectFile.value
         op_writer = OpH5N5WriterBigDataset(
             graph=self.topLevelOperator.graph,
             h5N5File=project_file,
             h5N5Path=inner_path,
             CompressionEnabled=False,
             BatchSize=1,
             Image=op_reader.Output,
         )
         op_writer.progressSignal.subscribe(progress_signal)
         success = op_writer.WriteImage.value
         for index, tag in enumerate(axistags):
             project_file[inner_path].dims[index].label = tag.key
         project_file[inner_path].attrs["axistags"] = axistags.toJSON()
         if op_reader.Output.meta.get("drange"):
             project_file[inner_path].attrs[
                 "drange"] = op_reader.Output.meta.get("drange")
         return inner_path
     finally:
         if op_writer:
             op_writer.Image.disconnect()
         if op_reader:
             op_reader.cleanUp()
         progress_signal(100)
    def test_tiff_stack_multi_file(self, sequence_axis):
        """Test stack/sequence reading in hdf5-files"""
        shape = (4, 8, 16, 3)
        data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
        for idx, data_slice in enumerate(data):
            im = Image.fromarray(data_slice, mode="RGB")
            im.save(self.testmultiTiffFileName.format(index=idx))

        if sequence_axis == "c":
            data = numpy.concatenate(data, axis=-1)

        reader = OpInputDataReader(graph=self.graph)
        reader.SequenceAxis.setValue(sequence_axis)
        globString = self.testmultiTiffFileName.replace("02d}", "s}").format(index="*")
        try:
            reader.FilePath.setValue(globString)
            tiffdata = reader.Output[:].wait()

            assert tiffdata.shape == data.shape, f"{tiffdata.shape}, {data.shape}"
            numpy.testing.assert_array_equal(tiffdata, data)
        finally:
            # Call cleanUp() to close the file that this operator opened
            reader.cleanUp()
    def test_npy(self):
        # Create Numpy test data
        a = numpy.zeros((10, 11))
        for x in range(0, 10):
            for y in range(0, 11):
                a[x, y] = x + y
        numpy.save(self.testNpyDataFileName, a)

        # Now read back our test data using an OpInputDataReader operator
        npyReader = OpInputDataReader(graph=self.graph)
        try:
            npyReader.FilePath.setValue(self.testNpyDataFileName)
            cwd = os.path.split(__file__)[0]
            npyReader.WorkingDirectory.setValue(cwd)

            # Read the entire NPY file and verify the contents
            npyData = npyReader.Output[:].wait()
            assert npyData.shape == (10, 11)
            for x in range(0, 10):
                for y in range(0, 11):
                    assert npyData[x, y] == x + y
        finally:
            npyReader.cleanUp()
Exemple #34
0
    def test_tiff_stack_multi_file(self, sequence_axis):
        """Test stack/sequence reading in hdf5-files"""
        shape = (4, 8, 16, 3)
        data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
        for idx, data_slice in enumerate(data):
            im = Image.fromarray(data_slice, mode="RGB")
            im.save(self.testmultiTiffFileName.format(index=idx))

        if sequence_axis == "c":
            data = numpy.concatenate(data, axis=-1)

        reader = OpInputDataReader(graph=self.graph)
        reader.SequenceAxis.setValue(sequence_axis)
        globString = self.testmultiTiffFileName.replace("02d}", "s}").format(index="*")
        try:
            reader.FilePath.setValue(globString)
            tiffdata = reader.Output[:].wait()

            assert tiffdata.shape == data.shape, f"{tiffdata.shape}, {data.shape}"
            numpy.testing.assert_array_equal(tiffdata, data)
        finally:
            # Call cleanUp() to close the file that this operator opened
            reader.cleanUp()
Exemple #35
0
    def test_npy(self):
        # Create Numpy test data
        a = numpy.zeros((10, 11))
        for x in range(0, 10):
            for y in range(0, 11):
                a[x, y] = x + y
        numpy.save(self.testNpyDataFileName, a)

        # Now read back our test data using an OpInputDataReader operator
        npyReader = OpInputDataReader(graph=self.graph)
        try:
            npyReader.FilePath.setValue(self.testNpyDataFileName)
            cwd = os.path.split(__file__)[0]
            npyReader.WorkingDirectory.setValue(cwd)

            # Read the entire NPY file and verify the contents
            npyData = npyReader.Output[:].wait()
            assert npyData.shape == (10, 11)
            for x in range(0, 10):
                for y in range(0, 11):
                    assert npyData[x, y] == x + y
        finally:
            npyReader.cleanUp()
    def test_npz(self):
        # Create two Numpy test arrays
        a = numpy.zeros((10, 11))
        for x in range(0, 10):
            for y in range(0, 11):
                a[x, y] = x + y

        b = numpy.arange((3 * 9)).reshape((3, 9))
        numpy.savez(self.testNpzDataFileName, a=a, b=b)

        # Now read back our test data using an OpInputDataReader operator
        npyReader = OpInputDataReader(graph=self.graph)

        try:
            for internalPath, referenceArray in zip(["a", "b"], [a, b]):
                npyReader.FilePath.setValue("{}/{}".format(self.testNpzDataFileName, internalPath))
                cwd = os.path.split(__file__)[0]
                npyReader.WorkingDirectory.setValue(cwd)

                npzData = npyReader.Output[:].wait()
                assert npzData.shape == referenceArray.shape
                numpy.testing.assert_array_equal(npzData, referenceArray)
        finally:
            npyReader.cleanUp()
Exemple #37
0
class OpImageOnDiskProvider(Operator):
    """
    This simply wraps a lazyflow OpInputDataReader, but ensures that the metadata 
    (axistags, drange) on the output matches the metadata from the original data 
    (even if the output file format doesn't support metadata fields).
    """
    TransactionSlot = InputSlot()
    Input = InputSlot() # Used for dtype and shape only. Data is always provided directly from the file.

    WorkingDirectory = InputSlot()
    DatasetPath = InputSlot() # A TOTAL path (possibly including a dataset name, e.g. myfile.h5/volume/data
    Dirty = InputSlot()
    
    Output = OutputSlot()

    def __init__(self, *args, **kwargs):
        super( OpImageOnDiskProvider, self ).__init__(*args, **kwargs)
        self._opReader = None
        self._opMetadataInjector = None
    
    # Block diagram:
    #
    # (Input.axistags, Input.drange)
    #                               \  
    # DatasetPath ---> opReader ---> opMetadataInjector --> Output
    #                 /
    # WorkingDirectory
    
    def setupOutputs( self ):
        if self._opReader is not None:
            self.Output.disconnect()
            if self._opMetadataInjector:
                self._opMetadataInjector.cleanUp()
                self._opMetadataInjector = None
            self._opReader.cleanUp()
            self._opReader = None

        try:
            # Configure the reader
            dataReady = True
            self._opReader = OpInputDataReader( parent=self )
            self._opReader.WorkingDirectory.setValue( self.WorkingDirectory.value )
            self._opReader.FilePath.setValue( self.DatasetPath.value )

            # Since most file formats don't save meta-info,
            # The reader output's axis order may be incorrect.
            # (For example, if we export in npy format with zxy order, 
            #  the Npy reader op will simply assume xyz order when it reads the data.)

            # Force the metadata back to the correct state by copying select items from Input.meta
            metadata = {}
            metadata['axistags'] = self.Input.meta.axistags
            metadata['drange'] = self.Input.meta.drange
            metadata['display_mode'] = self.Input.meta.display_mode
            self._opMetadataInjector = OpMetadataInjector( parent=self )
            self._opMetadataInjector.Input.connect( self._opReader.Output )
            self._opMetadataInjector.Metadata.setValue( metadata )

            dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape
            dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype
            if dataReady:
                self.Output.connect( self._opMetadataInjector.Output )
            else:
                self._opMetadataInjector.cleanUp()
                self._opMetadataInjector = None
                self._opReader.cleanUp()
                self._opReader = None
                self.Output.meta.NOTREADY = True

        #except OpInputDataReader.DatasetReadError:
        except Exception as ex:
            #logger.debug( "On-disk image can't be read: {}".format(ex) )
            # Note: If the data is exported as a 'sequence', then this will always be NOTREADY
            #       because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent.
            #       That's okay because a stack is probably too slow to be of use for a preview anyway.
            if self._opMetadataInjector:
                self._opMetadataInjector.cleanUp()
                self._opMetadataInjector = None
            self._opReader.cleanUp()
            self._opReader = None
            # The dataset doesn't exist yet.
            self.Output.meta.NOTREADY = True

    def execute(self, slot, subindex, roi, result):
        assert False, "Output is supposed to be directly connected to an internal operator."

    def propagateDirty(self, slot, subindex, roi):
        if slot == self.Input:
            self.Output.setDirty( roi )
        else:
            self.Output.setDirty( slice(None) )
Exemple #38
0
def generate_trained_project_file(
    new_project_path, raw_data_paths, label_data_paths, feature_selections, classifier_factory=None
):
    """
    Create a new project file from scratch, add the given raw data files,
    inject the corresponding labels, configure the given feature selections,
    and (if provided) override the classifier type ('factory').
    
    Finally, request the classifier object from the pipeline (which forces training),
    and save the project.
    
    new_project_path: Where to save the new project file
    raw_data_paths: A list of paths to the raw data images to train with
    label_data_paths: A list of paths to the label image data to train with
    feature_selections: A matrix of bool, representing the selected features
    classifier_factory: Override the classifier type.  Must be a subclass of either:
                        - lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
                        - lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
    """
    assert len(raw_data_paths) == len(label_data_paths), "Number of label images must match number of raw images."

    import ilastik_main
    from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
    from lazyflow.graph import Graph
    from lazyflow.operators.ioOperators import OpInputDataReader
    from lazyflow.roi import roiToSlice, roiFromShape

    ##
    ## CREATE PROJECT
    ##

    # Manually configure the arguments to ilastik, as if they were parsed from the command line.
    # (Start with empty args and fill in below.)
    ilastik_args = ilastik_main.parser.parse_args([])
    ilastik_args.new_project = new_project_path
    ilastik_args.headless = True
    ilastik_args.workflow = "Pixel Classification"

    shell = ilastik_main.main(ilastik_args)
    assert isinstance(shell.workflow, PixelClassificationWorkflow)

    ##
    ## CONFIGURE GRAYSCALE INPUT
    ##

    data_selection_applet = shell.workflow.dataSelectionApplet

    # To configure data selection, start with empty cmdline args and manually fill them in
    data_selection_args, _ = data_selection_applet.parse_known_cmdline_args([], PixelClassificationWorkflow.ROLE_NAMES)
    data_selection_args.raw_data = raw_data_paths
    data_selection_args.preconvert_stacks = True

    # Simplest thing here is to configure using cmd-line interface
    data_selection_applet.configure_operator_with_parsed_args(data_selection_args)

    ##
    ## APPLY FEATURE MATRIX (from matrix above)
    ##

    opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
    opFeatures.Scales.setValue(ScalesList)
    opFeatures.FeatureIds.setValue(FeatureIds)
    opFeatures.SelectionMatrix.setValue(feature_selections)

    ##
    ## CUSTOMIZE CLASSIFIER TYPE
    ##

    opPixelClassification = shell.workflow.pcApplet.topLevelOperator
    if classifier_factory is not None:
        opPixelClassification.ClassifierFactory.setValue(classifier_factory)

    ##
    ## READ/APPLY LABEL VOLUMES
    ##

    # Read each label volume and inject the label data into the appropriate training slot
    cwd = os.getcwd()
    max_label_class = 0
    for lane, label_data_path in enumerate(label_data_paths):
        graph = Graph()
        opReader = OpInputDataReader(graph=graph)
        try:
            opReader.WorkingDirectory.setValue(cwd)
            opReader.FilePath.setValue(label_data_path)

            print "Reading label volume: {}".format(label_data_path)
            label_volume = opReader.Output[:].wait()
        finally:
            opReader.cleanUp()

        raw_shape = opPixelClassification.InputImages[lane].meta.shape
        if label_volume.ndim != len(raw_shape):
            # Append a singleton channel axis
            assert label_volume.ndim == len(raw_shape) - 1
            label_volume = label_volume[..., None]

        # Auto-calculate the max label value
        max_label_class = max(max_label_class, label_volume.max())

        print "Applying label volume to lane #{}".format(lane)
        entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
        opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume

    assert max_label_class > 1, "Not enough label classes were found in your label data."
    label_names = map(str, range(max_label_class))
    opPixelClassification.LabelNames.setValue(label_names)

    ##
    ## TRAIN CLASSIFIER
    ##

    # Make sure the caches in the pipeline are not 'frozen'.
    # (This is the equivalent of 'live update' mode in the GUI.)
    opPixelClassification.FreezePredictions.setValue(False)

    # Request the classifier object from the pipeline.
    # This forces the pipeline to produce (train) the classifier.
    _ = opPixelClassification.Classifier.value

    ##
    ## SAVE PROJECT
    ##

    # save project file (includes the new classifier).
    shell.projectManager.saveProject(force_all_save=False)
Exemple #39
0
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
    """
    Prompt the user for layer import settings, and perform the layer import.
    :param labelLayer: The top label layer source
    :param labelingSlots: An instance of LabelingGui.LabelingSlots
    :param parent_widget: The Qt GUI parent object
    """
    writeSeeds = labelingSlots.labelInput
    assert isinstance(
        writeSeeds,
        lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
    opLabels = writeSeeds.getRealOperator()
    assert isinstance(opLabels, lazyflow.graph.Operator
                      ), "slot's operator is of type %r" % (type(opLabels))

    recentlyImported = PreferencesManager().get('labeling',
                                                'recently imported')
    mostRecentProjectPath = PreferencesManager().get('shell',
                                                     'recently opened')
    mostRecentImageFile = PreferencesManager().get('DataSelection',
                                                   'recent image')
    if recentlyImported:
        defaultDirectory = os.path.split(recentlyImported)[0]
    elif mostRecentProjectPath:
        defaultDirectory = os.path.split(mostRecentProjectPath)[0]
    elif mostRecentImageFile:
        defaultDirectory = os.path.split(mostRecentImageFile)[0]
    else:
        defaultDirectory = os.path.expanduser('~')

    fileNames = DataSelectionGui.getImageFileNamesToOpen(
        parent_widget, defaultDirectory)
    fileNames = map(str, fileNames)

    if not fileNames:
        return

    PreferencesManager().set('labeling', 'recently imported', fileNames[0])

    try:
        # Initialize operators
        opImport = OpInputDataReader(parent=opLabels.parent)
        opCache = OpArrayCache(parent=opLabels.parent)
        opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
        opReorderAxes = OpReorderAxes(parent=opLabels.parent)

        # Set up the pipeline as follows:
        #
        #   opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
        #                           /                            /
        #   User-specified axisorder    labelInput.meta.axistags

        opImport.WorkingDirectory.setValue(defaultDirectory)
        opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
                                   1 else os.path.pathsep.join(fileNames))
        assert opImport.Output.ready()

        opCache.blockShape.setValue(opImport.Output.meta.shape)
        opCache.Input.connect(opImport.Output)
        assert opCache.Output.ready()

        opMetadataInjector.Input.connect(opCache.Output)
        metadata = opCache.Output.meta.copy()
        opMetadataInjector.Metadata.setValue(metadata)
        opReorderAxes.Input.connect(opMetadataInjector.Output)

        # Transpose the axes for assignment to the labeling operator.
        opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())

        # We'll show a little window with a busy indicator while the data is loading
        busy_dlg = QProgressDialog(parent=parent_widget)
        busy_dlg.setLabelText("Importing Label Data...")
        busy_dlg.setCancelButton(None)
        busy_dlg.setMinimum(100)
        busy_dlg.setMaximum(100)

        def close_busy_dlg(*args):
            QApplication.postEvent(busy_dlg, QCloseEvent())

        # Load the data from file into our cache
        # When it's done loading, close the progress dialog.
        req = opCache.Output[:]
        req.notify_finished(close_busy_dlg)
        req.notify_failed(close_busy_dlg)
        req.submit()
        busy_dlg.exec_()

        readData = req.result

        maxLabels = len(labelingSlots.labelNames.value)

        # Can't use return_counts feature because that requires numpy >= 1.9
        #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)

        # This does the same as the above, albeit slower, and probably with more ram.
        unique_read_labels = numpy.unique(readData)
        readLabelCounts = numpy.bincount(readData.flat)[unique_read_labels]

        labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
        del readData

        # Ask the user how to interpret the data.
        settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
                                            opMetadataInjector.Output,
                                            labelingSlots.labelInput,
                                            labelInfo)

        def handle_updated_axes():
            # The user is specifying a new interpretation of the file's axes
            updated_axisorder = str(settingsDlg.axesEdit.text())
            metadata = opMetadataInjector.Metadata.value.copy()
            metadata.axistags = vigra.defaultAxistags(updated_axisorder)
            opMetadataInjector.Metadata.setValue(metadata)

        settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)

        dlg_result = settingsDlg.exec_()
        if dlg_result != LabelImportOptionsDlg.Accepted:
            return

        # Get user's chosen label mapping from dlg
        labelMapping = settingsDlg.labelMapping

        # Get user's chosen offsets.
        # Offsets in dlg only include the file axes, not the 5D axes expected by the label input,
        # so expand them to full 5D
        axes_5d = opReorderAxes.Output.meta.getAxisKeys()
        tagged_offsets = collections.OrderedDict(
            zip(axes_5d, [0] * len(axes_5d)))
        tagged_offsets.update(
            dict(
                zip(opMetadataInjector.Output.meta.getAxisKeys(),
                    settingsDlg.imageOffsets)))
        imageOffsets = tagged_offsets.values()

        # Optimization if mapping is identity
        if labelMapping.keys() == labelMapping.values():
            labelMapping = None

        # This will be fast (it's already cached)
        label_data = opReorderAxes.Output[:].wait()

        # Map input labels to output labels
        if labelMapping:
            # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
            # But this supports potentially huge values of unique_read_labels (in the billions),
            # without needing GB of RAM.
            mapping_indexes = numpy.searchsorted(unique_read_labels,
                                                 label_data)
            new_labels = numpy.array(
                [labelMapping[x] for x in unique_read_labels])
            label_data[:] = new_labels[mapping_indexes]

        label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
        label_roi += imageOffsets
        label_slice = roiToSlice(*label_roi)
        writeSeeds[label_slice] = label_data

    finally:
        opReorderAxes.cleanUp()
        opMetadataInjector.cleanUp()
        opCache.cleanUp()
        opImport.cleanUp()
Exemple #40
0
        def impl():
            shell = self.shell
            workflow = shell.projectManager.workflow
            carvingApplet = workflow.carvingApplet
            gui = carvingApplet.getMultiLaneGui()
            op_carving = carvingApplet.topLevelOperator.getLane(0)

            # activate the carving applet
            shell.setSelectedAppletDrawer(2)
            # let the gui catch up
            QApplication.processEvents()
            self.waitForViews(gui.currentGui().editor.imageViews)
            # inject the labels
            op5 = OpReorderAxes(parent=op_carving.parent)
            opReader = OpInputDataReader(parent=op_carving.parent)
            try:
                opReader.FilePath.setValue(f"{self.reference_files['carving_label_file']}/exported_data")
                op5.AxisOrder.setValue(op_carving.WriteSeeds.meta.getAxisKeys())
                op5.Input.connect(opReader.Output)
                label_data = op5.Output[:].wait()
            finally:
                op5.cleanUp()
                opReader.cleanUp()
            slicing = roi.fullSlicing(label_data.shape)
            op_carving.WriteSeeds[slicing] = label_data

            gui.currentGui().labelingDrawerUi.segment.click()
            QApplication.processEvents()

            op_carving.saveObjectAs("Object 1")
            op_carving.deleteObject("<not saved yet>")

            # export the mesh:
            req = gui.currentGui()._exportMeshes(["Object 1"], [self.output_obj_file])
            req.wait()

            # compare meshes
            with open(self.output_obj_file, "r") as f:
                left = f.read()

            with open(self.reference_files["output_obj_file"], "r") as f:
                right = f.read()

            # TODO: might result in errors due to rounding on different systems
            assert left == right

            # export the completed segments layer
            layermatch = [
                x.name.startswith("Completed segments (unicolor)") for x in gui.currentGui().editor.layerStack
            ]
            assert sum(layermatch) == 1, "Completed segments (unicolor) Layer expected."
            completed_segments_layer = gui.currentGui().editor.layerStack[layermatch.index(True)]
            opExport = get_export_operator(completed_segments_layer)
            try:
                opExport.OutputFilenameFormat.setValue(self.output_file)
                opExport.run_export()
            finally:
                opExport.cleanUp()

            assert os.path.exists(self.output_file)

            # compare completed segments
            with h5py.File(self.reference_files["output_file"], "r") as f_left:
                data_left = f_left["exported_data"][:]

            with h5py.File(self.output_file, "r") as f_right:
                data_right = f_right["exported_data"][:]

            numpy.testing.assert_array_almost_equal(data_left, data_right)

            # Save the project
            saveThread = self.shell.onSaveProjectActionTriggered()
            saveThread.join()
Exemple #41
0
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
    """
    Prompt the user for layer import settings, and perform the layer import.
    :param labelLayer: The top label layer source
    :param labelingSlots: An instance of LabelingGui.LabelingSlots
    :param parent_widget: The Qt GUI parent object
    """
    writeSeeds = labelingSlots.labelInput
    assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
    opLabels = writeSeeds.getRealOperator()
    assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels))


    recentlyImported = PreferencesManager().get('labeling', 'recently imported')
    mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened')
    mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
    if recentlyImported:
        defaultDirectory = os.path.split(recentlyImported)[0]
    elif mostRecentProjectPath:
        defaultDirectory = os.path.split(mostRecentProjectPath)[0]
    elif mostRecentImageFile:
        defaultDirectory = os.path.split(mostRecentImageFile)[0]
    else:
        defaultDirectory = os.path.expanduser('~')

    fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory)
    fileNames = map(str, fileNames)

    if not fileNames:
        return

    PreferencesManager().set('labeling', 'recently imported', fileNames[0])

    try:
        # Initialize operators
        opImport = OpInputDataReader( parent=opLabels.parent )
        opCache = OpArrayCache( parent=opLabels.parent )
        opMetadataInjector = OpMetadataInjector( parent=opLabels.parent )
        opReorderAxes = OpReorderAxes( parent=opLabels.parent )
    
        # Set up the pipeline as follows:
        #
        #   opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
        #                           /                            /
        #   User-specified axisorder    labelInput.meta.axistags
    
        opImport.WorkingDirectory.setValue(defaultDirectory)
        opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else
                                   os.path.pathsep.join(fileNames))
        assert opImport.Output.ready()
    
        opCache.blockShape.setValue( opImport.Output.meta.shape )
        opCache.Input.connect( opImport.Output )
        assert opCache.Output.ready()

        opMetadataInjector.Input.connect( opCache.Output )
        metadata = opCache.Output.meta.copy()
        opMetadataInjector.Metadata.setValue( metadata )
        opReorderAxes.Input.connect( opMetadataInjector.Output )

        # Transpose the axes for assignment to the labeling operator.
        opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() )
    
        # We'll show a little window with a busy indicator while the data is loading
        busy_dlg = QProgressDialog(parent=parent_widget)
        busy_dlg.setLabelText("Importing Label Data...")
        busy_dlg.setCancelButton(None)
        busy_dlg.setMinimum(100)
        busy_dlg.setMaximum(100)
        def close_busy_dlg(*args):
            QApplication.postEvent(busy_dlg, QCloseEvent())
    
        # Load the data from file into our cache
        # When it's done loading, close the progress dialog.
        req = opCache.Output[:]
        req.notify_finished( close_busy_dlg )
        req.notify_failed( close_busy_dlg )
        req.submit()
        busy_dlg.exec_()

        readData = req.result
        
        maxLabels = len(labelingSlots.labelNames.value)

        # Can't use return_counts feature because that requires numpy >= 1.9
        #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)

        # This does the same as the above, albeit slower, and probably with more ram.
        unique_read_labels = numpy.unique(readData)
        readLabelCounts = vigra_bincount(readData)[unique_read_labels]

        labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
        del readData
    
        # Ask the user how to interpret the data.
        settingsDlg = LabelImportOptionsDlg( parent_widget,
                                             fileNames, opMetadataInjector.Output,
                                             labelingSlots.labelInput, labelInfo )

        def handle_updated_axes():
            # The user is specifying a new interpretation of the file's axes
            updated_axisorder = str(settingsDlg.axesEdit.text())
            metadata = opMetadataInjector.Metadata.value.copy()
            metadata.axistags = vigra.defaultAxistags(updated_axisorder)
            opMetadataInjector.Metadata.setValue( metadata )
            
            if opReorderAxes._invalid_axes:
                settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
                # Red background
                settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);"
                                                   "selection-background-color: rgb(128, 128, 255); }")
        settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes )
        
        # Initialize
        handle_updated_axes()

        dlg_result = settingsDlg.exec_()
        if dlg_result != LabelImportOptionsDlg.Accepted:
            return

        # Get user's chosen label mapping from dlg
        labelMapping = settingsDlg.labelMapping    

        # Get user's chosen offsets.
        # Offsets in dlg only include the file axes, not the 5D axes expected by the label input,
        # so expand them to full 5D 
        axes_5d = opReorderAxes.Output.meta.getAxisKeys()
        tagged_offsets = collections.OrderedDict( zip( axes_5d, [0]*len(axes_5d) ) )
        tagged_offsets.update( dict( zip( opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets ) ) )
        imageOffsets = tagged_offsets.values()

        # Optimization if mapping is identity
        if labelMapping.keys() == labelMapping.values():
            labelMapping = None

        # This will be fast (it's already cached)
        label_data = opReorderAxes.Output[:].wait()
        
        # Map input labels to output labels
        if labelMapping:
            # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
            # But this supports potentially huge values of unique_read_labels (in the billions),
            # without needing GB of RAM.
            mapping_indexes = numpy.searchsorted(unique_read_labels, label_data)
            new_labels = numpy.array([labelMapping[x] for x in unique_read_labels])
            label_data[:] = new_labels[mapping_indexes]

        label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) )
        label_roi += imageOffsets
        label_slice = roiToSlice(*label_roi)
        writeSeeds[label_slice] = label_data

    finally:
        opReorderAxes.cleanUp()
        opMetadataInjector.cleanUp()
        opCache.cleanUp()
        opImport.cleanUp()
Exemple #42
0
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
    """
    Prompt the user for layer import settings, and perform the layer import.
    :param labelLayer: The top label layer source
    :param labelingSlots: An instance of LabelingGui.LabelingSlots
    :param parent_widget: The Qt GUI parent object
    """
    writeSeeds = labelingSlots.labelInput
    assert isinstance(
        writeSeeds,
        lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
    opLabels = writeSeeds.getRealOperator()
    assert isinstance(opLabels, lazyflow.graph.Operator
                      ), "slot's operator is of type %r" % (type(opLabels))

    recentlyImported = PreferencesManager().get('labeling',
                                                'recently imported')
    mostRecentProjectPath = PreferencesManager().get('shell',
                                                     'recently opened')
    mostRecentImageFile = PreferencesManager().get('DataSelection',
                                                   'recent image')
    if recentlyImported:
        defaultDirectory = os.path.split(recentlyImported)[0]
    elif mostRecentProjectPath:
        defaultDirectory = os.path.split(mostRecentProjectPath)[0]
    elif mostRecentImageFile:
        defaultDirectory = os.path.split(mostRecentImageFile)[0]
    else:
        defaultDirectory = os.path.expanduser('~')

    fileNames = DataSelectionGui.getImageFileNamesToOpen(
        parent_widget, defaultDirectory)
    fileNames = map(str, fileNames)

    if not fileNames:
        return

    PreferencesManager().set('labeling', 'recently imported', fileNames[0])

    try:
        # Initialize operators
        opImport = OpInputDataReader(parent=opLabels.parent)
        opCache = OpBlockedArrayCache(parent=opLabels.parent)
        opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
        opReorderAxes = OpReorderAxes(parent=opLabels.parent)

        # Set up the pipeline as follows:
        #
        #   opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
        #                             /                            /
        #     User-specified axisorder    labelInput.meta.axistags

        opImport.WorkingDirectory.setValue(defaultDirectory)
        opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
                                   1 else os.path.pathsep.join(fileNames))
        assert opImport.Output.ready()

        maxLabels = len(labelingSlots.labelNames.value)

        # We don't bother with counting the label pixels
        # (and caching the data) if it's big (1 GB)
        if numpy.prod(opImport.Output.meta.shape) > 1e9:
            reading_slot = opImport.Output

            # For huge data, we don't go through and search for the pixel values,
            # because that takes an annoyingly long amount of time.
            # Instead, we make the reasonable assumption that the input labels are already 1,2,3..N
            # and we don't tell the user what the label pixel counts are.
            unique_read_labels = numpy.array(range(maxLabels + 1))
            readLabelCounts = numpy.array([-1] * (maxLabels + 1))
            labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
        else:
            opCache.Input.connect(opImport.Output)
            opCache.CompressionEnabled.setValue(True)
            assert opCache.Output.ready()
            reading_slot = opCache.Output

            # We'll show a little window with a busy indicator while the data is loading
            busy_dlg = QProgressDialog(parent=parent_widget)
            busy_dlg.setLabelText("Scanning Label Data...")
            busy_dlg.setCancelButton(None)
            busy_dlg.setMinimum(100)
            busy_dlg.setMaximum(100)

            def close_busy_dlg(*args):
                QApplication.postEvent(busy_dlg, QCloseEvent())

            # Load the data from file into our cache
            # When it's done loading, close the progress dialog.
            req = reading_slot[:]
            req.notify_finished(close_busy_dlg)
            req.notify_failed(close_busy_dlg)
            req.submit()
            busy_dlg.exec_()

            readData = req.result

            # Can't use return_counts feature because that requires numpy >= 1.9
            #unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)

            # This does the same as the above, albeit slower, and probably with more ram.
            bincounts = chunked_bincount(readData)
            unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype,
                                                               copy=False)
            readLabelCounts = bincounts[unique_read_labels]

            labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
            del readData

        opMetadataInjector.Input.connect(reading_slot)
        metadata = reading_slot.meta.copy()
        opMetadataInjector.Metadata.setValue(metadata)
        opReorderAxes.Input.connect(opMetadataInjector.Output)

        # Transpose the axes for assignment to the labeling operator.
        opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())

        # Ask the user how to interpret the data.
        settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
                                            opMetadataInjector.Output,
                                            labelingSlots.labelInput,
                                            labelInfo)

        def handle_updated_axes():
            # The user is specifying a new interpretation of the file's axes
            updated_axisorder = str(settingsDlg.axesEdit.text())
            metadata = opMetadataInjector.Metadata.value.copy()
            metadata.axistags = vigra.defaultAxistags(updated_axisorder)
            opMetadataInjector.Metadata.setValue(metadata)

            if opReorderAxes._invalid_axes:
                settingsDlg.buttonBox.button(
                    QDialogButtonBox.Ok).setEnabled(False)
                # Red background
                settingsDlg.axesEdit.setStyleSheet(
                    "QLineEdit { background: rgb(255, 128, 128);"
                    "selection-background-color: rgb(128, 128, 255); }")

        settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)

        # Initialize
        handle_updated_axes()

        dlg_result = settingsDlg.exec_()
        if dlg_result != LabelImportOptionsDlg.Accepted:
            return

        # Get user's chosen label mapping from dlg
        labelMapping = settingsDlg.labelMapping

        # Get user's chosen offsets, ordered by the 'write seeds' slot
        axes_5d = opReorderAxes.Output.meta.getAxisKeys()
        tagged_offsets = collections.OrderedDict(
            zip(axes_5d, [0] * len(axes_5d)))
        tagged_offsets.update(
            dict(
                zip(opReorderAxes.Output.meta.getAxisKeys(),
                    settingsDlg.imageOffsets)))
        imageOffsets = tagged_offsets.values()

        # Optimization if mapping is identity
        if labelMapping.keys() == labelMapping.values():
            labelMapping = None

        # If the data was already cached, this will be fast.
        label_data = opReorderAxes.Output[:].wait()

        # Map input labels to output labels
        if labelMapping:
            # There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
            # But this supports potentially huge values of unique_read_labels (in the billions),
            # without needing GB of RAM.
            mapping_indexes = numpy.searchsorted(unique_read_labels,
                                                 label_data)
            new_labels = numpy.array(
                [labelMapping[x] for x in unique_read_labels])
            label_data[:] = new_labels[mapping_indexes]

        label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
        label_roi += imageOffsets
        label_slice = roiToSlice(*label_roi)
        writeSeeds[label_slice] = label_data

    finally:
        opReorderAxes.cleanUp()
        opMetadataInjector.cleanUp()
        opCache.cleanUp()
        opImport.cleanUp()
Exemple #43
0
def generate_trained_project_file(new_project_path,
                                  raw_data_paths,
                                  label_data_paths,
                                  feature_selections,
                                  classifier_factory=None):
    """
    Create a new project file from scratch, add the given raw data files,
    inject the corresponding labels, configure the given feature selections,
    and (if provided) override the classifier type ('factory').

    Finally, request the classifier object from the pipeline (which forces training),
    and save the project.

    new_project_path: Where to save the new project file
    raw_data_paths: A list of paths to the raw data images to train with
    label_data_paths: A list of paths to the label image data to train with
    feature_selections: A matrix of bool, representing the selected features
    classifier_factory: Override the classifier type.  Must be a subclass of either:
                        - lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
                        - lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
    """
    assert len(raw_data_paths) == len(
        label_data_paths
    ), "Number of label images must match number of raw images."

    import ilastik_main
    from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
    from lazyflow.graph import Graph
    from lazyflow.operators.ioOperators import OpInputDataReader
    from lazyflow.roi import roiToSlice, roiFromShape

    ##
    ## CREATE PROJECT
    ##

    # Manually configure the arguments to ilastik, as if they were parsed from the command line.
    # (Start with empty args and fill in below.)
    ilastik_args = ilastik_main.parse_args([])
    ilastik_args.new_project = new_project_path
    ilastik_args.headless = True
    ilastik_args.workflow = "Pixel Classification"

    shell = ilastik_main.main(ilastik_args)
    assert isinstance(shell.workflow, PixelClassificationWorkflow)

    ##
    ## CONFIGURE GRAYSCALE INPUT
    ##

    data_selection_applet = shell.workflow.dataSelectionApplet

    # To configure data selection, start with empty cmdline args and manually fill them in
    data_selection_args, _ = data_selection_applet.parse_known_cmdline_args(
        [], PixelClassificationWorkflow.ROLE_NAMES)
    data_selection_args.raw_data = raw_data_paths
    data_selection_args.preconvert_stacks = True

    # Simplest thing here is to configure using cmd-line interface
    data_selection_applet.configure_operator_with_parsed_args(
        data_selection_args)

    ##
    ## APPLY FEATURE MATRIX (from matrix above)
    ##

    opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
    opFeatures.Scales.setValue(ScalesList)
    opFeatures.FeatureIds.setValue(FeatureIds)
    opFeatures.SelectionMatrix.setValue(feature_selections)

    ##
    ## CUSTOMIZE CLASSIFIER TYPE
    ##

    opPixelClassification = shell.workflow.pcApplet.topLevelOperator
    if classifier_factory is not None:
        opPixelClassification.ClassifierFactory.setValue(classifier_factory)

    ##
    ## READ/APPLY LABEL VOLUMES
    ##

    # Read each label volume and inject the label data into the appropriate training slot
    cwd = os.getcwd()
    max_label_class = 0
    for lane, label_data_path in enumerate(label_data_paths):
        graph = Graph()
        opReader = OpInputDataReader(graph=graph)
        try:
            opReader.WorkingDirectory.setValue(cwd)
            opReader.FilePath.setValue(label_data_path)

            print("Reading label volume: {}".format(label_data_path))
            label_volume = opReader.Output[:].wait()
        finally:
            opReader.cleanUp()

        raw_shape = opPixelClassification.InputImages[lane].meta.shape
        if label_volume.ndim != len(raw_shape):
            # Append a singleton channel axis
            assert label_volume.ndim == len(raw_shape) - 1
            label_volume = label_volume[..., None]

        # Auto-calculate the max label value
        max_label_class = max(max_label_class, label_volume.max())

        print("Applying label volume to lane #{}".format(lane))
        entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
        opPixelClassification.LabelInputs[lane][
            entire_volume_slicing] = label_volume

    assert max_label_class > 1, "Not enough label classes were found in your label data."
    label_names = list(map(str, list(range(max_label_class))))
    opPixelClassification.LabelNames.setValue(label_names)

    ##
    ## TRAIN CLASSIFIER
    ##

    # Make sure the caches in the pipeline are not 'frozen'.
    # (This is the equivalent of 'live update' mode in the GUI.)
    opPixelClassification.FreezePredictions.setValue(False)

    # Request the classifier object from the pipeline.
    # This forces the pipeline to produce (train) the classifier.
    _ = opPixelClassification.Classifier.value

    ##
    ## SAVE PROJECT
    ##

    # save project file (includes the new classifier).
    shell.projectManager.saveProject(force_all_save=False)
Exemple #44
0
class OpImageOnDiskProvider(Operator):
    """
    This simply wraps a lazyflow OpInputDataReader, but ensures that the metadata 
    (axistags, drange) on the output matches the metadata from the original data 
    (even if the output file format doesn't support metadata fields).
    """
    TransactionSlot = InputSlot()
    Input = InputSlot() # Used for dtype and shape only. Data is always provided directly from the file.

    WorkingDirectory = InputSlot()
    DatasetPath = InputSlot() # A TOTAL path (possibly including a dataset name, e.g. myfile.h5/volume/data
    Dirty = InputSlot()
    
    Output = OutputSlot()

    def __init__(self, *args, **kwargs):
        super( OpImageOnDiskProvider, self ).__init__(*args, **kwargs)
        self._opReader = None
        self._opMetadataInjector = None
    
    # Block diagram:
    #
    # (Input.axistags, Input.drange)
    #                               \  
    # DatasetPath ---> opReader ---> opMetadataInjector --> Output
    #                 /
    # WorkingDirectory
    
    def setupOutputs( self ):
        if self._opReader is not None:
            self.Output.disconnect()
            if self._opMetadataInjector:
                self._opMetadataInjector.cleanUp()
                self._opMetadataInjector = None
            self._opReader.cleanUp()
            self._opReader = None

        try:
            # Configure the reader
            dataReady = True
            self._opReader = OpInputDataReader( parent=self )
            self._opReader.WorkingDirectory.setValue( self.WorkingDirectory.value )
            self._opReader.FilePath.setValue( self.DatasetPath.value )

            # Since most file formats don't save meta-info,
            # The reader output's axis order may be incorrect.
            # (For example, if we export in npy format with zxy order, 
            #  the Npy reader op will simply assume xyz order when it reads the data.)

            # Force the metadata back to the correct state by copying select items from Input.meta
            metadata = {}
            metadata['axistags'] = self.Input.meta.axistags
            metadata['drange'] = self.Input.meta.drange
            self._opMetadataInjector = OpMetadataInjector( parent=self )
            self._opMetadataInjector.Input.connect( self._opReader.Output )
            self._opMetadataInjector.Metadata.setValue( metadata )

            dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape
            dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype
            if dataReady:
                self.Output.connect( self._opMetadataInjector.Output )
            else:
                self._opMetadataInjector.cleanUp()
                self._opMetadataInjector = None
                self._opReader.cleanUp()
                self._opReader = None
                self.Output.meta.NOTREADY = True

        except OpInputDataReader.DatasetReadError:
            # Note: If the data is exported as a 'sequence', then this will always be NOTREADY
            #       because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent.
            #       That's okay because a stack is probably too slow to be of use for a preview anyway.
            if self._opMetadataInjector:
                self._opMetadataInjector.cleanUp()
                self._opMetadataInjector = None
            self._opReader.cleanUp()
            self._opReader = None
            # The dataset doesn't exist yet.
            self.Output.meta.NOTREADY = True

    def execute(self, slot, subindex, roi, result):
        assert False, "Output is supposed to be directly connected to an internal operator."

    def propagateDirty(self, slot, subindex, roi):
        if slot == self.Input:
            self.Output.setDirty( roi )
        else:
            self.Output.setDirty( slice(None) )
Exemple #45
0
        def impl():
            shell = self.shell
            workflow = shell.projectManager.workflow
            carvingApplet = workflow.carvingApplet
            gui = carvingApplet.getMultiLaneGui()
            op_carving = carvingApplet.topLevelOperator.getLane(0)

            # activate the carving applet
            shell.setSelectedAppletDrawer(2)
            # let the gui catch up
            QApplication.processEvents()
            self.waitForViews(gui.currentGui().editor.imageViews)
            # inject the labels
            op5 = OpReorderAxes(parent=op_carving.parent)
            opReader = OpInputDataReader(parent=op_carving.parent)
            try:
                opReader.FilePath.setValue(f"{self.reference_files['carving_label_file']}/exported_data")
                op5.AxisOrder.setValue(op_carving.WriteSeeds.meta.getAxisKeys())
                op5.Input.connect(opReader.Output)
                label_data = op5.Output[:].wait()
            finally:
                op5.cleanUp()
                opReader.cleanUp()
            slicing = roi.fullSlicing(label_data.shape)
            op_carving.WriteSeeds[slicing] = label_data

            gui.currentGui().labelingDrawerUi.segment.click()
            QApplication.processEvents()

            op_carving.saveObjectAs("Object 1")
            op_carving.deleteObject("<not saved yet>")

            # export the mesh:
            req = gui.currentGui()._exportMeshes(["Object 1"], [self.output_obj_file])
            req.wait()

            # compare meshes
            with open(self.output_obj_file, "r") as f:
                left = f.read()

            with open(self.reference_files["output_obj_file"], "r") as f:
                right = f.read()

            # TODO: might result in errors due to rounding on different systems
            assert left == right

            # export the completed segments layer
            layermatch = [
                x.name.startswith("Completed segments (unicolor)") for x in gui.currentGui().editor.layerStack
            ]
            assert sum(layermatch) == 1, "Completed segments (unicolor) Layer expected."
            completed_segments_layer = gui.currentGui().editor.layerStack[layermatch.index(True)]
            opExport = get_export_operator(completed_segments_layer)
            try:
                opExport.OutputFilenameFormat.setValue(self.output_file)
                opExport.run_export()
            finally:
                opExport.cleanUp()

            assert os.path.exists(self.output_file)

            # compare completed segments
            with h5py.File(self.reference_files["output_file"], "r") as f_left:
                data_left = f_left["exported_data"][:]

            with h5py.File(self.output_file, "r") as f_right:
                data_right = f_right["exported_data"][:]

            numpy.testing.assert_array_almost_equal(data_left, data_right)

            # Save the project
            saveThread = self.shell.onSaveProjectActionTriggered()
            saveThread.join()