def testBasic_2d(self):
data = 255 * numpy.random.random((50, 100))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("yx"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("png")
opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x20-120_y10-60.png"
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
# Note: vigra inserts a channel axis, so read_data is xyc
assert (read_data[..., 0] == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test_via_OpExportSlot(self):
data = 255 * numpy.random.random( (64, 128, 128, 1) )
data = data.astype( numpy.uint8 )
data = vigra.taggedView( data, vigra.defaultAxistags('zyxc') )
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue( data )
opExport = OpExportSlot(graph=graph)
opExport.Input.connect( opPiper.Output )
opExport.OutputFormat.setValue( 'dvid' )
url = 'http://{hostname}/api/node/{data_uuid}/{data_name}'.format( **self.__dict__ )
opExport.OutputFilenameFormat.setValue( url )
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
try:
opRead.FilePath.setValue( opExport.ExportPath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output( *roiFromShape(data.shape) ).wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic_Hdf5(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
self._tmpdir +
'/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
opExport.OutputInternalPath.setValue('volume/data')
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
export_file = PathComponents(opExport.ExportPath.value).externalPath
assert os.path.split(
export_file)[1] == 'test_export_x10-110_y20-120.h5'
#print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
opRead.cleanUp()
def testBasic_Hdf5(self):
data = numpy.random.random( (100,100) ).astype( numpy.float32 )
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
graph = Graph()
opExport = OpExportSlot(graph=graph)
opExport.Input.setValue(data)
opExport.OutputFormat.setValue( 'hdf5' )
opExport.OutputFilenameFormat.setValue( self._tmpdir + '/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
opExport.OutputInternalPath.setValue('volume/data')
opExport.CoordinateOffset.setValue( (10, 20) )
assert opExport.ExportPath.ready()
export_file = PathComponents( opExport.ExportPath.value ).externalPath
assert os.path.split(export_file)[1] == 'test_export_x10-110_y20-120.h5'
#print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
opRead.FilePath.setValue( opExport.ExportPath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
opRead.cleanUp()
def test_h5_stack_multi_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files"""
shape = (4, 8, 16, 32, 3)
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
for index, t_slice in enumerate(data):
fname = self.testmultiH5FileName.format(index=index)
with h5py.File(fname) as f:
data_group = f.create_group("volume")
data_group.create_dataset("data", data=t_slice)
if sequence_axis == "z":
data = numpy.concatenate(data, axis=0)
elif sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
h5SequenceReader = OpInputDataReader(graph=self.graph)
h5SequenceReader.SequenceAxis.setValue(sequence_axis)
globString = self.testmultiH5FileName.replace("02d}", "s}").format(index="*")
filenamePlusGlob = "{}/volume/data".format(globString)
try:
h5SequenceReader.FilePath.setValue(filenamePlusGlob)
h5data = h5SequenceReader.Output[:].wait()
assert h5data.shape == data.shape
numpy.testing.assert_array_equal(h5data, data)
finally:
# Call cleanUp() to close the file that this operator opened
h5SequenceReader.cleanUp()
def test_npz(self):
# Create two Numpy test arrays
a = numpy.zeros((10, 11))
for x in range(0, 10):
for y in range(0, 11):
a[x, y] = x + y
b = numpy.arange((3 * 9)).reshape((3, 9))
numpy.savez(self.testNpzDataFileName, a=a, b=b)
# Now read back our test data using an OpInputDataReader operator
npyReader = OpInputDataReader(graph=self.graph)
try:
for internalPath, referenceArray in zip(["a", "b"], [a, b]):
npyReader.FilePath.setValue("{}/{}".format(
self.testNpzDataFileName, internalPath))
cwd = os.path.split(__file__)[0]
npyReader.WorkingDirectory.setValue(cwd)
npzData = npyReader.Output[:].wait()
assert npzData.shape == referenceArray.shape
numpy.testing.assert_array_equal(npzData, referenceArray)
finally:
npyReader.cleanUp()
def test_h5(self):
# Create HDF5 test data
with h5py.File(self.testH5FileName) as f:
f.create_group("volume")
shape = (1, 2, 3, 4, 5)
f["volume"].create_dataset(
"data", data=numpy.indices(shape).sum(0).astype(numpy.float32), chunks=True, compression="gzip"
)
# Read the entire HDF5 file and verify the contents
h5Reader = OpInputDataReader(graph=self.graph)
try:
h5Reader.FilePath.setValue(self.testH5FileName + "/volume/data") # Append internal path
cwd = os.path.split(__file__)[0]
h5Reader.WorkingDirectory.setValue(cwd)
# Grab a section of the h5 data
h5Data = h5Reader.Output[0, 0, :, :, :].wait()
assert h5Data.shape == (1, 1, 3, 4, 5)
# (Just check part of the data)
for k in range(0, shape[2]):
for l in range(0, shape[3]):
for m in range(0, shape[4]):
assert h5Data[0, 0, k, l, m] == k + l + m
finally:
# Call cleanUp() to close the file that this operator opened
h5Reader.cleanUp()
assert not h5Reader._file # Whitebox assertion...
def testBasic_Npy(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags("xy"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("numpy")
opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x10-110_y20-120.npy"
# print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test_h5(self):
# Create HDF5 test data
with h5py.File(self.testH5FileName, "w") as f:
f.create_group("volume")
shape = (1, 2, 3, 4, 5)
f["volume"].create_dataset(
"data", data=numpy.indices(shape).sum(0).astype(numpy.float32), chunks=True, compression="gzip"
)
# Read the entire HDF5 file and verify the contents
h5Reader = OpInputDataReader(graph=self.graph)
try:
h5Reader.FilePath.setValue(self.testH5FileName + "/volume/data") # Append internal path
cwd = os.path.split(__file__)[0]
h5Reader.WorkingDirectory.setValue(cwd)
# Grab a section of the h5 data
h5Data = h5Reader.Output[0, 0, :, :, :].wait()
assert h5Data.shape == (1, 1, 3, 4, 5)
# (Just check part of the data)
for k in range(0, shape[2]):
for l in range(0, shape[3]):
for m in range(0, shape[4]):
assert h5Data[0, 0, k, l, m] == k + l + m
finally:
# Call cleanUp() to close the file that this operator opened
h5Reader.cleanUp()
assert not h5Reader._file # Whitebox assertion...
def test_basic(self):
opData = OpArrayCache(graph=self.graph)
opData.blockShape.setValue(self.testData.shape)
opData.Input.setValue(self.testData)
filepath = os.path.join(self._tmpdir, 'multipage.tiff')
logger.debug("writing to: {}".format(filepath))
opExport = OpExportMultipageTiff(graph=self.graph)
opExport.Filepath.setValue(filepath)
opExport.Input.connect(opData.Output)
# Run the export
opExport.run_export()
opReader = OpInputDataReader(graph=self.graph)
opReader.FilePath.setValue(filepath)
# The reader assumes xyzc order.
# We have to transpose the data before we compare.
opReorderAxes = OpReorderAxes(graph=self.graph)
opReorderAxes.AxisOrder.setValue(self._axisorder)
opReorderAxes.Input.connect(opReader.Output)
readData = opReorderAxes.Output[:].wait()
logger.debug("Expected shape={}".format(self.testData.shape))
logger.debug("Read shape={}".format(readData.shape))
assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == self.testData.view(
numpy.ndarray)).all(), "Exported data was not correct"
# Cleanup
opReorderAxes.cleanUp()
opReader.cleanUp()
def test_h5_stack_single_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files for given 'sequence_axis'"""
shape = (4, 8, 16, 32, 3) # assuming axis guess order is 'tzyxc'
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
with h5py.File(self.testH5FileName, "w") as f:
data_group = f.create_group("volumes")
for index, t_slice in enumerate(data):
data_group.create_dataset("timepoint-{index:02d}".format(index=index), data=t_slice)
if sequence_axis == "z":
data = numpy.concatenate(data, axis=0)
elif sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
h5SequenceReader = OpInputDataReader(graph=self.graph)
h5SequenceReader.SequenceAxis.setValue(sequence_axis)
filenamePlusGlob = "{}/volumes/timepoint-*".format(self.testH5FileName)
try:
h5SequenceReader.FilePath.setValue(filenamePlusGlob)
h5data = h5SequenceReader.Output[:].wait()
assert h5data.shape == data.shape, f"{h5data.shape}, {data.shape}"
numpy.testing.assert_array_equal(h5data, data)
finally:
# Call cleanUp() to close the file that this operator opened
h5SequenceReader.cleanUp()
def test_h5_stack_multi_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files"""
shape = (4, 8, 16, 32, 3)
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
for index, t_slice in enumerate(data):
fname = self.testmultiH5FileName.format(index=index)
with h5py.File(fname, "w") as f:
data_group = f.create_group("volume")
data_group.create_dataset("data", data=t_slice)
if sequence_axis == "z":
data = numpy.concatenate(data, axis=0)
elif sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
h5SequenceReader = OpInputDataReader(graph=self.graph)
h5SequenceReader.SequenceAxis.setValue(sequence_axis)
globString = self.testmultiH5FileName.replace("02d}", "s}").format(index="*")
filenamePlusGlob = "{}/volume/data".format(globString)
try:
h5SequenceReader.FilePath.setValue(filenamePlusGlob)
h5data = h5SequenceReader.Output[:].wait()
assert h5data.shape == data.shape
numpy.testing.assert_array_equal(h5data, data)
finally:
# Call cleanUp() to close the file that this operator opened
h5SequenceReader.cleanUp()
def handleImportLabelsAction():
# Find the directory of the most recently opened image file
mostRecentImageFile = PreferencesManager().get(
'DataSelection', 'recent image')
if mostRecentImageFile is not None:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
self, defaultDirectory)
fileNames = map(str, fileNames)
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(
self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DataSelectionGui.getPossibleInternalPaths(
file_path)
if len(internal_paths) == 0:
QMessageBox.critical(
self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = H5VolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue(path_components.totalPath())
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue(
top_op.LabelInputs.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
# Finally, import the labels
top_op.importLabels(top_op.current_view_index(),
op5.Output)
finally:
op5.cleanUp()
opReader.cleanUp()
def testBasic(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opWriter = OpNpyWriter(graph=graph)
opWriter.Input.connect(opPiper.Output)
opWriter.Filepath.setValue(self._tmpdir +
'/npy_writer_test_output.npy')
# Write it...
opWriter.write()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opWriter.Filepath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test_h5_stack_single_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files for given 'sequence_axis'"""
shape = (4, 8, 16, 32, 3) # assuming axis guess order is 'tzyxc'
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
with h5py.File(self.testH5FileName) as f:
data_group = f.create_group("volumes")
for index, t_slice in enumerate(data):
data_group.create_dataset("timepoint-{index:02d}".format(index=index), data=t_slice)
if sequence_axis == "z":
data = numpy.concatenate(data, axis=0)
elif sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
h5SequenceReader = OpInputDataReader(graph=self.graph)
h5SequenceReader.SequenceAxis.setValue(sequence_axis)
filenamePlusGlob = "{}/volumes/timepoint-*".format(self.testH5FileName)
try:
h5SequenceReader.FilePath.setValue(filenamePlusGlob)
h5data = h5SequenceReader.Output[:].wait()
assert h5data.shape == data.shape, f"{h5data.shape}, {data.shape}"
numpy.testing.assert_array_equal(h5data, data)
finally:
# Call cleanUp() to close the file that this operator opened
h5SequenceReader.cleanUp()
def test_via_OpExportSlot(self):
data = 255 * numpy.random.random((64, 128, 128, 1))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("dvid")
url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format(
**self.__dict__)
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output(*roiFromShape(data.shape)).wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test_basic(self):
opData = OpArrayCache( graph=self.graph )
opData.blockShape.setValue( self.testData.shape )
opData.Input.setValue( self.testData )
filepath = os.path.join( self._tmpdir, 'multipage.tiff' )
logger.debug( "writing to: {}".format(filepath) )
opExport = OpExportMultipageTiff(graph=self.graph)
opExport.Filepath.setValue( filepath )
opExport.Input.connect( opData.Output )
# Run the export
opExport.run_export()
opReader = OpInputDataReader( graph=self.graph )
opReader.FilePath.setValue( filepath )
# The reader assumes xyzc order.
# We have to transpose the data before we compare.
opReorderAxes = OpReorderAxes( graph=self.graph )
opReorderAxes.AxisOrder.setValue( self._axisorder )
opReorderAxes.Input.connect( opReader.Output )
readData = opReorderAxes.Output[:].wait()
logger.debug("Expected shape={}".format( self.testData.shape ) )
logger.debug("Read shape={}".format( readData.shape ) )
assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == self.testData.view( numpy.ndarray )).all(), "Exported data was not correct"
# Cleanup
opReorderAxes.cleanUp()
opReader.cleanUp()
def testBasic(self):
graph = Graph()
opExport = OpFormattedDataExport(graph=graph)
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
opExport.Input.setValue(data)
sub_roi = [(10, 0), (None, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.InputMin.setValue(0.0)
opExport.InputMax.setValue(100.0)
opExport.ExportMin.setValue(100)
opExport.ExportMax.setValue(200)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
opExport.OutputInternalPath.setValue('volume/data')
opExport.TransactionSlot.setValue(True)
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
assert opExport.ImageToExport.meta.drange == (100, 200)
#print "exporting data to: {}".format( opExport.ExportPath.value )
assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
# Compare with the correct subregion and convert dtype.
sub_roi[1] = (100, 80) # Replace 'None' with full extent
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
expected_data += 100 # see renormalization settings
assert opRead.Output.meta.shape == expected_data.shape
assert opRead.Output.meta.dtype == expected_data.dtype
read_data = opRead.Output[:].wait()
# Due to rounding errors, the actual result and the expected result may differ by 1
# e.g. if the original pixel value was 32.99999999
# Also, must promote to signed values to avoid unsigned rollover
# See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
expected_data_signed = expected_data.astype(numpy.int16)
read_data_signed = expected_data.astype(numpy.int16)
difference_from_expected = expected_data_signed - read_data_signed
assert (numpy.abs(difference_from_expected) <=
1).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic(self):
graph = Graph()
opExport = OpDataExport(graph=graph)
try:
opExport.TransactionSlot.setValue(True)
opExport.WorkingDirectory.setValue(self._tmpdir)
# Simulate the important fields of a DatasetInfo object
class MockDatasetInfo(object):
pass
rawInfo = MockDatasetInfo()
rawInfo.nickname = 'test_nickname'
rawInfo.filePath = './somefile.h5'
opExport.RawDatasetInfo.setValue(rawInfo)
opExport.SelectionNames.setValue(['Mock Export Data'])
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
opExport.Inputs.resize(1)
opExport.Inputs[0].setValue(data)
sub_roi = [(10, 20), (90, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
'{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}'
)
opExport.OutputInternalPath.setValue('volume/data')
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
expected_path = self._tmpdir + '/' + rawInfo.nickname + '_export_x10-90_y20-80.h5/volume/data'
computed_path = opExport.ExportPath.value
assert os.path.normpath(computed_path) == os.path.normpath(expected_path), \
"Expected {}\nGot: {}".format( expected_path, computed_path )
opExport.run_export()
finally:
opExport.cleanUp()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(computed_path)
# Compare with the correct subregion and convert dtype.
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic(self, tmp_h5_single_dataset: Path):
graph = Graph()
opExport = OpDataExport(graph=graph)
try:
opExport.TransactionSlot.setValue(True)
opExport.WorkingDirectory.setValue(self._tmpdir)
rawInfo = FilesystemDatasetInfo(filePath=str(
tmp_h5_single_dataset / "test_group/test_data"),
nickname="test_nickname")
opExport.RawDatasetInfo.setValue(rawInfo)
opExport.SelectionNames.setValue(["Mock Export Data"])
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags("xy"))
opExport.Inputs.resize(1)
opExport.Inputs[0].setValue(data)
sub_roi = [(10, 20), (90, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.OutputFormat.setValue("hdf5")
opExport.OutputFilenameFormat.setValue(
"{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}"
)
opExport.OutputInternalPath.setValue("volume/data")
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
expected_path = tmp_h5_single_dataset.parent.joinpath(
rawInfo.nickname +
"_export_x10-90_y20-80.h5/volume/data").as_posix()
computed_path = opExport.ExportPath.value
assert os.path.normpath(computed_path) == os.path.normpath(
expected_path)
opExport.run_export()
finally:
opExport.cleanUp()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(computed_path)
# Compare with the correct subregion and convert dtype.
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic(self):
graph = Graph()
opExport = OpFormattedDataExport(graph=graph)
data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
opExport.Input.setValue(data)
sub_roi = [(10, 0), (None, 80)]
opExport.RegionStart.setValue( sub_roi[0] )
opExport.RegionStop.setValue( sub_roi[1] )
opExport.ExportDtype.setValue( numpy.uint8 )
opExport.InputMin.setValue( 0.0 )
opExport.InputMax.setValue( 100.0 )
opExport.ExportMin.setValue( 100 )
opExport.ExportMax.setValue( 200 )
opExport.OutputFormat.setValue( 'hdf5' )
opExport.OutputFilenameFormat.setValue( self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
opExport.OutputInternalPath.setValue('volume/data')
opExport.TransactionSlot.setValue( True )
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
assert opExport.ImageToExport.meta.drange == (100,200)
#print "exporting data to: {}".format( opExport.ExportPath.value )
assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
opExport.run_export()
opRead = OpInputDataReader( graph=graph )
try:
opRead.FilePath.setValue( opExport.ExportPath.value )
# Compare with the correct subregion and convert dtype.
sub_roi[1] = (100, 80) # Replace 'None' with full extent
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
expected_data += 100 # see renormalization settings
assert opRead.Output.meta.shape == expected_data.shape
assert opRead.Output.meta.dtype == expected_data.dtype
read_data = opRead.Output[:].wait()
# Due to rounding errors, the actual result and the expected result may differ by 1
# e.g. if the original pixel value was 32.99999999
# Also, must promote to signed values to avoid unsigned rollover
# See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
expected_data_signed = expected_data.astype(numpy.int16)
read_data_signed = expected_data.astype(numpy.int16)
difference_from_expected = expected_data_signed - read_data_signed
assert (numpy.abs(difference_from_expected) <= 1).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic(self):
graph = Graph()
opExport = OpDataExport(graph=graph)
try:
opExport.TransactionSlot.setValue(True)
opExport.WorkingDirectory.setValue( self._tmpdir )
# Simulate the important fields of a DatasetInfo object
class MockDatasetInfo(object): pass
rawInfo = MockDatasetInfo()
rawInfo.nickname = 'test_nickname'
rawInfo.filePath = './somefile.h5'
opExport.RawDatasetInfo.setValue( rawInfo )
opExport.SelectionNames.setValue(['Mock Export Data'])
data = numpy.random.random( (100,100) ).astype( numpy.float32 ) * 100
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
opExport.Inputs.resize(1)
opExport.Inputs[0].setValue(data)
sub_roi = [(10, 20), (90, 80)]
opExport.RegionStart.setValue( sub_roi[0] )
opExport.RegionStop.setValue( sub_roi[1] )
opExport.ExportDtype.setValue( numpy.uint8 )
opExport.OutputFormat.setValue( 'hdf5' )
opExport.OutputFilenameFormat.setValue( '{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}' )
opExport.OutputInternalPath.setValue('volume/data')
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
expected_path = self._tmpdir + '/' + rawInfo.nickname + '_export_x10-90_y20-80.h5/volume/data'
computed_path = opExport.ExportPath.value
assert os.path.normpath(computed_path) == os.path.normpath(expected_path), \
"Expected {}\nGot: {}".format( expected_path, computed_path )
opExport.run_export()
finally:
opExport.cleanUp()
opRead = OpInputDataReader( graph=graph )
try:
opRead.FilePath.setValue( computed_path )
# Compare with the correct subregion and convert dtype.
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def handleImportLabelsAction():
# Find the directory of the most recently opened image file
mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
if mostRecentImageFile is not None:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(self, defaultDirectory)
fileNames = list(map(str, fileNames))
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DataSelectionGui.getPossibleInternalPaths(file_path)
if len(internal_paths) == 0:
QMessageBox.critical(self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = H5VolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue( path_components.totalPath() )
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue( top_op.LabelInputs.meta.getAxisKeys() )
op5.Input.connect( opReader.Output )
# Finally, import the labels
top_op.importLabels( top_op.current_view_index(), op5.Output )
finally:
op5.cleanUp()
opReader.cleanUp()
def handleImportLabelsAction():
fileNames = ImageFileDialog(
self,
preferences_group="DataSelection",
preferences_setting="recent image").getSelectedPaths()
fileNames = list(map(str, fileNames))
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(
self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DatasetInfo.getPossibleInternalPathsFor(file_path)
if len(internal_paths) == 0:
QMessageBox.critical(
self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = SubvolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue(path_components.totalPath())
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue(top_op.LabelInputs.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
# Finally, import the labels
top_op.importLabels(top_op.current_view_index(), op5.Output)
finally:
op5.cleanUp()
opReader.cleanUp()
def test_npy_with_roi(self):
a = numpy.indices((100, 100, 200)).astype(numpy.uint8).sum(0)
assert a.shape == (100, 100, 200)
numpy.save(self.testNpyDataFileName, a)
opReader = OpInputDataReader(graph=lazyflow.graph.Graph())
try:
opReader.FilePath.setValue(self.testNpyDataFileName)
opReader.SubVolumeRoi.setValue(((10, 20, 30), (50, 70, 90)))
all_data = opReader.Output[:].wait()
assert all_data.shape == (40, 50, 60)
assert (all_data == a[10:50, 20:70, 30:90]).all()
finally:
opReader.cleanUp()
def test_npy_with_roi(self):
a = numpy.indices((100, 100, 200)).astype(numpy.uint8).sum(0)
assert a.shape == (100, 100, 200)
numpy.save(self.testNpyDataFileName, a)
opReader = OpInputDataReader(graph=lazyflow.graph.Graph())
try:
opReader.FilePath.setValue(self.testNpyDataFileName)
opReader.SubVolumeRoi.setValue(((10, 20, 30), (50, 70, 90)))
all_data = opReader.Output[:].wait()
assert all_data.shape == (40, 50, 60)
assert (all_data == a[10:50, 20:70, 30:90]).all()
finally:
opReader.cleanUp()
def testBasic_Dvid(self):
if _skip_dvid:
raise nose.SkipTest
# Spin up a mock dvid server to test with.
dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
with H5MockServerDataFile(mockserver_data_file) as test_h5file:
test_h5file.add_node(dvid_dataset, data_uuid)
server_proc, shutdown_event = H5MockServer.create_and_start(
mockserver_data_file,
"localhost",
8000,
same_process=False,
disable_server_logging=True)
try:
data = 255 * numpy.random.random((100, 100, 4))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags('xyc'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue('dvid')
url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format(
**locals())
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
try:
opRead = OpInputDataReader(graph=graph)
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
finally:
shutdown_event.set()
server_proc.join()
def testBasic_Dvid(self):
if _skip_dvid:
raise nose.SkipTest
# Spin up a mock dvid server to test with.
dvid_dataset, data_uuid, data_name = "datasetA", "abcde", "indices_data"
mockserver_data_file = self._tmpdir + '/mockserver_data.h5'
with H5MockServerDataFile( mockserver_data_file ) as test_h5file:
test_h5file.add_node( dvid_dataset, data_uuid )
server_proc, shutdown_event = H5MockServer.create_and_start( mockserver_data_file, "localhost", 8000,
same_process=False, disable_server_logging=True )
try:
data = 255 * numpy.random.random( (100,100, 4) )
data = data.astype( numpy.uint8 )
data = vigra.taggedView( data, vigra.defaultAxistags('xyc') )
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue( data )
opExport = OpExportSlot(graph=graph)
opExport.Input.connect( opPiper.Output )
opExport.OutputFormat.setValue( 'dvid' )
url = 'http://localhost:8000/api/node/{data_uuid}/{data_name}'.format( **locals() )
opExport.OutputFilenameFormat.setValue( url )
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
try:
opRead = OpInputDataReader( graph=graph )
opRead.FilePath.setValue( opExport.ExportPath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
finally:
shutdown_event.set()
server_proc.join()
def __init__(
self,
*,
filePath: str,
project_file: h5py.File = None,
sequence_axis: str = None,
nickname: str = "",
drange: Tuple[Number, Number] = None,
**info_kwargs,
):
"""
sequence_axis: Axis along which to stack (only applicable for stacks).
"""
self.sequence_axis = sequence_axis
self.base_dir = str(Path(project_file.filename).absolute().parent
) if project_file else os.getcwd()
assert os.path.isabs(
self.base_dir
) # FIXME: if file_project was opened as a relative path, this would break
self.expanded_paths = self.expand_path(filePath, cwd=self.base_dir)
assert len(self.expanded_paths) == 1 or self.sequence_axis
if len({PathComponents(ep).extension
for ep in self.expanded_paths}) > 1:
raise Exception(
f"Multiple extensions unsupported as a single data source: {self.expanded_paths}"
)
self.filePath = os.path.pathsep.join(self.expanded_paths)
op_reader = OpInputDataReader(graph=Graph(),
WorkingDirectory=self.base_dir,
FilePath=self.filePath,
SequenceAxis=self.sequence_axis)
meta = op_reader.Output.meta.copy()
op_reader.cleanUp()
super().__init__(
default_tags=meta.axistags,
nickname=nickname or self.create_nickname(self.expanded_paths),
laneShape=meta.shape,
laneDtype=meta.dtype,
drange=drange or meta.get("drange"),
**info_kwargs,
)
def testBasic(self):
data = numpy.random.random( (100,100) ).astype( numpy.float32 )
data = vigra.taggedView( data, vigra.defaultAxistags('xy') )
graph = Graph()
opWriter = OpNpyWriter(graph=graph)
opWriter.Input.setValue(data)
opWriter.Filepath.setValue( self._tmpdir + '/npy_writer_test_output.npy' )
# Write it...
opWriter.write()
opRead = OpInputDataReader( graph=graph )
opRead.FilePath.setValue( opWriter.Filepath.value )
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
opRead.cleanUp()
def importStackAsLocalDataset(self,
abs_paths: List[str],
sequence_axis: str = "z",
progress_signal: Callable[[int],
None] = None):
progress_signal = progress_signal or self.progressSignal
progress_signal(0)
op_reader = None
op_writer = None
try:
colon_paths = os.path.pathsep.join(abs_paths)
op_reader = OpInputDataReader(graph=self.topLevelOperator.graph,
FilePath=colon_paths,
SequenceAxis=sequence_axis)
axistags = op_reader.Output.meta.axistags
inner_path = self.local_data_path.joinpath(
DatasetInfo.generate_id()).as_posix()
project_file = self.topLevelOperator.ProjectFile.value
op_writer = OpH5N5WriterBigDataset(
graph=self.topLevelOperator.graph,
h5N5File=project_file,
h5N5Path=inner_path,
CompressionEnabled=False,
BatchSize=1,
Image=op_reader.Output,
)
op_writer.progressSignal.subscribe(progress_signal)
success = op_writer.WriteImage.value
for index, tag in enumerate(axistags):
project_file[inner_path].dims[index].label = tag.key
project_file[inner_path].attrs["axistags"] = axistags.toJSON()
if op_reader.Output.meta.get("drange"):
project_file[inner_path].attrs[
"drange"] = op_reader.Output.meta.get("drange")
return inner_path
finally:
if op_writer:
op_writer.Image.disconnect()
if op_reader:
op_reader.cleanUp()
progress_signal(100)
def test_tiff_stack_multi_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files"""
shape = (4, 8, 16, 3)
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
for idx, data_slice in enumerate(data):
im = Image.fromarray(data_slice, mode="RGB")
im.save(self.testmultiTiffFileName.format(index=idx))
if sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
reader = OpInputDataReader(graph=self.graph)
reader.SequenceAxis.setValue(sequence_axis)
globString = self.testmultiTiffFileName.replace("02d}", "s}").format(index="*")
try:
reader.FilePath.setValue(globString)
tiffdata = reader.Output[:].wait()
assert tiffdata.shape == data.shape, f"{tiffdata.shape}, {data.shape}"
numpy.testing.assert_array_equal(tiffdata, data)
finally:
# Call cleanUp() to close the file that this operator opened
reader.cleanUp()
def test_npy(self):
# Create Numpy test data
a = numpy.zeros((10, 11))
for x in range(0, 10):
for y in range(0, 11):
a[x, y] = x + y
numpy.save(self.testNpyDataFileName, a)
# Now read back our test data using an OpInputDataReader operator
npyReader = OpInputDataReader(graph=self.graph)
try:
npyReader.FilePath.setValue(self.testNpyDataFileName)
cwd = os.path.split(__file__)[0]
npyReader.WorkingDirectory.setValue(cwd)
# Read the entire NPY file and verify the contents
npyData = npyReader.Output[:].wait()
assert npyData.shape == (10, 11)
for x in range(0, 10):
for y in range(0, 11):
assert npyData[x, y] == x + y
finally:
npyReader.cleanUp()
def test_tiff_stack_multi_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files"""
shape = (4, 8, 16, 3)
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
for idx, data_slice in enumerate(data):
im = Image.fromarray(data_slice, mode="RGB")
im.save(self.testmultiTiffFileName.format(index=idx))
if sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
reader = OpInputDataReader(graph=self.graph)
reader.SequenceAxis.setValue(sequence_axis)
globString = self.testmultiTiffFileName.replace("02d}", "s}").format(index="*")
try:
reader.FilePath.setValue(globString)
tiffdata = reader.Output[:].wait()
assert tiffdata.shape == data.shape, f"{tiffdata.shape}, {data.shape}"
numpy.testing.assert_array_equal(tiffdata, data)
finally:
# Call cleanUp() to close the file that this operator opened
reader.cleanUp()
def test_npy(self):
# Create Numpy test data
a = numpy.zeros((10, 11))
for x in range(0, 10):
for y in range(0, 11):
a[x, y] = x + y
numpy.save(self.testNpyDataFileName, a)
# Now read back our test data using an OpInputDataReader operator
npyReader = OpInputDataReader(graph=self.graph)
try:
npyReader.FilePath.setValue(self.testNpyDataFileName)
cwd = os.path.split(__file__)[0]
npyReader.WorkingDirectory.setValue(cwd)
# Read the entire NPY file and verify the contents
npyData = npyReader.Output[:].wait()
assert npyData.shape == (10, 11)
for x in range(0, 10):
for y in range(0, 11):
assert npyData[x, y] == x + y
finally:
npyReader.cleanUp()
def test_npz(self):
# Create two Numpy test arrays
a = numpy.zeros((10, 11))
for x in range(0, 10):
for y in range(0, 11):
a[x, y] = x + y
b = numpy.arange((3 * 9)).reshape((3, 9))
numpy.savez(self.testNpzDataFileName, a=a, b=b)
# Now read back our test data using an OpInputDataReader operator
npyReader = OpInputDataReader(graph=self.graph)
try:
for internalPath, referenceArray in zip(["a", "b"], [a, b]):
npyReader.FilePath.setValue("{}/{}".format(self.testNpzDataFileName, internalPath))
cwd = os.path.split(__file__)[0]
npyReader.WorkingDirectory.setValue(cwd)
npzData = npyReader.Output[:].wait()
assert npzData.shape == referenceArray.shape
numpy.testing.assert_array_equal(npzData, referenceArray)
finally:
npyReader.cleanUp()
class OpImageOnDiskProvider(Operator):
"""
This simply wraps a lazyflow OpInputDataReader, but ensures that the metadata
(axistags, drange) on the output matches the metadata from the original data
(even if the output file format doesn't support metadata fields).
"""
TransactionSlot = InputSlot()
Input = InputSlot() # Used for dtype and shape only. Data is always provided directly from the file.
WorkingDirectory = InputSlot()
DatasetPath = InputSlot() # A TOTAL path (possibly including a dataset name, e.g. myfile.h5/volume/data
Dirty = InputSlot()
Output = OutputSlot()
def __init__(self, *args, **kwargs):
super( OpImageOnDiskProvider, self ).__init__(*args, **kwargs)
self._opReader = None
self._opMetadataInjector = None
# Block diagram:
#
# (Input.axistags, Input.drange)
# \
# DatasetPath ---> opReader ---> opMetadataInjector --> Output
# /
# WorkingDirectory
def setupOutputs( self ):
if self._opReader is not None:
self.Output.disconnect()
if self._opMetadataInjector:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
try:
# Configure the reader
dataReady = True
self._opReader = OpInputDataReader( parent=self )
self._opReader.WorkingDirectory.setValue( self.WorkingDirectory.value )
self._opReader.FilePath.setValue( self.DatasetPath.value )
# Since most file formats don't save meta-info,
# The reader output's axis order may be incorrect.
# (For example, if we export in npy format with zxy order,
# the Npy reader op will simply assume xyz order when it reads the data.)
# Force the metadata back to the correct state by copying select items from Input.meta
metadata = {}
metadata['axistags'] = self.Input.meta.axistags
metadata['drange'] = self.Input.meta.drange
metadata['display_mode'] = self.Input.meta.display_mode
self._opMetadataInjector = OpMetadataInjector( parent=self )
self._opMetadataInjector.Input.connect( self._opReader.Output )
self._opMetadataInjector.Metadata.setValue( metadata )
dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape
dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype
if dataReady:
self.Output.connect( self._opMetadataInjector.Output )
else:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
self.Output.meta.NOTREADY = True
#except OpInputDataReader.DatasetReadError:
except Exception as ex:
#logger.debug( "On-disk image can't be read: {}".format(ex) )
# Note: If the data is exported as a 'sequence', then this will always be NOTREADY
# because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent.
# That's okay because a stack is probably too slow to be of use for a preview anyway.
if self._opMetadataInjector:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
# The dataset doesn't exist yet.
self.Output.meta.NOTREADY = True
def execute(self, slot, subindex, roi, result):
assert False, "Output is supposed to be directly connected to an internal operator."
def propagateDirty(self, slot, subindex, roi):
if slot == self.Input:
self.Output.setDirty( roi )
else:
self.Output.setDirty( slice(None) )
def generate_trained_project_file(
new_project_path, raw_data_paths, label_data_paths, feature_selections, classifier_factory=None
):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(label_data_paths), "Number of label images must match number of raw images."
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parser.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = ilastik_main.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args([], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue(classifier_factory)
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue(cwd)
opReader.FilePath.setValue(label_data_path)
print "Reading label volume: {}".format(label_data_path)
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape) - 1
label_volume = label_volume[..., None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print "Applying label volume to lane #{}".format(lane)
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = map(str, range(max_label_class))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(
writeSeeds,
lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator
), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling',
'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell',
'recently opened')
mostRecentImageFile = PreferencesManager().get('DataSelection',
'recent image')
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
parent_widget, defaultDirectory)
fileNames = map(str, fileNames)
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader(parent=opLabels.parent)
opCache = OpArrayCache(parent=opLabels.parent)
opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
opReorderAxes = OpReorderAxes(parent=opLabels.parent)
# Set up the pipeline as follows:
#
# opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
1 else os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
opCache.blockShape.setValue(opImport.Output.meta.shape)
opCache.Input.connect(opImport.Output)
assert opCache.Output.ready()
opMetadataInjector.Input.connect(opCache.Output)
metadata = opCache.Output.meta.copy()
opMetadataInjector.Metadata.setValue(metadata)
opReorderAxes.Input.connect(opMetadataInjector.Output)
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Importing Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = opCache.Output[:]
req.notify_finished(close_busy_dlg)
req.notify_failed(close_busy_dlg)
req.submit()
busy_dlg.exec_()
readData = req.result
maxLabels = len(labelingSlots.labelNames.value)
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
unique_read_labels = numpy.unique(readData)
readLabelCounts = numpy.bincount(readData.flat)[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
opMetadataInjector.Output,
labelingSlots.labelInput,
labelInfo)
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue(metadata)
settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets.
# Offsets in dlg only include the file axes, not the 5D axes expected by the label input,
# so expand them to full 5D
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict(
zip(axes_5d, [0] * len(axes_5d)))
tagged_offsets.update(
dict(
zip(opMetadataInjector.Output.meta.getAxisKeys(),
settingsDlg.imageOffsets)))
imageOffsets = tagged_offsets.values()
# Optimization if mapping is identity
if labelMapping.keys() == labelMapping.values():
labelMapping = None
# This will be fast (it's already cached)
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels,
label_data)
new_labels = numpy.array(
[labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
def impl():
shell = self.shell
workflow = shell.projectManager.workflow
carvingApplet = workflow.carvingApplet
gui = carvingApplet.getMultiLaneGui()
op_carving = carvingApplet.topLevelOperator.getLane(0)
# activate the carving applet
shell.setSelectedAppletDrawer(2)
# let the gui catch up
QApplication.processEvents()
self.waitForViews(gui.currentGui().editor.imageViews)
# inject the labels
op5 = OpReorderAxes(parent=op_carving.parent)
opReader = OpInputDataReader(parent=op_carving.parent)
try:
opReader.FilePath.setValue(f"{self.reference_files['carving_label_file']}/exported_data")
op5.AxisOrder.setValue(op_carving.WriteSeeds.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
label_data = op5.Output[:].wait()
finally:
op5.cleanUp()
opReader.cleanUp()
slicing = roi.fullSlicing(label_data.shape)
op_carving.WriteSeeds[slicing] = label_data
gui.currentGui().labelingDrawerUi.segment.click()
QApplication.processEvents()
op_carving.saveObjectAs("Object 1")
op_carving.deleteObject("<not saved yet>")
# export the mesh:
req = gui.currentGui()._exportMeshes(["Object 1"], [self.output_obj_file])
req.wait()
# compare meshes
with open(self.output_obj_file, "r") as f:
left = f.read()
with open(self.reference_files["output_obj_file"], "r") as f:
right = f.read()
# TODO: might result in errors due to rounding on different systems
assert left == right
# export the completed segments layer
layermatch = [
x.name.startswith("Completed segments (unicolor)") for x in gui.currentGui().editor.layerStack
]
assert sum(layermatch) == 1, "Completed segments (unicolor) Layer expected."
completed_segments_layer = gui.currentGui().editor.layerStack[layermatch.index(True)]
opExport = get_export_operator(completed_segments_layer)
try:
opExport.OutputFilenameFormat.setValue(self.output_file)
opExport.run_export()
finally:
opExport.cleanUp()
assert os.path.exists(self.output_file)
# compare completed segments
with h5py.File(self.reference_files["output_file"], "r") as f_left:
data_left = f_left["exported_data"][:]
with h5py.File(self.output_file, "r") as f_right:
data_right = f_right["exported_data"][:]
numpy.testing.assert_array_almost_equal(data_left, data_right)
# Save the project
saveThread = self.shell.onSaveProjectActionTriggered()
saveThread.join()
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(writeSeeds, lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling', 'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell', 'recently opened')
mostRecentImageFile = PreferencesManager().get( 'DataSelection', 'recent image' )
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(parent_widget, defaultDirectory)
fileNames = map(str, fileNames)
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader( parent=opLabels.parent )
opCache = OpArrayCache( parent=opLabels.parent )
opMetadataInjector = OpMetadataInjector( parent=opLabels.parent )
opReorderAxes = OpReorderAxes( parent=opLabels.parent )
# Set up the pipeline as follows:
#
# opImport --> opCache --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) == 1 else
os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
opCache.blockShape.setValue( opImport.Output.meta.shape )
opCache.Input.connect( opImport.Output )
assert opCache.Output.ready()
opMetadataInjector.Input.connect( opCache.Output )
metadata = opCache.Output.meta.copy()
opMetadataInjector.Metadata.setValue( metadata )
opReorderAxes.Input.connect( opMetadataInjector.Output )
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue( writeSeeds.meta.getAxisKeys() )
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Importing Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = opCache.Output[:]
req.notify_finished( close_busy_dlg )
req.notify_failed( close_busy_dlg )
req.submit()
busy_dlg.exec_()
readData = req.result
maxLabels = len(labelingSlots.labelNames.value)
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
unique_read_labels = numpy.unique(readData)
readLabelCounts = vigra_bincount(readData)[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg( parent_widget,
fileNames, opMetadataInjector.Output,
labelingSlots.labelInput, labelInfo )
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue( metadata )
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet("QLineEdit { background: rgb(255, 128, 128);"
"selection-background-color: rgb(128, 128, 255); }")
settingsDlg.axesEdit.editingFinished.connect( handle_updated_axes )
# Initialize
handle_updated_axes()
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets.
# Offsets in dlg only include the file axes, not the 5D axes expected by the label input,
# so expand them to full 5D
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict( zip( axes_5d, [0]*len(axes_5d) ) )
tagged_offsets.update( dict( zip( opMetadataInjector.Output.meta.getAxisKeys(), settingsDlg.imageOffsets ) ) )
imageOffsets = tagged_offsets.values()
# Optimization if mapping is identity
if labelMapping.keys() == labelMapping.values():
labelMapping = None
# This will be fast (it's already cached)
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels, label_data)
new_labels = numpy.array([labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array( roiFromShape(opReorderAxes.Output.meta.shape) )
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
def import_labeling_layer(labelLayer, labelingSlots, parent_widget=None):
"""
Prompt the user for layer import settings, and perform the layer import.
:param labelLayer: The top label layer source
:param labelingSlots: An instance of LabelingGui.LabelingSlots
:param parent_widget: The Qt GUI parent object
"""
writeSeeds = labelingSlots.labelInput
assert isinstance(
writeSeeds,
lazyflow.graph.Slot), "slot is of type %r" % (type(writeSeeds))
opLabels = writeSeeds.getRealOperator()
assert isinstance(opLabels, lazyflow.graph.Operator
), "slot's operator is of type %r" % (type(opLabels))
recentlyImported = PreferencesManager().get('labeling',
'recently imported')
mostRecentProjectPath = PreferencesManager().get('shell',
'recently opened')
mostRecentImageFile = PreferencesManager().get('DataSelection',
'recent image')
if recentlyImported:
defaultDirectory = os.path.split(recentlyImported)[0]
elif mostRecentProjectPath:
defaultDirectory = os.path.split(mostRecentProjectPath)[0]
elif mostRecentImageFile:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
parent_widget, defaultDirectory)
fileNames = map(str, fileNames)
if not fileNames:
return
PreferencesManager().set('labeling', 'recently imported', fileNames[0])
try:
# Initialize operators
opImport = OpInputDataReader(parent=opLabels.parent)
opCache = OpBlockedArrayCache(parent=opLabels.parent)
opMetadataInjector = OpMetadataInjector(parent=opLabels.parent)
opReorderAxes = OpReorderAxes(parent=opLabels.parent)
# Set up the pipeline as follows:
#
# opImport --> (opCache) --> opMetadataInjector --------> opReorderAxes --(inject via setInSlot)--> labelInput
# / /
# User-specified axisorder labelInput.meta.axistags
opImport.WorkingDirectory.setValue(defaultDirectory)
opImport.FilePath.setValue(fileNames[0] if len(fileNames) ==
1 else os.path.pathsep.join(fileNames))
assert opImport.Output.ready()
maxLabels = len(labelingSlots.labelNames.value)
# We don't bother with counting the label pixels
# (and caching the data) if it's big (1 GB)
if numpy.prod(opImport.Output.meta.shape) > 1e9:
reading_slot = opImport.Output
# For huge data, we don't go through and search for the pixel values,
# because that takes an annoyingly long amount of time.
# Instead, we make the reasonable assumption that the input labels are already 1,2,3..N
# and we don't tell the user what the label pixel counts are.
unique_read_labels = numpy.array(range(maxLabels + 1))
readLabelCounts = numpy.array([-1] * (maxLabels + 1))
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
else:
opCache.Input.connect(opImport.Output)
opCache.CompressionEnabled.setValue(True)
assert opCache.Output.ready()
reading_slot = opCache.Output
# We'll show a little window with a busy indicator while the data is loading
busy_dlg = QProgressDialog(parent=parent_widget)
busy_dlg.setLabelText("Scanning Label Data...")
busy_dlg.setCancelButton(None)
busy_dlg.setMinimum(100)
busy_dlg.setMaximum(100)
def close_busy_dlg(*args):
QApplication.postEvent(busy_dlg, QCloseEvent())
# Load the data from file into our cache
# When it's done loading, close the progress dialog.
req = reading_slot[:]
req.notify_finished(close_busy_dlg)
req.notify_failed(close_busy_dlg)
req.submit()
busy_dlg.exec_()
readData = req.result
# Can't use return_counts feature because that requires numpy >= 1.9
#unique_read_labels, readLabelCounts = numpy.unique(readData, return_counts=True)
# This does the same as the above, albeit slower, and probably with more ram.
bincounts = chunked_bincount(readData)
unique_read_labels = bincounts.nonzero()[0].astype(readData.dtype,
copy=False)
readLabelCounts = bincounts[unique_read_labels]
labelInfo = (maxLabels, (unique_read_labels, readLabelCounts))
del readData
opMetadataInjector.Input.connect(reading_slot)
metadata = reading_slot.meta.copy()
opMetadataInjector.Metadata.setValue(metadata)
opReorderAxes.Input.connect(opMetadataInjector.Output)
# Transpose the axes for assignment to the labeling operator.
opReorderAxes.AxisOrder.setValue(writeSeeds.meta.getAxisKeys())
# Ask the user how to interpret the data.
settingsDlg = LabelImportOptionsDlg(parent_widget, fileNames,
opMetadataInjector.Output,
labelingSlots.labelInput,
labelInfo)
def handle_updated_axes():
# The user is specifying a new interpretation of the file's axes
updated_axisorder = str(settingsDlg.axesEdit.text())
metadata = opMetadataInjector.Metadata.value.copy()
metadata.axistags = vigra.defaultAxistags(updated_axisorder)
opMetadataInjector.Metadata.setValue(metadata)
if opReorderAxes._invalid_axes:
settingsDlg.buttonBox.button(
QDialogButtonBox.Ok).setEnabled(False)
# Red background
settingsDlg.axesEdit.setStyleSheet(
"QLineEdit { background: rgb(255, 128, 128);"
"selection-background-color: rgb(128, 128, 255); }")
settingsDlg.axesEdit.editingFinished.connect(handle_updated_axes)
# Initialize
handle_updated_axes()
dlg_result = settingsDlg.exec_()
if dlg_result != LabelImportOptionsDlg.Accepted:
return
# Get user's chosen label mapping from dlg
labelMapping = settingsDlg.labelMapping
# Get user's chosen offsets, ordered by the 'write seeds' slot
axes_5d = opReorderAxes.Output.meta.getAxisKeys()
tagged_offsets = collections.OrderedDict(
zip(axes_5d, [0] * len(axes_5d)))
tagged_offsets.update(
dict(
zip(opReorderAxes.Output.meta.getAxisKeys(),
settingsDlg.imageOffsets)))
imageOffsets = tagged_offsets.values()
# Optimization if mapping is identity
if labelMapping.keys() == labelMapping.values():
labelMapping = None
# If the data was already cached, this will be fast.
label_data = opReorderAxes.Output[:].wait()
# Map input labels to output labels
if labelMapping:
# There are other ways to do a relabeling (e.g skimage.segmentation.relabel_sequential)
# But this supports potentially huge values of unique_read_labels (in the billions),
# without needing GB of RAM.
mapping_indexes = numpy.searchsorted(unique_read_labels,
label_data)
new_labels = numpy.array(
[labelMapping[x] for x in unique_read_labels])
label_data[:] = new_labels[mapping_indexes]
label_roi = numpy.array(roiFromShape(opReorderAxes.Output.meta.shape))
label_roi += imageOffsets
label_slice = roiToSlice(*label_roi)
writeSeeds[label_slice] = label_data
finally:
opReorderAxes.cleanUp()
opMetadataInjector.cleanUp()
opCache.cleanUp()
opImport.cleanUp()
def generate_trained_project_file(new_project_path,
raw_data_paths,
label_data_paths,
feature_selections,
classifier_factory=None):
"""
Create a new project file from scratch, add the given raw data files,
inject the corresponding labels, configure the given feature selections,
and (if provided) override the classifier type ('factory').
Finally, request the classifier object from the pipeline (which forces training),
and save the project.
new_project_path: Where to save the new project file
raw_data_paths: A list of paths to the raw data images to train with
label_data_paths: A list of paths to the label image data to train with
feature_selections: A matrix of bool, representing the selected features
classifier_factory: Override the classifier type. Must be a subclass of either:
- lazyflow.classifiers.LazyflowVectorwiseClassifierFactoryABC
- lazyflow.classifiers.LazyflowPixelwiseClassifierFactoryABC
"""
assert len(raw_data_paths) == len(
label_data_paths
), "Number of label images must match number of raw images."
import ilastik_main
from ilastik.workflows.pixelClassification import PixelClassificationWorkflow
from lazyflow.graph import Graph
from lazyflow.operators.ioOperators import OpInputDataReader
from lazyflow.roi import roiToSlice, roiFromShape
##
## CREATE PROJECT
##
# Manually configure the arguments to ilastik, as if they were parsed from the command line.
# (Start with empty args and fill in below.)
ilastik_args = ilastik_main.parse_args([])
ilastik_args.new_project = new_project_path
ilastik_args.headless = True
ilastik_args.workflow = "Pixel Classification"
shell = ilastik_main.main(ilastik_args)
assert isinstance(shell.workflow, PixelClassificationWorkflow)
##
## CONFIGURE GRAYSCALE INPUT
##
data_selection_applet = shell.workflow.dataSelectionApplet
# To configure data selection, start with empty cmdline args and manually fill them in
data_selection_args, _ = data_selection_applet.parse_known_cmdline_args(
[], PixelClassificationWorkflow.ROLE_NAMES)
data_selection_args.raw_data = raw_data_paths
data_selection_args.preconvert_stacks = True
# Simplest thing here is to configure using cmd-line interface
data_selection_applet.configure_operator_with_parsed_args(
data_selection_args)
##
## APPLY FEATURE MATRIX (from matrix above)
##
opFeatures = shell.workflow.featureSelectionApplet.topLevelOperator
opFeatures.Scales.setValue(ScalesList)
opFeatures.FeatureIds.setValue(FeatureIds)
opFeatures.SelectionMatrix.setValue(feature_selections)
##
## CUSTOMIZE CLASSIFIER TYPE
##
opPixelClassification = shell.workflow.pcApplet.topLevelOperator
if classifier_factory is not None:
opPixelClassification.ClassifierFactory.setValue(classifier_factory)
##
## READ/APPLY LABEL VOLUMES
##
# Read each label volume and inject the label data into the appropriate training slot
cwd = os.getcwd()
max_label_class = 0
for lane, label_data_path in enumerate(label_data_paths):
graph = Graph()
opReader = OpInputDataReader(graph=graph)
try:
opReader.WorkingDirectory.setValue(cwd)
opReader.FilePath.setValue(label_data_path)
print("Reading label volume: {}".format(label_data_path))
label_volume = opReader.Output[:].wait()
finally:
opReader.cleanUp()
raw_shape = opPixelClassification.InputImages[lane].meta.shape
if label_volume.ndim != len(raw_shape):
# Append a singleton channel axis
assert label_volume.ndim == len(raw_shape) - 1
label_volume = label_volume[..., None]
# Auto-calculate the max label value
max_label_class = max(max_label_class, label_volume.max())
print("Applying label volume to lane #{}".format(lane))
entire_volume_slicing = roiToSlice(*roiFromShape(label_volume.shape))
opPixelClassification.LabelInputs[lane][
entire_volume_slicing] = label_volume
assert max_label_class > 1, "Not enough label classes were found in your label data."
label_names = list(map(str, list(range(max_label_class))))
opPixelClassification.LabelNames.setValue(label_names)
##
## TRAIN CLASSIFIER
##
# Make sure the caches in the pipeline are not 'frozen'.
# (This is the equivalent of 'live update' mode in the GUI.)
opPixelClassification.FreezePredictions.setValue(False)
# Request the classifier object from the pipeline.
# This forces the pipeline to produce (train) the classifier.
_ = opPixelClassification.Classifier.value
##
## SAVE PROJECT
##
# save project file (includes the new classifier).
shell.projectManager.saveProject(force_all_save=False)
class OpImageOnDiskProvider(Operator):
"""
This simply wraps a lazyflow OpInputDataReader, but ensures that the metadata
(axistags, drange) on the output matches the metadata from the original data
(even if the output file format doesn't support metadata fields).
"""
TransactionSlot = InputSlot()
Input = InputSlot() # Used for dtype and shape only. Data is always provided directly from the file.
WorkingDirectory = InputSlot()
DatasetPath = InputSlot() # A TOTAL path (possibly including a dataset name, e.g. myfile.h5/volume/data
Dirty = InputSlot()
Output = OutputSlot()
def __init__(self, *args, **kwargs):
super( OpImageOnDiskProvider, self ).__init__(*args, **kwargs)
self._opReader = None
self._opMetadataInjector = None
# Block diagram:
#
# (Input.axistags, Input.drange)
# \
# DatasetPath ---> opReader ---> opMetadataInjector --> Output
# /
# WorkingDirectory
def setupOutputs( self ):
if self._opReader is not None:
self.Output.disconnect()
if self._opMetadataInjector:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
try:
# Configure the reader
dataReady = True
self._opReader = OpInputDataReader( parent=self )
self._opReader.WorkingDirectory.setValue( self.WorkingDirectory.value )
self._opReader.FilePath.setValue( self.DatasetPath.value )
# Since most file formats don't save meta-info,
# The reader output's axis order may be incorrect.
# (For example, if we export in npy format with zxy order,
# the Npy reader op will simply assume xyz order when it reads the data.)
# Force the metadata back to the correct state by copying select items from Input.meta
metadata = {}
metadata['axistags'] = self.Input.meta.axistags
metadata['drange'] = self.Input.meta.drange
self._opMetadataInjector = OpMetadataInjector( parent=self )
self._opMetadataInjector.Input.connect( self._opReader.Output )
self._opMetadataInjector.Metadata.setValue( metadata )
dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape
dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype
if dataReady:
self.Output.connect( self._opMetadataInjector.Output )
else:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
self.Output.meta.NOTREADY = True
except OpInputDataReader.DatasetReadError:
# Note: If the data is exported as a 'sequence', then this will always be NOTREADY
# because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent.
# That's okay because a stack is probably too slow to be of use for a preview anyway.
if self._opMetadataInjector:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
# The dataset doesn't exist yet.
self.Output.meta.NOTREADY = True
def execute(self, slot, subindex, roi, result):
assert False, "Output is supposed to be directly connected to an internal operator."
def propagateDirty(self, slot, subindex, roi):
if slot == self.Input:
self.Output.setDirty( roi )
else:
self.Output.setDirty( slice(None) )
def impl():
shell = self.shell
workflow = shell.projectManager.workflow
carvingApplet = workflow.carvingApplet
gui = carvingApplet.getMultiLaneGui()
op_carving = carvingApplet.topLevelOperator.getLane(0)
# activate the carving applet
shell.setSelectedAppletDrawer(2)
# let the gui catch up
QApplication.processEvents()
self.waitForViews(gui.currentGui().editor.imageViews)
# inject the labels
op5 = OpReorderAxes(parent=op_carving.parent)
opReader = OpInputDataReader(parent=op_carving.parent)
try:
opReader.FilePath.setValue(f"{self.reference_files['carving_label_file']}/exported_data")
op5.AxisOrder.setValue(op_carving.WriteSeeds.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
label_data = op5.Output[:].wait()
finally:
op5.cleanUp()
opReader.cleanUp()
slicing = roi.fullSlicing(label_data.shape)
op_carving.WriteSeeds[slicing] = label_data
gui.currentGui().labelingDrawerUi.segment.click()
QApplication.processEvents()
op_carving.saveObjectAs("Object 1")
op_carving.deleteObject("<not saved yet>")
# export the mesh:
req = gui.currentGui()._exportMeshes(["Object 1"], [self.output_obj_file])
req.wait()
# compare meshes
with open(self.output_obj_file, "r") as f:
left = f.read()
with open(self.reference_files["output_obj_file"], "r") as f:
right = f.read()
# TODO: might result in errors due to rounding on different systems
assert left == right
# export the completed segments layer
layermatch = [
x.name.startswith("Completed segments (unicolor)") for x in gui.currentGui().editor.layerStack
]
assert sum(layermatch) == 1, "Completed segments (unicolor) Layer expected."
completed_segments_layer = gui.currentGui().editor.layerStack[layermatch.index(True)]
opExport = get_export_operator(completed_segments_layer)
try:
opExport.OutputFilenameFormat.setValue(self.output_file)
opExport.run_export()
finally:
opExport.cleanUp()
assert os.path.exists(self.output_file)
# compare completed segments
with h5py.File(self.reference_files["output_file"], "r") as f_left:
data_left = f_left["exported_data"][:]
with h5py.File(self.output_file, "r") as f_right:
data_right = f_right["exported_data"][:]
numpy.testing.assert_array_almost_equal(data_left, data_right)
# Save the project
saveThread = self.shell.onSaveProjectActionTriggered()
saveThread.join()