def handleImportLabelsAction():
# Find the directory of the most recently opened image file
mostRecentImageFile = PreferencesManager().get(
'DataSelection', 'recent image')
if mostRecentImageFile is not None:
defaultDirectory = os.path.split(mostRecentImageFile)[0]
else:
defaultDirectory = os.path.expanduser('~')
fileNames = DataSelectionGui.getImageFileNamesToOpen(
self, defaultDirectory)
fileNames = map(str, fileNames)
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(
self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DataSelectionGui.getPossibleInternalPaths(
file_path)
if len(internal_paths) == 0:
QMessageBox.critical(
self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = H5VolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue(path_components.totalPath())
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue(
top_op.LabelInputs.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
# Finally, import the labels
top_op.importLabels(top_op.current_view_index(),
op5.Output)
finally:
op5.cleanUp()
opReader.cleanUp()
def testBasic_2d(self):
data = 255 * numpy.random.random((50, 100))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("yx"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("png")
opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x20-120_y10-60.png"
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
# Note: vigra inserts a channel axis, so read_data is xyc
assert (read_data[..., 0] == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic_Npy(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags("xy"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("numpy")
opExport.OutputFilenameFormat.setValue(self._tmpdir + "/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}")
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
assert os.path.split(opExport.ExportPath.value)[1] == "test_export_x10-110_y20-120.npy"
# print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def setupOutputs(self):
if self._opReader is not None:
self.Output.disconnect()
self._opReader.cleanUp()
self._opDummyData.Input.connect(self.Input)
dataReady = True
try:
# Configure the reader (shouldn't need to specify axis order; should be provided in data)
self._opReader = OpInputDataReader(parent=self)
self._opReader.WorkingDirectory.setValue(
self.WorkingDirectory.value)
self._opReader.FilePath.setValue(self.DatasetPath.value)
dataReady &= self._opReader.Output.meta.shape == self.Input.meta.shape
dataReady &= self._opReader.Output.meta.dtype == self.Input.meta.dtype
if dataReady:
self.Output.connect(self._opReader.Output)
except OpInputDataReader.DatasetReadError:
# The dataset doesn't exist yet.
dataReady = False
if not dataReady:
# The dataset isn't ready.
# That's okay, we'll just return dummy data.
self._opReader = None
self.Output.connect(self._opDummyData.Output)
def aaaNumpyFile(self):
g =graph.Graph()
npfile = "/home/akreshuk/data/synapse_small_4d.npy"
reader = OpInputDataReader(graph=g)
reader.FilePath.setValue(npfile)
#out = reader.Output[:].wait()
#print out.shape
opFeatures = OpPixelFeaturesPresmoothed(graph=g)
opFeatures.Scales.setValue(self.scales)
opFeatures.FeatureIds.setValue(self.featureIds)
opFeatures.Input.connect(reader.Output)
opFeatures.Matrix.setValue(self.selectedFeatures[5])
out = opFeatures.Output[:].wait()
print out.shape
opFeaturesInterp = OpPixelFeaturesInterpPresmoothed(graph=g)
opFeaturesInterp.Scales.setValue(self.scales)
opFeaturesInterp.FeatureIds.setValue(self.featureIds)
opFeaturesInterp.Input.connect(reader.Output)
opFeaturesInterp.Matrix.setValue(self.selectedFeatures[5])
opFeaturesInterp.InterpolationScaleZ.setValue(2)
out = opFeaturesInterp.Output[:].wait()
print out.shape
def test_h5_stack_multi_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files"""
shape = (4, 8, 16, 32, 3)
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
for index, t_slice in enumerate(data):
fname = self.testmultiH5FileName.format(index=index)
with h5py.File(fname, "w") as f:
data_group = f.create_group("volume")
data_group.create_dataset("data", data=t_slice)
if sequence_axis == "z":
data = numpy.concatenate(data, axis=0)
elif sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
h5SequenceReader = OpInputDataReader(graph=self.graph)
h5SequenceReader.SequenceAxis.setValue(sequence_axis)
globString = self.testmultiH5FileName.replace("02d}", "s}").format(index="*")
filenamePlusGlob = "{}/volume/data".format(globString)
try:
h5SequenceReader.FilePath.setValue(filenamePlusGlob)
h5data = h5SequenceReader.Output[:].wait()
assert h5data.shape == data.shape
numpy.testing.assert_array_equal(h5data, data)
finally:
# Call cleanUp() to close the file that this operator opened
h5SequenceReader.cleanUp()
def test_h5(self):
# Create HDF5 test data
with h5py.File(self.testH5FileName, "w") as f:
f.create_group("volume")
shape = (1, 2, 3, 4, 5)
f["volume"].create_dataset(
"data", data=numpy.indices(shape).sum(0).astype(numpy.float32), chunks=True, compression="gzip"
)
# Read the entire HDF5 file and verify the contents
h5Reader = OpInputDataReader(graph=self.graph)
try:
h5Reader.FilePath.setValue(self.testH5FileName + "/volume/data") # Append internal path
cwd = os.path.split(__file__)[0]
h5Reader.WorkingDirectory.setValue(cwd)
# Grab a section of the h5 data
h5Data = h5Reader.Output[0, 0, :, :, :].wait()
assert h5Data.shape == (1, 1, 3, 4, 5)
# (Just check part of the data)
for k in range(0, shape[2]):
for l in range(0, shape[3]):
for m in range(0, shape[4]):
assert h5Data[0, 0, k, l, m] == k + l + m
finally:
# Call cleanUp() to close the file that this operator opened
h5Reader.cleanUp()
assert not h5Reader._file # Whitebox assertion...
def test_npz(self):
# Create two Numpy test arrays
a = numpy.zeros((10, 11))
for x in range(0, 10):
for y in range(0, 11):
a[x, y] = x + y
b = numpy.arange((3 * 9)).reshape((3, 9))
numpy.savez(self.testNpzDataFileName, a=a, b=b)
# Now read back our test data using an OpInputDataReader operator
npyReader = OpInputDataReader(graph=self.graph)
try:
for internalPath, referenceArray in zip(["a", "b"], [a, b]):
npyReader.FilePath.setValue("{}/{}".format(
self.testNpzDataFileName, internalPath))
cwd = os.path.split(__file__)[0]
npyReader.WorkingDirectory.setValue(cwd)
npzData = npyReader.Output[:].wait()
assert npzData.shape == referenceArray.shape
numpy.testing.assert_array_equal(npzData, referenceArray)
finally:
npyReader.cleanUp()
def setupOutputs(self):
datasetInfo = self.Dataset.value
internalPath = self.ProjectDataGroup.value + '/' + datasetInfo.datasetId
# Data only comes from the project file if the user said so AND it exists in the project
datasetInProject = (datasetInfo.location == DatasetInfo.Location.ProjectInternal)
datasetInProject &= self.ProjectFile.connected() and \
internalPath in self.ProjectFile.value
if self._opReader is not None:
self.Image.disconnect()
self._opReader.cleanUp()
# If we should find the data in the project file, use a dataset reader
if datasetInProject:
self._opReader = OpStreamingHdf5Reader(parent=self)
self._opReader.Hdf5File.setValue(self.ProjectFile.value)
self._opReader.InternalPath.setValue(internalPath)
providerSlot = self._opReader.OutputImage
else:
# Use a normal (filesystem) reader
self._opReader = OpInputDataReader(parent=self)
if datasetInfo.axisorder is not None:
self._opReader.DefaultAxisOrder.setValue( datasetInfo.axisorder )
self._opReader.WorkingDirectory.connect( self.WorkingDirectory )
self._opReader.FilePath.setValue(datasetInfo.filePath)
providerSlot = self._opReader.Output
# Connect our external outputs to the internal operators we chose
self.Image.connect(providerSlot)
# Set the image name and usage flag
self.AllowLabels.setValue( datasetInfo.allowLabels )
self.ImageName.setValue(datasetInfo.filePath)
def testBasic(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opWriter = OpNpyWriter(graph=graph)
opWriter.Input.connect(opPiper.Output)
opWriter.Filepath.setValue(self._tmpdir +
'/npy_writer_test_output.npy')
# Write it...
opWriter.write()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opWriter.Filepath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic_Hdf5(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
self._tmpdir +
'/test_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
opExport.OutputInternalPath.setValue('volume/data')
opExport.CoordinateOffset.setValue((10, 20))
assert opExport.ExportPath.ready()
export_file = PathComponents(opExport.ExportPath.value).externalPath
assert os.path.split(
export_file)[1] == 'test_export_x10-110_y20-120.h5'
#print "exporting data to: {}".format( opExport.ExportPath.value )
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
opRead.cleanUp()
def compare_results(self, opReaderResult, compare_path, input_axes, post_process=None,
max_mse=None, max_part_uneqaul=None):
if os.path.exists(compare_path):
result = opReaderResult.Output[:].wait()
opReaderCompare = OpInputDataReader(graph=Graph())
opReaderCompare.FilePath.setValue(compare_path)
opReorderCompare = OpReorderAxes(parent=opReaderCompare)
opReorderCompare.Input.connect(opReaderCompare.Output)
opReorderCompare.AxisOrder.setValue(input_axes)
compare = opReorderCompare.Output[:].wait()
assert result.shape == compare.shape, (result.shape, compare.shape)
if post_process:
result = post_process(result)
compare = post_process(compare)
# for easy debugging:
# -----------------------------------------------------------------
# import matplotlib.pyplot as plt
# res = result.squeeze()
# comp = compare.squeeze()
# if len(res.shape) > 2:
# res = res.reshape(-1, max(res.shape))
# comp = comp.reshape(-1, max(comp.shape))
# plt.figure()
# plt.imshow(res)
# plt.title(f'res {result.shape}')
# plt.colorbar()
# plt.figure()
# plt.imshow(comp)
# plt.title('comp')
# plt.colorbar()
# plt.figure()
# plt.imshow(res - comp)
# plt.title('diff')
# plt.colorbar()
# plt.show()
# -----------------------------------------------------------------
if max_mse:
assert max_mse > numpy.mean(numpy.square(result - compare)), \
numpy.mean(numpy.square(result - compare))
elif max_part_uneqaul:
assert max_part_uneqaul > numpy.mean(~numpy.isclose(result, compare)), \
numpy.mean(~numpy.isclose(result, compare))
else:
assert numpy.allclose(result, compare), f'{result.shape}, {compare.shape}'
else:
writer = OpFormattedDataExport(graph=Graph())
writer.Input.connect(opReaderResult.Output)
writer.OutputFilenameFormat.setValue(compare_path)
writer.TransactionSlot.setValue(True)
writer.run_export()
warnings.warn(f'created comparison data: {compare_path} with axis order {input_axes}')
def convert_predictions_to_uncertainties( input_path, parsed_export_args ):
"""
Read exported pixel predictions and calculate/export the uncertainties.
input_path: The path to the prediction output file. If hdf5, must include the internal dataset name.
parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser.
"""
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue( os.getcwd() )
opReader.FilePath.setValue(input_path)
opUncertainty = OpEnsembleMargin( graph=graph )
opUncertainty.Input.connect( opReader.Output )
opExport = OpFormattedDataExport( graph=graph )
opExport.Input.connect( opUncertainty.Output )
# Apply command-line arguments.
DataExportApplet._configure_operator_with_parsed_args(parsed_export_args, opExport)
last_progress = [-1]
def print_progress(progress_percent):
if progress_percent != last_progress[0]:
last_progress[0] = progress_percent
sys.stdout.write( " {}".format(progress_percent) )
print "Exporting results to : {}".format( opExport.ExportPath.value )
sys.stdout.write("Progress:")
opExport.progressSignal.subscribe(print_progress)
# Begin export
opExport.run_export()
sys.stdout.write("\n")
print "DONE."
def test_via_OpExportSlot(self):
data = 255 * numpy.random.random((64, 128, 128, 1))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("dvid")
url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format(
**self.__dict__)
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output(*roiFromShape(data.shape)).wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test_basic(self):
opData = OpArrayCache(graph=self.graph)
opData.blockShape.setValue(self.testData.shape)
opData.Input.setValue(self.testData)
filepath = os.path.join(self._tmpdir, 'multipage.tiff')
logger.debug("writing to: {}".format(filepath))
opExport = OpExportMultipageTiff(graph=self.graph)
opExport.Filepath.setValue(filepath)
opExport.Input.connect(opData.Output)
# Run the export
opExport.run_export()
opReader = OpInputDataReader(graph=self.graph)
opReader.FilePath.setValue(filepath)
# The reader assumes xyzc order.
# We have to transpose the data before we compare.
opReorderAxes = OpReorderAxes(graph=self.graph)
opReorderAxes.AxisOrder.setValue(self._axisorder)
opReorderAxes.Input.connect(opReader.Output)
readData = opReorderAxes.Output[:].wait()
logger.debug("Expected shape={}".format(self.testData.shape))
logger.debug("Read shape={}".format(readData.shape))
assert opReorderAxes.Output.meta.shape == self.testData.shape, "Exported files were of the wrong shape or number."
assert (opReorderAxes.Output[:].wait() == self.testData.view(
numpy.ndarray)).all(), "Exported data was not correct"
# Cleanup
opReorderAxes.cleanUp()
opReader.cleanUp()
def test_h5_stack_single_file(self, sequence_axis):
"""Test stack/sequence reading in hdf5-files for given 'sequence_axis'"""
shape = (4, 8, 16, 32, 3) # assuming axis guess order is 'tzyxc'
data = numpy.random.randint(0, 255, size=shape).astype(numpy.uint8)
with h5py.File(self.testH5FileName, "w") as f:
data_group = f.create_group("volumes")
for index, t_slice in enumerate(data):
data_group.create_dataset("timepoint-{index:02d}".format(index=index), data=t_slice)
if sequence_axis == "z":
data = numpy.concatenate(data, axis=0)
elif sequence_axis == "c":
data = numpy.concatenate(data, axis=-1)
h5SequenceReader = OpInputDataReader(graph=self.graph)
h5SequenceReader.SequenceAxis.setValue(sequence_axis)
filenamePlusGlob = "{}/volumes/timepoint-*".format(self.testH5FileName)
try:
h5SequenceReader.FilePath.setValue(filenamePlusGlob)
h5data = h5SequenceReader.Output[:].wait()
assert h5data.shape == data.shape, f"{h5data.shape}, {data.shape}"
numpy.testing.assert_array_equal(h5data, data)
finally:
# Call cleanUp() to close the file that this operator opened
h5SequenceReader.cleanUp()
def create_input(cls,
filepath,
input_axes,
outmin=None,
outmax=None,
dtype=None):
""" Creates a file from the data at 'filepath' that has 'input_axes' """
basename = os.path.basename(filepath)
reader = OpInputDataReader(graph=Graph())
assert os.path.exists(filepath), "{} not found".format(filepath)
reader.FilePath.setValue(os.path.abspath(filepath))
writer = OpFormattedDataExport(parent=reader)
if outmin is not None:
writer.ExportMin.setValue(outmin)
if outmax is not None:
writer.ExportMax.setValue(outmax)
if dtype is not None:
writer.ExportDtype.setValue(dtype)
writer.OutputAxisOrder.setValue(input_axes)
writer.Input.connect(reader.Output)
writer.OutputFilenameFormat.setValue(
os.path.join(cls.dir,
basename.split(".")[0] + "_" + input_axes))
writer.TransactionSlot.setValue(True)
input_path = writer.ExportPath.value
writer.run_export()
return input_path
def testBasic(self):
graph = Graph()
opExport = OpDataExport(graph=graph)
try:
opExport.TransactionSlot.setValue(True)
opExport.WorkingDirectory.setValue(self._tmpdir)
# Simulate the important fields of a DatasetInfo object
class MockDatasetInfo(object):
pass
rawInfo = MockDatasetInfo()
rawInfo.nickname = 'test_nickname'
rawInfo.filePath = './somefile.h5'
opExport.RawDatasetInfo.setValue(rawInfo)
opExport.SelectionNames.setValue(['Mock Export Data'])
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
opExport.Inputs.resize(1)
opExport.Inputs[0].setValue(data)
sub_roi = [(10, 20), (90, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
'{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}'
)
opExport.OutputInternalPath.setValue('volume/data')
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
expected_path = self._tmpdir + '/' + rawInfo.nickname + '_export_x10-90_y20-80.h5/volume/data'
computed_path = opExport.ExportPath.value
assert os.path.normpath(computed_path) == os.path.normpath(expected_path), \
"Expected {}\nGot: {}".format( expected_path, computed_path )
opExport.run_export()
finally:
opExport.cleanUp()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(computed_path)
# Compare with the correct subregion and convert dtype.
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testBasic(self):
graph = Graph()
opExport = OpFormattedDataExport(graph=graph)
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
opExport.Input.setValue(data)
sub_roi = [(10, 0), (None, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.InputMin.setValue(0.0)
opExport.InputMax.setValue(100.0)
opExport.ExportMin.setValue(100)
opExport.ExportMax.setValue(200)
opExport.OutputFormat.setValue('hdf5')
opExport.OutputFilenameFormat.setValue(
self._tmpdir + '/export_x{x_start}-{x_stop}_y{y_start}-{y_stop}')
opExport.OutputInternalPath.setValue('volume/data')
opExport.TransactionSlot.setValue(True)
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
assert opExport.ImageToExport.meta.drange == (100, 200)
#print "exporting data to: {}".format( opExport.ExportPath.value )
assert opExport.ExportPath.value == self._tmpdir + '/' + 'export_x10-100_y0-80.h5/volume/data'
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
# Compare with the correct subregion and convert dtype.
sub_roi[1] = (100, 80) # Replace 'None' with full extent
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
expected_data += 100 # see renormalization settings
assert opRead.Output.meta.shape == expected_data.shape
assert opRead.Output.meta.dtype == expected_data.dtype
read_data = opRead.Output[:].wait()
# Due to rounding errors, the actual result and the expected result may differ by 1
# e.g. if the original pixel value was 32.99999999
# Also, must promote to signed values to avoid unsigned rollover
# See issue ( https://github.com/ilastik/lazyflow/issues/165 ).
expected_data_signed = expected_data.astype(numpy.int16)
read_data_signed = expected_data.astype(numpy.int16)
difference_from_expected = expected_data_signed - read_data_signed
assert (numpy.abs(difference_from_expected) <=
1).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def get_non_transposed_provider_slot(
self,
parent: Optional[Operator] = None,
graph: Optional[Graph] = None) -> OutputSlot:
op_reader = OpInputDataReader(parent=parent,
graph=graph,
FilePath=self.url)
return op_reader.Output
def testBasic(self, tmp_h5_single_dataset: Path):
graph = Graph()
opExport = OpDataExport(graph=graph)
try:
opExport.TransactionSlot.setValue(True)
opExport.WorkingDirectory.setValue(self._tmpdir)
rawInfo = FilesystemDatasetInfo(filePath=str(
tmp_h5_single_dataset / "test_group/test_data"),
nickname="test_nickname")
opExport.RawDatasetInfo.setValue(rawInfo)
opExport.SelectionNames.setValue(["Mock Export Data"])
data = numpy.random.random((100, 100)).astype(numpy.float32) * 100
data = vigra.taggedView(data, vigra.defaultAxistags("xy"))
opExport.Inputs.resize(1)
opExport.Inputs[0].setValue(data)
sub_roi = [(10, 20), (90, 80)]
opExport.RegionStart.setValue(sub_roi[0])
opExport.RegionStop.setValue(sub_roi[1])
opExport.ExportDtype.setValue(numpy.uint8)
opExport.OutputFormat.setValue("hdf5")
opExport.OutputFilenameFormat.setValue(
"{dataset_dir}/{nickname}_export_x{x_start}-{x_stop}_y{y_start}-{y_stop}"
)
opExport.OutputInternalPath.setValue("volume/data")
assert opExport.ImageToExport.ready()
assert opExport.ExportPath.ready()
expected_path = tmp_h5_single_dataset.parent.joinpath(
rawInfo.nickname +
"_export_x10-90_y20-80.h5/volume/data").as_posix()
computed_path = opExport.ExportPath.value
assert os.path.normpath(computed_path) == os.path.normpath(
expected_path)
opExport.run_export()
finally:
opExport.cleanUp()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(computed_path)
# Compare with the correct subregion and convert dtype.
expected_data = data.view(numpy.ndarray)[roiToSlice(*sub_roi)]
expected_data = expected_data.astype(numpy.uint8)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def testCreateExportDirectory(self):
"""
Test that the batch operator can create the export directory if it doesn't exist yet.
"""
# Start by writing some test data to disk.
self.testData = numpy.random.random((1, 10, 10, 10, 1))
numpy.save(self.testDataFileName, self.testData)
cwd = os.getcwd()
info = DatasetInfo()
info.filePath = os.path.join(cwd, 'NpyTestData.npy')
graph = Graph()
opBatchIo = OpBatchIo(graph=graph)
opInput = OpInputDataReader(graph=graph)
opInput.FilePath.setValue(info.filePath)
# Our test "processing pipeline" is just a smoothing operator.
opSmooth = OpGaussianSmoothing(graph=graph)
opSmooth.Input.connect(opInput.Output)
opSmooth.sigma.setValue(3.0)
exportDir = os.path.join(cwd, 'exported_data')
opBatchIo.ExportDirectory.setValue(exportDir)
opBatchIo.Suffix.setValue('_smoothed')
opBatchIo.Format.setValue(ExportFormat.H5)
opBatchIo.DatasetPath.setValue(info.filePath)
internalPath = 'path/to/data'
opBatchIo.InternalPath.setValue(internalPath)
opBatchIo.ImageToExport.connect(opSmooth.Output)
dirty = opBatchIo.Dirty.value
assert dirty == True
outputPath = opBatchIo.OutputDataPath.value
assert outputPath == os.path.join(exportDir, 'NpyTestData_smoothed.h5',
internalPath)
result = opBatchIo.ExportResult.value
assert result
dirty = opBatchIo.Dirty.value
assert dirty == False
# Check the file
smoothedPath = PathComponents(outputPath).externalPath
with h5py.File(smoothedPath, 'r') as f:
assert internalPath in f
assert f[internalPath].shape == self.testData.shape
try:
os.remove(smoothedPath)
os.rmdir(exportDir)
except:
pass
def _test_object_classification(self, dims, variant, input_axes):
project = f'ObjectClassification{dims}_{variant}.ilp'
try:
self.untested_projects.remove(project)
except ValueError:
pass
project_file = os.path.join(self.PROJECT_FILE_BASE, project)
if not os.path.exists(project_file):
raise IOError(f'project file "{project_file}" not found')
compare_path = os.path.join(self.dir, f'Object_Predictions_{dims}_{variant}_compare')
output_path = compare_path.replace('_compare', f'_{input_axes}_result')
args = [
'--headless',
'--project', project_file,
# Batch export options
'--export_source', 'Object Predictions',
'--output_filename_format', output_path,
'--output_format', 'hdf5',
'--export_dtype', 'uint8',
'--pipeline_result_drange', '(0,255)',
'--export_drange', '(0,255)',
# Input args
'--input_axes', input_axes,
'--raw_data',
self.create_input(os.path.join(self.PROJECT_FILE_BASE, 'inputdata', f'{dims}.h5'), input_axes)
]
if variant == 'wPred':
args.append('--prediction_maps')
args.append(self.create_input(
os.path.join(self.PROJECT_FILE_BASE, 'inputdata', f'{dims}_Probabilities.h5'),
input_axes))
elif variant == 'wSeg':
args.append('--segmentation_image')
args.append(self.create_input(
os.path.join(self.PROJECT_FILE_BASE, 'inputdata', f'{dims}_Binary_Segmentation.h5'),
input_axes))
else:
raise NotImplementedError(f'unknown variant: {variant}')
# Start up the ilastik.py entry script as if we had launched it from the command line
# This will execute the batch mode script
sys.argv = ['ilastik.py'] + args
self.ilastik_startup.main()
output_path += '.h5'
compare_path += '.h5'
opReaderResult = OpInputDataReader(graph=Graph())
opReaderResult.FilePath.setValue(output_path)
self.compare_results(opReaderResult, compare_path, input_axes)
def setupOutputs(self):
if self._opReader is not None:
self.Output.disconnect()
if self._opMetadataInjector:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
try:
# Configure the reader
dataReady = True
self._opReader = OpInputDataReader(parent=self)
self._opReader.WorkingDirectory.setValue(
self.WorkingDirectory.value)
self._opReader.FilePath.setValue(self.DatasetPath.value)
# Since most file formats don't save meta-info,
# The reader output's axis order may be incorrect.
# (For example, if we export in npy format with zxy order,
# the Npy reader op will simply assume xyz order when it reads the data.)
# Force the metadata back to the correct state by copying select items from Input.meta
metadata = {}
metadata['axistags'] = self.Input.meta.axistags
metadata['drange'] = self.Input.meta.drange
metadata['display_mode'] = self.Input.meta.display_mode
self._opMetadataInjector = OpMetadataInjector(parent=self)
self._opMetadataInjector.Input.connect(self._opReader.Output)
self._opMetadataInjector.Metadata.setValue(metadata)
dataReady &= self._opMetadataInjector.Output.meta.shape == self.Input.meta.shape
dataReady &= self._opMetadataInjector.Output.meta.dtype == self.Input.meta.dtype
if dataReady:
self.Output.connect(self._opMetadataInjector.Output)
else:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
self.Output.meta.NOTREADY = True
#except OpInputDataReader.DatasetReadError:
except Exception as ex:
#logger.debug( "On-disk image can't be read: {}".format(ex) )
# Note: If the data is exported as a 'sequence', then this will always be NOTREADY
# because the 'path' (e.g. 'myfile_{slice_index}.png' will be nonexistent.
# That's okay because a stack is probably too slow to be of use for a preview anyway.
if self._opMetadataInjector:
self._opMetadataInjector.cleanUp()
self._opMetadataInjector = None
self._opReader.cleanUp()
self._opReader = None
# The dataset doesn't exist yet.
self.Output.meta.NOTREADY = True
def _test_counting(self, dims, input_axes):
project = f"CellDensityCounting{dims}.ilp"
try:
self.untested_projects.remove(project)
except ValueError:
pass
project_file = os.path.join(self.PROJECT_FILE_BASE, project)
if not os.path.exists(project_file):
raise IOError('project file "{}" not found'.format(project_file))
compare_path = os.path.join(self.dir, f"Counts{dims}_compare")
output_path = compare_path.replace("_compare", f"_{input_axes}_result")
args = []
args.append("--headless")
args.append("--project=" + project_file)
# Batch export options
# If we were actually launching from the command line, 'png sequence'
# would be in quotes...
# args.append('--output_format=png sequence')
args.append("--export_source=Probabilities")
args.append("--output_filename_format=" + output_path)
args.append("--output_format=hdf5")
args.append("--export_dtype=float32")
args.append("--pipeline_result_drange=(0,1)")
args.append("--export_drange=(0,1)")
# Input args
args.append("--input_axes={}".format(input_axes))
input_source_path1 = os.path.join(self.PROJECT_FILE_BASE, "inputdata",
"{}.h5".format(dims))
input_path1 = self.create_input(input_source_path1, input_axes)
args.append("--raw_data=" + input_path1)
# Clear the existing commandline args so it looks like we're starting fresh.
sys.argv = ["ilastik.py"]
sys.argv += args
# Start up the ilastik.py entry script as if we had launched it from the command line
# This will execute the batch mode script
self.ilastik_startup.main()
output_path += ".h5"
compare_path += ".h5"
opReaderResult = OpInputDataReader(graph=Graph())
opReaderResult.FilePath.setValue(output_path)
self.compare_results(opReaderResult,
compare_path,
input_axes,
max_mse=0.001)
def get_non_transposed_provider_slot(
self, parent: Optional[Operator] = None, graph: Optional[Graph] = None
) -> OutputSlot:
op_reader = OpInputDataReader(
parent=parent,
graph=graph,
WorkingDirectory=self.base_dir,
FilePath=self.filePath,
SequenceAxis=self.sequence_axis,
)
return op_reader.Output
def __init__(self, *, url: str, nickname: str = "", **info_kwargs):
self.url = url
op_reader = OpInputDataReader(graph=Graph(), FilePath=self.url)
meta = op_reader.Output.meta.copy()
super().__init__(
default_tags=meta.axistags,
nickname=nickname or self.url.rstrip("/").split("/")[-1],
laneShape=meta.shape,
laneDtype=meta.dtype,
**info_kwargs,
)
def test_npy_with_roi(self):
a = numpy.indices((100, 100, 200)).astype(numpy.uint8).sum(0)
assert a.shape == (100, 100, 200)
numpy.save(self.testNpyDataFileName, a)
opReader = OpInputDataReader(graph=lazyflow.graph.Graph())
opReader.FilePath.setValue(self.testNpyDataFileName)
opReader.SubVolumeRoi.setValue(((10, 20, 30), (50, 70, 90)))
all_data = opReader.Output[:].wait()
assert all_data.shape == (40, 50, 60)
assert (all_data == a[10:50, 20:70, 30:90]).all()
def handleImportLabelsAction():
fileNames = ImageFileDialog(
self,
preferences_group="DataSelection",
preferences_setting="recent image").getSelectedPaths()
fileNames = list(map(str, fileNames))
# For now, we require a single hdf5 file
if len(fileNames) > 1:
QMessageBox.critical(
self, "Too many files",
"Labels must be contained in a single hdf5 volume.")
return
if len(fileNames) == 0:
# user cancelled
return
file_path = fileNames[0]
internal_paths = DatasetInfo.getPossibleInternalPathsFor(file_path)
if len(internal_paths) == 0:
QMessageBox.critical(
self, "No volumes in file",
"Couldn't find a suitable dataset in your hdf5 file.")
return
if len(internal_paths) == 1:
internal_path = internal_paths[0]
else:
dlg = SubvolumeSelectionDlg(internal_paths, self)
if dlg.exec_() == QDialog.Rejected:
return
selected_index = dlg.combo.currentIndex()
internal_path = str(internal_paths[selected_index])
path_components = PathComponents(file_path)
path_components.internalPath = str(internal_path)
try:
top_op = self.topLevelOperatorView
opReader = OpInputDataReader(parent=top_op.parent)
opReader.FilePath.setValue(path_components.totalPath())
# Reorder the axes
op5 = OpReorderAxes(parent=top_op.parent)
op5.AxisOrder.setValue(top_op.LabelInputs.meta.getAxisKeys())
op5.Input.connect(opReader.Output)
# Finally, import the labels
top_op.importLabels(top_op.current_view_index(), op5.Output)
finally:
op5.cleanUp()
opReader.cleanUp()
def convert_predictions_to_segmentation(input_paths, parsed_export_args):
"""
Read exported pixel predictions and calculate/export the segmentation.
input_path: The path to the prediction output file. If hdf5, must include the internal dataset name.
parsed_export_args: The already-parsed cmd-line arguments generated from a DataExportApplet-compatible ArgumentParser.
"""
graph = Graph()
opReader = OpInputDataReader(graph=graph)
opReader.WorkingDirectory.setValue(os.getcwd())
opArgmaxChannel = OpArgmaxChannel(graph=graph)
opArgmaxChannel.Input.connect(opReader.Output)
opExport = OpFormattedDataExport(graph=graph)
opExport.Input.connect(opArgmaxChannel.Output)
# Apply command-line arguments.
DataExportApplet._configure_operator_with_parsed_args(
parsed_export_args, opExport)
last_progress = [-1]
def print_progress(progress_percent):
if progress_percent != last_progress[0]:
last_progress[0] = progress_percent
sys.stdout.write(" {}".format(progress_percent))
opExport.progressSignal.subscribe(print_progress)
for input_path in input_paths:
opReader.FilePath.setValue(input_path)
input_pathcomp = PathComponents(input_path)
opExport.OutputFilenameFormat.setValue(str(
input_pathcomp.externalPath))
output_path = opExport.ExportPath.value
output_pathcomp = PathComponents(output_path)
output_pathcomp.filenameBase += "_Segmentation"
opExport.OutputFilenameFormat.setValue(
str(output_pathcomp.externalPath))
print("Exporting results to : {}".format(opExport.ExportPath.value))
sys.stdout.write("Progress:")
# Begin export
opExport.run_export()
sys.stdout.write("\n")
print("DONE.")
