def testHDF5(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((150, 250, 150), dtype=numpy.float32).sum(0)
sampleData = sampleData.view(vigra.VigraArray)
sampleData.axistags = vigra.defaultAxistags('xyz')
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([75, 125, 150])
op.Input.connect(opData.Output)
assert op.OutputHdf5.ready()
slicing = numpy.s_[0:75, 125:250, 0:150]
slicing_str = str(
[list(_) for _ in zip(*[[_.start, _.stop] for _ in slicing])])
expectedData = sampleData[slicing].view(numpy.ndarray)
slicing_2 = numpy.s_[0:75, 0:125, 0:150]
expectedData_2 = expectedData[slicing_2].view(numpy.ndarray)
#logger.debug("Requesting data...")
tempdir = tempfile.mkdtemp()
try:
with h5py.File(os.path.join(tempdir, "data.h5"), "w") as h5_file:
op.OutputHdf5[slicing].writeInto(h5_file).wait()
assert slicing_str in h5_file, "Missing dataset!"
assert (h5_file[slicing_str][()] == expectedData
).all(), "Incorrect output!"
with h5py.File(os.path.join(tempdir, "data.h5"), "r") as h5_file:
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.meta.axistags = vigra.AxisTags('xyz')
opData.Input.setValue(numpy.empty_like(expectedData_2))
op = OpCompressedCache(parent=None, graph=graph)
op.InputHdf5.meta.axistags = vigra.AxisTags('xyz')
op.InputHdf5.meta.shape = (75, 125, 150)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([75, 125, 150])
op.Input.connect(opData.Output)
op.InputHdf5[slicing_2] = h5_file[slicing_str]
result = op.Output[slicing_2].wait()
assert (result == expectedData_2).all(), "Incorrect output!"
finally:
shutil.rmtree(tempdir)
def testDirtyPropagation(self):
graph = Graph()
data = numpy.random.random((1, 100, 100, 10, 1))
opProvider = OpArrayPiper(graph=graph)
opProvider.Input.setValue(data)
opSubRegion = OpSubRegion(graph=graph)
opSubRegion.Input.connect(opProvider.Output)
opSubRegion.Start.setValue((0, 20, 30, 5, 0))
opSubRegion.Stop.setValue((1, 30, 50, 8, 1))
gotDirtyRois = []
def handleDirty(slot, roi):
gotDirtyRois.append(roi)
opSubRegion.Output.notifyDirty(handleDirty)
# Set an input dirty region that overlaps with the subregion
key = make_key[0:1, 15:35, 32:33, 0:10, 0:1]
opProvider.Input.setDirty(key)
assert len(gotDirtyRois) == 1
assert gotDirtyRois[0].start == [0, 0, 2, 0, 0]
assert gotDirtyRois[0].stop == [1, 10, 3, 3, 1]
# Now mark a region that DOESN'T overlap with the subregion
key = make_key[0:1, 70:80, 32:33, 0:10, 0:1]
opProvider.Input.setDirty(key)
# Should have gotten no extra dirty notifications
assert len(gotDirtyRois) == 1
def testBasic1(self):
space = numpy.array([100, 100])
radii = numpy.array([7, 6, 6, 6, 7, 6])
magnitudes = numpy.array([15, 16, 15, 17, 16, 16])
points = numpy.array([[30, 24], [59, 65], [21, 65], [13, 12], [72, 16],
[45, 32]])
masks = nanshe.syn.data.generate_hypersphere_masks(
space, points, radii)
images = nanshe.syn.data.generate_gaussian_images(
space, points, radii / 3.0, magnitudes)
images *= masks
bases_indices = [[1, 3, 4], [0, 2], [5]]
bases_masks = numpy.zeros((len(bases_indices), ) + masks.shape[1:],
dtype=masks.dtype)
bases_images = numpy.zeros((len(bases_indices), ) + images.shape[1:],
dtype=images.dtype)
for i, each_basis_indices in enumerate(bases_indices):
bases_masks[i] = masks[list(each_basis_indices)].max(axis=0)
bases_images[i] = images[list(each_basis_indices)].max(axis=0)
bases_images = vigra.taggedView(bases_images, "cyx")
graph = Graph()
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(bases_images)
op = OpNanshePostprocessData(graph=graph)
op.Input.connect(opPrep.Output)
op.SignificanceThreshold.setValue(3.0)
op.WaveletTransformScale.setValue(4)
op.NoiseThreshold.setValue(3.0)
op.AcceptedRegionShapeConstraints_MajorAxisLength_Min.setValue(0.0)
op.AcceptedRegionShapeConstraints_MajorAxisLength_Min_Enabled.setValue(
True)
op.AcceptedRegionShapeConstraints_MajorAxisLength_Max.setValue(25.0)
op.AcceptedRegionShapeConstraints_MajorAxisLength_Max_Enabled.setValue(
True)
op.PercentagePixelsBelowMax.setValue(0.0)
op.MinLocalMaxDistance.setValue(10.0)
op.AcceptedNeuronShapeConstraints_Area_Min.setValue(30)
op.AcceptedNeuronShapeConstraints_Area_Min_Enabled.setValue(True)
op.AcceptedNeuronShapeConstraints_Area_Max.setValue(600)
op.AcceptedNeuronShapeConstraints_Area_Max_Enabled.setValue(True)
op.AcceptedNeuronShapeConstraints_Eccentricity_Min.setValue(0.0)
op.AcceptedNeuronShapeConstraints_Eccentricity_Min_Enabled.setValue(
True)
op.AcceptedNeuronShapeConstraints_Eccentricity_Max.setValue(0.9)
op.AcceptedNeuronShapeConstraints_Eccentricity_Max_Enabled.setValue(
True)
op.AlignmentMinThreshold.setValue(0.6)
op.OverlapMinThreshold.setValue(0.6)
op.Fuse_FractionMeanNeuronMaxThreshold.setValue(0.6)
neurons = op.Output[...].wait()
def test_export(self):
# For now, we require block-aligned POST
data = numpy.random.randint(0, 255,
(32, 128, 256, 1)).astype(numpy.uint8)
data = numpy.asfortranarray(data, numpy.uint8)
assert data.shape == (32, 128, 256, 1)
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))
# Retrieve from server
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportDvidVolume(transpose_axes=True, graph=graph)
# Reverse data order for dvid export
opExport.Input.connect(opPiper.Output)
opExport.NodeDataUrl.setValue(
"http://localhost:8000/api/node/{uuid}/{dataname}".format(
uuid=self.data_uuid, dataname=self.data_name))
# Export!
opExport.run_export()
# Read back. (transposed, because of transposed_axes, above)
accessor = VoxelsAccessor(TEST_DVID_SERVER, self.data_uuid,
self.data_name)
read_data = accessor[:]
# Compare.
assert (data.view(numpy.ndarray) == read_data.transpose()
).all(), "Exported data is not correct"
def testReshape(self):
opSlicer = self.opSlicer
dirtyRois = {}
def handleDirty(i, slot, roi):
assert opSlicer.Slices[i] == slot
dirtyRois[i] = roi
for i, slot in enumerate(opSlicer.Slices):
slot.notifyDirty(partial(handleDirty, i))
# Initial data has 3 channels with values 0,1,2
# Simulate removing the middle one: Now data has two channels with values 0,2
data = numpy.indices((10, 10, 10, 2))[3]
data = data * 2
data = data.view(vigra.VigraArray)
data.axistags = vigra.defaultAxistags('xyzc')
# Connect a new data provider
opNewProvider = OpArrayPiper(graph=opSlicer.graph)
opNewProvider.Input.setValue(data)
opSlicer.Input.connect(opNewProvider.Output)
assert len(opSlicer.Slices) == 2
for i, slot in enumerate(opSlicer.Slices):
assert slot.meta.shape == (10, 10, 10, 1)
assert (slot[...].wait() == i * 2).all()
assert len(dirtyRois) == 2
assert dirtyRois[0].start == [0, 0, 0, 0]
assert dirtyRois[0].stop == [10, 10, 10, 1]
assert dirtyRois[1].start == [0, 0, 0, 0]
assert dirtyRois[1].stop == [10, 10, 10, 1]
def testBasic2(self):
a = numpy.eye(3, dtype = numpy.float32)
a = a[None, ..., None]
a = vigra.taggedView(a, "tyxc")
expected_b = numpy.array([[ 0.59375, -0.375 , -0.34375],
[-0.375 , 0.625 , -0.375 ],
[-0.34375, -0.375 , 0.59375]], dtype=numpy.float32)
expected_b = expected_b[None, ..., None]
expected_b = vigra.taggedView(expected_b, "tyxc")
graph = Graph()
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(a)
op = OpNansheWaveletTransform(graph=graph)
op.Input.connect(opPrep.Output)
op.Scale.setValue((0, 1, 1))
b = op.Output[...].wait()
b = vigra.taggedView(b, "tyxc")
assert((b == expected_b).all())
def testBasicOneBlock(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((3, 100, 200, 150, 2),
dtype=numpy.float32).sum(0)
sampleData = sampleData.view(vigra.VigraArray)
sampleData.axistags = vigra.defaultAxistags('txyzc')
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
# NO Block shape for this test.
#op.BlockShape.setValue( [1, 100, 75, 50, 2] )
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1]
expectedData = sampleData[slicing].view(numpy.ndarray)
#logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
#logger.debug("Checking data...")
assert (readData == expectedData).all(), "Incorrect output!"
def testFailedProcessing(self):
op = OpArrayPiper(graph=Graph())
inputData = numpy.indices((100, 100)).sum(0)
op.Input.setValue(inputData)
roiList = []
block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100]))
for block_start in block_starts:
roiList.append(getBlockBounds([100, 100], [10, 10], block_start))
class SpecialException(Exception):
pass
def handleResult(roi, result):
raise SpecialException(
"Intentional Exception: raised while handling the result")
totalVolume = numpy.prod(inputData.shape)
batch = RoiRequestBatch(op.Output,
roiList.__iter__(),
totalVolume,
batchSize=10,
allowParallelResults=False)
batch.resultSignal.subscribe(handleResult)
# FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions.
# This only tests one of them (in the notify_finished() handler)
try:
batch.execute()
except SpecialException:
pass
else:
assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def test_export(self):
"""
hostname: The dvid server host
h5filename: The h5 file to compare against
h5group: The hdf5 group, also used as the uuid of the dvid dataset
h5dataset: The dataset name, also used as the name of the dvid dataset
start, stop: The bounds of the cutout volume to retrieve from the server. C ORDER FOR THIS TEST BECAUSE we use transpose_axes=True.
"""
data = numpy.indices( (10, 100, 200, 4) )
assert data.shape == (4, 10, 100, 200, 4)
data = data.astype( numpy.uint8 )
data = vigra.taggedView( data, vigra.defaultAxistags('tzyxc') )
# Retrieve from server
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue( data )
opExport = OpExportDvidVolume( transpose_axes=True, graph=graph )
# Reverse data order for dvid export
opExport.Input.connect( opPiper.Output )
opExport.NodeDataUrl.setValue( 'http://localhost:8000/api/node/{uuid}/{dataname}'.format( uuid=self.data_uuid, dataname=self.data_name ) )
# Export!
opExport.run_export()
# Retrieve from file
with h5py.File(self.test_filepath, 'r') as f:
exported_data = f['all_nodes'][self.data_uuid][self.data_name][:]
# Compare.
assert ( data.view(numpy.ndarray) == exported_data.transpose() ).all(),\
"Exported data is not correct"
def testBasicOneBlock_masked(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((3, 100, 200, 150, 2),
dtype=numpy.float32).sum(0)
sampleData = sampleData.view(numpy.ma.masked_array)
sampleData.set_fill_value(numpy.float32(numpy.nan))
sampleData[0] = numpy.ma.masked
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.meta.has_mask = True
opData.Input.meta.axistags = vigra.defaultAxistags('txyzc')
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
# NO Block shape for this test.
#op.BlockShape.setValue( [1, 100, 75, 50, 2] )
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1]
expectedData = sampleData[slicing]
#logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
#logger.debug("Checking data...")
assert (readData == expectedData).all() and \
(readData.mask == expectedData.mask).all() and \
((readData.fill_value == expectedData.fill_value) |
(numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\
"Incorrect output!"
def test_in_ascending_order(graph, monkeypatch):
"""Make sure iterator returns data slices in ascending order"""
op = OpArrayPiper(graph=graph)
data = vigra.taggedView(
numpy.ones((3, 2, 10, 16, 32), dtype="uint8") *
numpy.arange(32, dtype="uint8"), vigra.defaultAxistags("tzcyx"))
op.Input.setValue(data)
class RI:
"""Roi iter class that returns rois over the last dimension in descending order"""
def __init__(self, slot, iterate_axes):
self._shape = slot.meta.shape
def to_index(self, roi):
return roi[0][-1]
def __len__(self):
return self._shape[-1]
def __iter__(self):
for i in range(self._shape[-1], 0, -1):
start = (0, 0, 0, 0) + (i - 1, )
stop = self._shape[:-1] + (i, )
yield start, stop
monkeypatch.setattr(lazyflow.utility.roiRequestBuffer, "_RoiIter", RI)
rb = RoiRequestBufferIter(op.Output, batchsize=2, iterate_axes="tzc")
for i, item in enumerate(rb):
assert item.shape == (3, 2, 10, 16, 1)
assert item.min() == i
assert item.max() == i
def testBasic2(self):
a = numpy.zeros((
2,
2,
2,
))
a[1, 1, 1] = 1
a[0, 0, 0] = 1
a = a[..., None]
a = vigra.taggedView(a, "tyxc")
expected_b = a.mean(axis=0)
expected_b = vigra.taggedView(expected_b, "yxc")
graph = Graph()
op = OpMeanProjectionCached(graph=graph)
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(a)
op.Input.connect(opPrep.Output)
op.Axis.setValue(0)
b = op.Output[...].wait()
b = vigra.taggedView(b, "yxc")
assert ((b == expected_b).all())
def testBasic3(self):
a = numpy.ones((1, 100, 101))
a = a[..., None]
a = vigra.taggedView(a, "tyxc")
r = numpy.array([[0, 0, 0], [1, 3, 4]]).T.copy()
ar = a.copy()
for each_r in r:
nanshe.util.xnumpy.index_axis_at_pos(
nanshe.util.xnumpy.index_axis_at_pos(ar, 0, each_r[0]), -1,
each_r[-1])[:] = 0
graph = Graph()
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(ar)
op = OpNansheRemoveZeroedLinesCached(graph=graph)
op.Input.connect(opPrep.Output)
op.ErosionShape.setValue([21, 1])
op.DilationShape.setValue([1, 3])
b = op.Output[...].wait()
b = vigra.taggedView(b, "tyxc")
assert ((a == b).all())
def testBasic2(self):
a = numpy.ones((100, 101, 102))
a = a[..., None]
a = vigra.taggedView(a, "tyxc")
graph = Graph()
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(a)
op = OpNansheExtractF0Cached(graph=graph)
op.Input.connect(opPrep.Output)
op.HalfWindowSize.setValue(20)
op.WhichQuantile.setValue(0.5)
op.TemporalSmoothingGaussianFilterStdev.setValue(5.0)
op.SpatialSmoothingGaussianFilterStdev.setValue(5.0)
op.Bias.setValue(100)
op.BiasEnabled.setValue(True)
b = op.dF_F[...].wait()
b = vigra.taggedView(b, "tyxc")
assert (a.shape == b.shape)
assert ((b == 0).all())
def testSetInSlot(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0)
sampleData = sampleData.view(vigra.VigraArray)
sampleData.axistags = vigra.defaultAxistags('xyz')
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([100, 75, 50])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:100, 0:75, 0:50]
expectedData = numpy.ones(slicing2shape(slicing), dtype=int)
# This is what we're testing.
#logger.debug("Forcing external data...")
op.Input[slicing] = expectedData
#logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
#logger.debug("Checking data...")
assert (readData == expectedData).all(), "Incorrect output!"
def testBasic2(self):
a = numpy.zeros((
2,
2,
2,
), dtype=int)
a[1, 1, 1] = 1
a[0, 0, 0] = 1
a = a[..., None]
a = vigra.taggedView(a, "tyxc")
expected_b = a.astype(float)
expected_b = vigra.taggedView(expected_b, "tyxc")
graph = Graph()
op = OpConvertTypeCached(graph=graph)
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(a)
op.Input.connect(opPrep.Output)
op.Dtype.setValue(float)
b = op.Output[...].wait()
b = vigra.taggedView(b, "tyxc")
assert ((b == expected_b).all())
def testReconnectWithoutRequest_masked(self):
vol = numpy.ma.zeros((200, 100, 50), dtype=numpy.float32)
vol.set_fill_value(numpy.float32(numpy.nan))
vol[0] = numpy.ma.masked
vol1 = vol
vol2 = vol1.T
graph = Graph()
opData1 = OpArrayPiper(graph=graph)
opData1.Input.meta.has_mask = True
opData1.Input.meta.axistags = vigra.defaultAxistags("xyz")
opData1.Input.setValue(vol1)
op = OpCompressedCache(graph=graph)
op.Input.connect(opData1.Output)
op.BlockShape.setValue((200, 100, 10))
out = op.Output[...].wait()
assert ((out == vol).all() and (out.mask == vol.mask).all() and
((out.fill_value == vol.fill_value) |
(numpy.isnan(out.fill_value)
& numpy.isnan(vol.fill_value))).all()), "Incorrect output!"
op.BlockShape.setValue((50, 100, 10))
# Older versions of OpCompressedCache threw an exception here because
# we tried to access the cache after changing the blockshape.
# But in the current version, we claim that's okay.
out = op.Output[...].wait()
assert ((out == vol).all() and (out.mask == vol.mask).all() and
((out.fill_value == vol.fill_value) |
(numpy.isnan(out.fill_value)
& numpy.isnan(vol.fill_value))).all()), "Incorrect output!"
def test(self):
from lazyflow.graph import Graph, MetaDict
from lazyflow.operators import OpArrayPiper
graph = Graph()
opDataSource = OpArrayPiper(graph=graph)
opDataSource.Input.setValue(numpy.ndarray((9, 10), dtype=numpy.uint8))
# Create some metadata
shape = (1, 2, 3, 4, 5)
data = numpy.indices(shape, dtype=int).sum(0)
meta = MetaDict()
meta.specialdata = "Salutations"
meta.notsospecial = "Hey"
opProvider = OpOutputProvider(data, meta, graph=graph)
op = OpMetadataMerge(graph=graph)
op.Input.connect(opDataSource.Output)
op.MetadataSource.connect(opProvider.Output)
op.FieldsToClone.setValue(["specialdata"])
assert op.Output.ready()
assert op.Output.meta.shape == opDataSource.Output.meta.shape
assert op.Output.meta.dtype == opDataSource.Output.meta.dtype
assert op.Output.meta.specialdata == meta.specialdata
assert op.Output.meta.notsospecial is None
def testCleanup(self, cacheMemoryManager):
try:
cacheMemoryManager.disable()
sampleData = np.random.randint(0, 256, size=(50, 30, 10))
sampleData = sampleData.astype(np.uint8)
sampleData = vigra.taggedView(sampleData, axistags="xyz")
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.setValue(sampleData)
op = OpLabelVolume(graph=graph)
op.Input.connect(opData.Output)
x = op.Output[...].wait()
op.Input.disconnect()
op.cleanUp()
r = weakref.ref(op)
del op
gc.collect()
ref = r()
if ref is not None:
for i, o in enumerate(gc.get_referrers(ref)):
print("Object", i, ":", type(o), ":", o)
assert r(
) is None, "OpBlockedArrayCache was not cleaned up correctly"
finally:
cacheMemoryManager.enable()
def testBasic(self):
data = numpy.random.random((100, 100)).astype(numpy.float32)
data = vigra.taggedView(data, vigra.defaultAxistags('xy'))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opWriter = OpNpyWriter(graph=graph)
opWriter.Input.connect(opPiper.Output)
opWriter.Filepath.setValue(self._tmpdir +
'/npy_writer_test_output.npy')
# Write it...
opWriter.write()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opWriter.Filepath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output[:].wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def test_via_OpExportSlot(self):
data = 255 * numpy.random.random((64, 128, 128, 1))
data = data.astype(numpy.uint8)
data = vigra.taggedView(data, vigra.defaultAxistags("zyxc"))
graph = Graph()
opPiper = OpArrayPiper(graph=graph)
opPiper.Input.setValue(data)
opExport = OpExportSlot(graph=graph)
opExport.Input.connect(opPiper.Output)
opExport.OutputFormat.setValue("dvid")
url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format(
**self.__dict__)
opExport.OutputFilenameFormat.setValue(url)
assert opExport.ExportPath.ready()
assert opExport.ExportPath.value == url
opExport.run_export()
opRead = OpInputDataReader(graph=graph)
try:
opRead.FilePath.setValue(opExport.ExportPath.value)
expected_data = data.view(numpy.ndarray)
read_data = opRead.Output(*roiFromShape(data.shape)).wait()
assert (read_data == expected_data
).all(), "Read data didn't match exported data!"
finally:
opRead.cleanUp()
def setup_method(self, method):
self.testVol = vigra.VigraArray((200, 200, 200))
self.testVol[:] = numpy.random.rand(200, 200, 200)
self.graph = Graph()
self.op = OpArrayPiper(graph=self.graph)
self.op.inputs["Input"].setValue(self.testVol)
self.roiOp = OpRoiTest(graph=self.graph)
self.roiOp.inputs["input"].setValue(self.testVol)
def test_OpNansheDictionaryLearning(self):
p = numpy.array([[27, 51], [66, 85], [77, 45]])
space = numpy.array((100, 100))
radii = numpy.array((5, 6, 7))
g = nanshe.syn.data.generate_hypersphere_masks(space, p, radii)
gv = g[..., None]
gv = gv.astype(float)
gv = vigra.taggedView(gv, "tyxc")
graph = Graph()
op = OpNansheDictionaryLearning(graph=graph)
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(gv)
op.Input.connect(opPrep.Output)
op.K.setValue(len(g))
op.Gamma1.setValue(0)
op.Gamma2.setValue(0)
op.NumThreads.setValue(1)
op.Batchsize.setValue(256)
op.NumIter.setValue(10)
op.Lambda1.setValue(0.2)
op.Lambda2.setValue(0)
op.PosAlpha.setValue(True)
op.PosD.setValue(True)
op.Clean.setValue(True)
op.Mode.setValue(2)
op.ModeD.setValue(0)
d = op.Output[...].wait()
d = (d != 0)
assert (g.shape == d.shape)
assert ((g.astype(bool).max(axis=0) == d.astype(bool).max(
axis=0)).all())
unmatched_g = range(len(g))
matched = dict()
for i in xrange(len(d)):
new_unmatched_g = []
for j in unmatched_g:
if not (d[i] == g[j]).all():
new_unmatched_g.append(j)
else:
matched[i] = j
unmatched_g = new_unmatched_g
print(unmatched_g)
assert (len(unmatched_g) == 0)
def testChangeBlockshape_masked(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0)
sampleData = sampleData.view(numpy.ma.masked_array)
sampleData.set_fill_value(numpy.float32(numpy.nan))
sampleData[0] = numpy.ma.masked
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.meta.has_mask = True
opData.Input.meta.axistags = vigra.defaultAxistags('xyz')
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([100, 75, 50])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:100, 50:150, 75:150]
expectedData = sampleData[slicing]
#logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
#logger.debug("Checking data...")
assert (readData == expectedData).all() and \
(readData.mask == expectedData.mask).all() and \
((readData.fill_value == expectedData.fill_value) |
(numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\
"Incorrect output!"
# Now change the blockshape and the input and try again...
sampleDataWithChannel = sampleData[..., None]
opData.Input.meta.axistags = vigra.defaultAxistags('xyzc')
opData.Input.setValue(sampleDataWithChannel)
op.BlockShape.setValue([45, 33, 40, 1])
assert op.Output.ready()
slicing = numpy.s_[60:70, 50:110, 60:120, 0:1]
expectedData = sampleDataWithChannel[slicing]
#logger.debug("Requesting data...")
readData = op.Output[slicing].wait()
#logger.debug("Checking data...")
assert (readData == expectedData).all() and \
(readData.mask == expectedData.mask).all() and \
((readData.fill_value == expectedData.fill_value) |
(numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\
"Incorrect output!"
def setup_method(self, method):
graph = Graph()
self.graph = graph
# Data is tagged by channel
data = numpy.indices((10,10,10,3))[3]
data = data.view(vigra.VigraArray)
data.axistags = vigra.defaultAxistags('xyzc')
self.opProvider = OpArrayPiper(graph=graph)
self.opProvider.Input.setValue(data)
self.opSlicer = OpMultiArraySlicer2(graph=graph)
self.opSlicer.AxisFlag.setValue('c')
self.opSlicer.Input.connect(self.opProvider.Output)
def testMultiThread_masked(self):
logger.info("Generating sample data...")
sampleData = numpy.indices((3, 100, 200, 150, 2),
dtype=numpy.float32).sum(0)
sampleData = sampleData.view(numpy.ma.masked_array)
sampleData.set_fill_value(numpy.float32(numpy.nan))
sampleData[0] = numpy.ma.masked
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.meta.has_mask = True
opData.Input.meta.axistags = vigra.defaultAxistags("txyzc")
opData.Input.setValue(sampleData)
op = OpCompressedCache(parent=None, graph=graph)
# logger.debug("Setting block shape...")
op.BlockShape.setValue([1, 100, 75, 50, 2])
op.Input.connect(opData.Output)
assert op.Output.ready()
slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1]
expectedData = sampleData[slicing]
results = {}
def readData(resultIndex):
results[resultIndex] = op.Output[slicing].wait()
threads = []
for i in range(10):
threads.append(
threading.Thread(target=functools.partial(readData, i)))
for th in threads:
th.start()
for th in threads:
th.join()
assert len(results) == len(threads), "Didn't get all results."
# logger.debug("Checking data...")
for i, data in list(results.items()):
assert ((data == expectedData).all()
and (data.mask == expectedData.mask).all()
and ((data.fill_value == expectedData.fill_value)
| (numpy.isnan(data.fill_value)
& numpy.isnan(expectedData.fill_value))).all()
), "Incorrect output for index {}".format(i)
def testBasic(self):
op = OpArrayPiper(graph=Graph())
inputData = numpy.indices((100, 100)).sum(0)
op.Input.setValue(inputData)
roiList = []
block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100]))
for block_start in block_starts:
roiList.append(getBlockBounds([100, 100], [10, 10], block_start))
results = numpy.zeros((100, 100), dtype=numpy.int32)
resultslock = threading.Lock()
resultsCount = [0]
def handleResult(roi, result):
acquired = resultslock.acquire(False)
assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized."
results[roiToSlice(*roi)] = result
logger.debug("Got result for {}".format(roi))
resultslock.release()
resultsCount[0] += 1
progressList = []
def handleProgress(progress):
progressList.append(progress)
logger.debug("Progress update: {}".format(progress))
totalVolume = numpy.prod(inputData.shape)
batch = RoiRequestBatch(op.Output,
roiList.__iter__(),
totalVolume,
batchSize=10,
allowParallelResults=False)
batch.resultSignal.subscribe(handleResult)
batch.progressSignal.subscribe(handleProgress)
batch.execute()
logger.debug("Got {} results".format(resultsCount[0]))
assert (results == inputData).all()
# Progress reporting MUST start with 0 and end with 100
assert progressList[0] == 0, "Invalid progress reporting."
assert progressList[-1] == 100, "Invalid progress reporting."
# There should be some intermediate progress reporting, but exactly how much is unspecified.
assert len(progressList) >= 10
logger.debug("FINISHED")
def testOutput(self):
graph = Graph()
data = numpy.random.random((1, 100, 100, 10, 1))
opProvider = OpArrayPiper(graph=graph)
opProvider.Input.setValue(data)
opSubRegion = OpSubRegion(graph=graph)
opSubRegion.Input.connect(opProvider.Output)
opSubRegion.Start.setValue((0, 20, 30, 5, 0))
opSubRegion.Stop.setValue((1, 30, 50, 8, 1))
subData = opSubRegion.Output(start=(0, 5, 10, 1, 0),
stop=(1, 10, 20, 3, 1)).wait()
assert (subData == data[0:1, 25:30, 40:50, 6:8, 0:1]).all()
def testCleanup(self):
sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0)
sampleData = vigra.taggedView(sampleData, axistags='xyz')
graph = Graph()
opData = OpArrayPiper(graph=graph)
opData.Input.setValue(sampleData)
op = OpCompressedCache(graph=graph)
#logger.debug("Setting block shape...")
op.BlockShape.setValue([100, 75, 50])
op.Input.connect(opData.Output)
x = op.Output[...].wait()
op.Input.disconnect()
r = weakref.ref(op)
del op
gc.collect()
assert r() is None, "OpBlockedArrayCache was not cleaned up correctly"
def testBasic2(self):
a = numpy.arange(256)
a = expand_view(a, 256)
a = a[..., None]
a = vigra.taggedView(a, "xyc")
graph = Graph()
op = OpColorizeLabelImageCached(graph=graph)
opPrep = OpArrayPiper(graph=graph)
opPrep.Input.setValue(a)
op.Input.connect(opPrep.Output)
b = op.Output[...].wait()
b_colors = set([tuple(_) for _ in list(array_to_matrix(b.T).T)])
assert (len(b_colors) == 256)
data_unknown_meta_tag = op.Output.meta.whatsthis assert data_shape == input_array.shape assert data_dtype == input_array.dtype assert data_unknown_meta_tag is None # we could use dict queries, but it's not that comfortable assert op.Output.meta.shape is op.Output.meta["shape"] """ Now that we know how to query a single operator, it's time for building a chain of operators. """ graph = Graph() op1 = OpArrayPiper(graph=graph) op1.name = "OpArrayPiper1" op2 = OpArrayPiper(graph=graph) op2.name = "OpArrayPiper2" op3 = OpArrayPiper(graph=graph) op3.name = "OpArrayPiper3" op3.Input.connect(op2.Output) op2.Input.connect(op1.Output) op1.Input.setValue(input_array) """ We connected three operators to a chain. Three times no operation still equals no operation, but we just wanted to see how the connections are done. Operators are always connected from end point to start point (downstream to upstream).
