def testHDF5(self): logger.info("Generating sample data...") sampleData = numpy.indices((150, 250, 150), dtype=numpy.float32).sum(0) sampleData = sampleData.view(vigra.VigraArray) sampleData.axistags = vigra.defaultAxistags('xyz') graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) #logger.debug("Setting block shape...") op.BlockShape.setValue([75, 125, 150]) op.Input.connect(opData.Output) assert op.OutputHdf5.ready() slicing = numpy.s_[0:75, 125:250, 0:150] slicing_str = str( [list(_) for _ in zip(*[[_.start, _.stop] for _ in slicing])]) expectedData = sampleData[slicing].view(numpy.ndarray) slicing_2 = numpy.s_[0:75, 0:125, 0:150] expectedData_2 = expectedData[slicing_2].view(numpy.ndarray) #logger.debug("Requesting data...") tempdir = tempfile.mkdtemp() try: with h5py.File(os.path.join(tempdir, "data.h5"), "w") as h5_file: op.OutputHdf5[slicing].writeInto(h5_file).wait() assert slicing_str in h5_file, "Missing dataset!" assert (h5_file[slicing_str][()] == expectedData ).all(), "Incorrect output!" with h5py.File(os.path.join(tempdir, "data.h5"), "r") as h5_file: graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.meta.axistags = vigra.AxisTags('xyz') opData.Input.setValue(numpy.empty_like(expectedData_2)) op = OpCompressedCache(parent=None, graph=graph) op.InputHdf5.meta.axistags = vigra.AxisTags('xyz') op.InputHdf5.meta.shape = (75, 125, 150) #logger.debug("Setting block shape...") op.BlockShape.setValue([75, 125, 150]) op.Input.connect(opData.Output) op.InputHdf5[slicing_2] = h5_file[slicing_str] result = op.Output[slicing_2].wait() assert (result == expectedData_2).all(), "Incorrect output!" finally: shutil.rmtree(tempdir)
def testDirtyPropagation(self): graph = Graph() data = numpy.random.random((1, 100, 100, 10, 1)) opProvider = OpArrayPiper(graph=graph) opProvider.Input.setValue(data) opSubRegion = OpSubRegion(graph=graph) opSubRegion.Input.connect(opProvider.Output) opSubRegion.Start.setValue((0, 20, 30, 5, 0)) opSubRegion.Stop.setValue((1, 30, 50, 8, 1)) gotDirtyRois = [] def handleDirty(slot, roi): gotDirtyRois.append(roi) opSubRegion.Output.notifyDirty(handleDirty) # Set an input dirty region that overlaps with the subregion key = make_key[0:1, 15:35, 32:33, 0:10, 0:1] opProvider.Input.setDirty(key) assert len(gotDirtyRois) == 1 assert gotDirtyRois[0].start == [0, 0, 2, 0, 0] assert gotDirtyRois[0].stop == [1, 10, 3, 3, 1] # Now mark a region that DOESN'T overlap with the subregion key = make_key[0:1, 70:80, 32:33, 0:10, 0:1] opProvider.Input.setDirty(key) # Should have gotten no extra dirty notifications assert len(gotDirtyRois) == 1
def testBasic1(self): space = numpy.array([100, 100]) radii = numpy.array([7, 6, 6, 6, 7, 6]) magnitudes = numpy.array([15, 16, 15, 17, 16, 16]) points = numpy.array([[30, 24], [59, 65], [21, 65], [13, 12], [72, 16], [45, 32]]) masks = nanshe.syn.data.generate_hypersphere_masks( space, points, radii) images = nanshe.syn.data.generate_gaussian_images( space, points, radii / 3.0, magnitudes) images *= masks bases_indices = [[1, 3, 4], [0, 2], [5]] bases_masks = numpy.zeros((len(bases_indices), ) + masks.shape[1:], dtype=masks.dtype) bases_images = numpy.zeros((len(bases_indices), ) + images.shape[1:], dtype=images.dtype) for i, each_basis_indices in enumerate(bases_indices): bases_masks[i] = masks[list(each_basis_indices)].max(axis=0) bases_images[i] = images[list(each_basis_indices)].max(axis=0) bases_images = vigra.taggedView(bases_images, "cyx") graph = Graph() opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(bases_images) op = OpNanshePostprocessData(graph=graph) op.Input.connect(opPrep.Output) op.SignificanceThreshold.setValue(3.0) op.WaveletTransformScale.setValue(4) op.NoiseThreshold.setValue(3.0) op.AcceptedRegionShapeConstraints_MajorAxisLength_Min.setValue(0.0) op.AcceptedRegionShapeConstraints_MajorAxisLength_Min_Enabled.setValue( True) op.AcceptedRegionShapeConstraints_MajorAxisLength_Max.setValue(25.0) op.AcceptedRegionShapeConstraints_MajorAxisLength_Max_Enabled.setValue( True) op.PercentagePixelsBelowMax.setValue(0.0) op.MinLocalMaxDistance.setValue(10.0) op.AcceptedNeuronShapeConstraints_Area_Min.setValue(30) op.AcceptedNeuronShapeConstraints_Area_Min_Enabled.setValue(True) op.AcceptedNeuronShapeConstraints_Area_Max.setValue(600) op.AcceptedNeuronShapeConstraints_Area_Max_Enabled.setValue(True) op.AcceptedNeuronShapeConstraints_Eccentricity_Min.setValue(0.0) op.AcceptedNeuronShapeConstraints_Eccentricity_Min_Enabled.setValue( True) op.AcceptedNeuronShapeConstraints_Eccentricity_Max.setValue(0.9) op.AcceptedNeuronShapeConstraints_Eccentricity_Max_Enabled.setValue( True) op.AlignmentMinThreshold.setValue(0.6) op.OverlapMinThreshold.setValue(0.6) op.Fuse_FractionMeanNeuronMaxThreshold.setValue(0.6) neurons = op.Output[...].wait()
def test_export(self): # For now, we require block-aligned POST data = numpy.random.randint(0, 255, (32, 128, 256, 1)).astype(numpy.uint8) data = numpy.asfortranarray(data, numpy.uint8) assert data.shape == (32, 128, 256, 1) data = data.astype(numpy.uint8) data = vigra.taggedView(data, vigra.defaultAxistags("zyxc")) # Retrieve from server graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue(data) opExport = OpExportDvidVolume(transpose_axes=True, graph=graph) # Reverse data order for dvid export opExport.Input.connect(opPiper.Output) opExport.NodeDataUrl.setValue( "http://localhost:8000/api/node/{uuid}/{dataname}".format( uuid=self.data_uuid, dataname=self.data_name)) # Export! opExport.run_export() # Read back. (transposed, because of transposed_axes, above) accessor = VoxelsAccessor(TEST_DVID_SERVER, self.data_uuid, self.data_name) read_data = accessor[:] # Compare. assert (data.view(numpy.ndarray) == read_data.transpose() ).all(), "Exported data is not correct"
def testReshape(self): opSlicer = self.opSlicer dirtyRois = {} def handleDirty(i, slot, roi): assert opSlicer.Slices[i] == slot dirtyRois[i] = roi for i, slot in enumerate(opSlicer.Slices): slot.notifyDirty(partial(handleDirty, i)) # Initial data has 3 channels with values 0,1,2 # Simulate removing the middle one: Now data has two channels with values 0,2 data = numpy.indices((10, 10, 10, 2))[3] data = data * 2 data = data.view(vigra.VigraArray) data.axistags = vigra.defaultAxistags('xyzc') # Connect a new data provider opNewProvider = OpArrayPiper(graph=opSlicer.graph) opNewProvider.Input.setValue(data) opSlicer.Input.connect(opNewProvider.Output) assert len(opSlicer.Slices) == 2 for i, slot in enumerate(opSlicer.Slices): assert slot.meta.shape == (10, 10, 10, 1) assert (slot[...].wait() == i * 2).all() assert len(dirtyRois) == 2 assert dirtyRois[0].start == [0, 0, 0, 0] assert dirtyRois[0].stop == [10, 10, 10, 1] assert dirtyRois[1].start == [0, 0, 0, 0] assert dirtyRois[1].stop == [10, 10, 10, 1]
def testBasic2(self): a = numpy.eye(3, dtype = numpy.float32) a = a[None, ..., None] a = vigra.taggedView(a, "tyxc") expected_b = numpy.array([[ 0.59375, -0.375 , -0.34375], [-0.375 , 0.625 , -0.375 ], [-0.34375, -0.375 , 0.59375]], dtype=numpy.float32) expected_b = expected_b[None, ..., None] expected_b = vigra.taggedView(expected_b, "tyxc") graph = Graph() opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(a) op = OpNansheWaveletTransform(graph=graph) op.Input.connect(opPrep.Output) op.Scale.setValue((0, 1, 1)) b = op.Output[...].wait() b = vigra.taggedView(b, "tyxc") assert((b == expected_b).all())
def testBasicOneBlock(self): logger.info("Generating sample data...") sampleData = numpy.indices((3, 100, 200, 150, 2), dtype=numpy.float32).sum(0) sampleData = sampleData.view(vigra.VigraArray) sampleData.axistags = vigra.defaultAxistags('txyzc') graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) # NO Block shape for this test. #op.BlockShape.setValue( [1, 100, 75, 50, 2] ) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1] expectedData = sampleData[slicing].view(numpy.ndarray) #logger.debug("Requesting data...") readData = op.Output[slicing].wait() #logger.debug("Checking data...") assert (readData == expectedData).all(), "Incorrect output!"
def testFailedProcessing(self): op = OpArrayPiper(graph=Graph()) inputData = numpy.indices((100, 100)).sum(0) op.Input.setValue(inputData) roiList = [] block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100])) for block_start in block_starts: roiList.append(getBlockBounds([100, 100], [10, 10], block_start)) class SpecialException(Exception): pass def handleResult(roi, result): raise SpecialException( "Intentional Exception: raised while handling the result") totalVolume = numpy.prod(inputData.shape) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False) batch.resultSignal.subscribe(handleResult) # FIXME: There are multiple places where the RoiRequestBatch tool should be prepared to handle exceptions. # This only tests one of them (in the notify_finished() handler) try: batch.execute() except SpecialException: pass else: assert False, "Expected exception to be propagated out of the RoiRequestBatch."
def test_export(self): """ hostname: The dvid server host h5filename: The h5 file to compare against h5group: The hdf5 group, also used as the uuid of the dvid dataset h5dataset: The dataset name, also used as the name of the dvid dataset start, stop: The bounds of the cutout volume to retrieve from the server. C ORDER FOR THIS TEST BECAUSE we use transpose_axes=True. """ data = numpy.indices( (10, 100, 200, 4) ) assert data.shape == (4, 10, 100, 200, 4) data = data.astype( numpy.uint8 ) data = vigra.taggedView( data, vigra.defaultAxistags('tzyxc') ) # Retrieve from server graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue( data ) opExport = OpExportDvidVolume( transpose_axes=True, graph=graph ) # Reverse data order for dvid export opExport.Input.connect( opPiper.Output ) opExport.NodeDataUrl.setValue( 'http://localhost:8000/api/node/{uuid}/{dataname}'.format( uuid=self.data_uuid, dataname=self.data_name ) ) # Export! opExport.run_export() # Retrieve from file with h5py.File(self.test_filepath, 'r') as f: exported_data = f['all_nodes'][self.data_uuid][self.data_name][:] # Compare. assert ( data.view(numpy.ndarray) == exported_data.transpose() ).all(),\ "Exported data is not correct"
def testBasicOneBlock_masked(self): logger.info("Generating sample data...") sampleData = numpy.indices((3, 100, 200, 150, 2), dtype=numpy.float32).sum(0) sampleData = sampleData.view(numpy.ma.masked_array) sampleData.set_fill_value(numpy.float32(numpy.nan)) sampleData[0] = numpy.ma.masked graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.meta.has_mask = True opData.Input.meta.axistags = vigra.defaultAxistags('txyzc') opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) # NO Block shape for this test. #op.BlockShape.setValue( [1, 100, 75, 50, 2] ) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1] expectedData = sampleData[slicing] #logger.debug("Requesting data...") readData = op.Output[slicing].wait() #logger.debug("Checking data...") assert (readData == expectedData).all() and \ (readData.mask == expectedData.mask).all() and \ ((readData.fill_value == expectedData.fill_value) | (numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\ "Incorrect output!"
def test_in_ascending_order(graph, monkeypatch): """Make sure iterator returns data slices in ascending order""" op = OpArrayPiper(graph=graph) data = vigra.taggedView( numpy.ones((3, 2, 10, 16, 32), dtype="uint8") * numpy.arange(32, dtype="uint8"), vigra.defaultAxistags("tzcyx")) op.Input.setValue(data) class RI: """Roi iter class that returns rois over the last dimension in descending order""" def __init__(self, slot, iterate_axes): self._shape = slot.meta.shape def to_index(self, roi): return roi[0][-1] def __len__(self): return self._shape[-1] def __iter__(self): for i in range(self._shape[-1], 0, -1): start = (0, 0, 0, 0) + (i - 1, ) stop = self._shape[:-1] + (i, ) yield start, stop monkeypatch.setattr(lazyflow.utility.roiRequestBuffer, "_RoiIter", RI) rb = RoiRequestBufferIter(op.Output, batchsize=2, iterate_axes="tzc") for i, item in enumerate(rb): assert item.shape == (3, 2, 10, 16, 1) assert item.min() == i assert item.max() == i
def testBasic2(self): a = numpy.zeros(( 2, 2, 2, )) a[1, 1, 1] = 1 a[0, 0, 0] = 1 a = a[..., None] a = vigra.taggedView(a, "tyxc") expected_b = a.mean(axis=0) expected_b = vigra.taggedView(expected_b, "yxc") graph = Graph() op = OpMeanProjectionCached(graph=graph) opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(a) op.Input.connect(opPrep.Output) op.Axis.setValue(0) b = op.Output[...].wait() b = vigra.taggedView(b, "yxc") assert ((b == expected_b).all())
def testBasic3(self): a = numpy.ones((1, 100, 101)) a = a[..., None] a = vigra.taggedView(a, "tyxc") r = numpy.array([[0, 0, 0], [1, 3, 4]]).T.copy() ar = a.copy() for each_r in r: nanshe.util.xnumpy.index_axis_at_pos( nanshe.util.xnumpy.index_axis_at_pos(ar, 0, each_r[0]), -1, each_r[-1])[:] = 0 graph = Graph() opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(ar) op = OpNansheRemoveZeroedLinesCached(graph=graph) op.Input.connect(opPrep.Output) op.ErosionShape.setValue([21, 1]) op.DilationShape.setValue([1, 3]) b = op.Output[...].wait() b = vigra.taggedView(b, "tyxc") assert ((a == b).all())
def testBasic2(self): a = numpy.ones((100, 101, 102)) a = a[..., None] a = vigra.taggedView(a, "tyxc") graph = Graph() opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(a) op = OpNansheExtractF0Cached(graph=graph) op.Input.connect(opPrep.Output) op.HalfWindowSize.setValue(20) op.WhichQuantile.setValue(0.5) op.TemporalSmoothingGaussianFilterStdev.setValue(5.0) op.SpatialSmoothingGaussianFilterStdev.setValue(5.0) op.Bias.setValue(100) op.BiasEnabled.setValue(True) b = op.dF_F[...].wait() b = vigra.taggedView(b, "tyxc") assert (a.shape == b.shape) assert ((b == 0).all())
def testSetInSlot(self): logger.info("Generating sample data...") sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0) sampleData = sampleData.view(vigra.VigraArray) sampleData.axistags = vigra.defaultAxistags('xyz') graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) #logger.debug("Setting block shape...") op.BlockShape.setValue([100, 75, 50]) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:100, 0:75, 0:50] expectedData = numpy.ones(slicing2shape(slicing), dtype=int) # This is what we're testing. #logger.debug("Forcing external data...") op.Input[slicing] = expectedData #logger.debug("Requesting data...") readData = op.Output[slicing].wait() #logger.debug("Checking data...") assert (readData == expectedData).all(), "Incorrect output!"
def testBasic2(self): a = numpy.zeros(( 2, 2, 2, ), dtype=int) a[1, 1, 1] = 1 a[0, 0, 0] = 1 a = a[..., None] a = vigra.taggedView(a, "tyxc") expected_b = a.astype(float) expected_b = vigra.taggedView(expected_b, "tyxc") graph = Graph() op = OpConvertTypeCached(graph=graph) opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(a) op.Input.connect(opPrep.Output) op.Dtype.setValue(float) b = op.Output[...].wait() b = vigra.taggedView(b, "tyxc") assert ((b == expected_b).all())
def testReconnectWithoutRequest_masked(self): vol = numpy.ma.zeros((200, 100, 50), dtype=numpy.float32) vol.set_fill_value(numpy.float32(numpy.nan)) vol[0] = numpy.ma.masked vol1 = vol vol2 = vol1.T graph = Graph() opData1 = OpArrayPiper(graph=graph) opData1.Input.meta.has_mask = True opData1.Input.meta.axistags = vigra.defaultAxistags("xyz") opData1.Input.setValue(vol1) op = OpCompressedCache(graph=graph) op.Input.connect(opData1.Output) op.BlockShape.setValue((200, 100, 10)) out = op.Output[...].wait() assert ((out == vol).all() and (out.mask == vol.mask).all() and ((out.fill_value == vol.fill_value) | (numpy.isnan(out.fill_value) & numpy.isnan(vol.fill_value))).all()), "Incorrect output!" op.BlockShape.setValue((50, 100, 10)) # Older versions of OpCompressedCache threw an exception here because # we tried to access the cache after changing the blockshape. # But in the current version, we claim that's okay. out = op.Output[...].wait() assert ((out == vol).all() and (out.mask == vol.mask).all() and ((out.fill_value == vol.fill_value) | (numpy.isnan(out.fill_value) & numpy.isnan(vol.fill_value))).all()), "Incorrect output!"
def test(self): from lazyflow.graph import Graph, MetaDict from lazyflow.operators import OpArrayPiper graph = Graph() opDataSource = OpArrayPiper(graph=graph) opDataSource.Input.setValue(numpy.ndarray((9, 10), dtype=numpy.uint8)) # Create some metadata shape = (1, 2, 3, 4, 5) data = numpy.indices(shape, dtype=int).sum(0) meta = MetaDict() meta.specialdata = "Salutations" meta.notsospecial = "Hey" opProvider = OpOutputProvider(data, meta, graph=graph) op = OpMetadataMerge(graph=graph) op.Input.connect(opDataSource.Output) op.MetadataSource.connect(opProvider.Output) op.FieldsToClone.setValue(["specialdata"]) assert op.Output.ready() assert op.Output.meta.shape == opDataSource.Output.meta.shape assert op.Output.meta.dtype == opDataSource.Output.meta.dtype assert op.Output.meta.specialdata == meta.specialdata assert op.Output.meta.notsospecial is None
def testCleanup(self, cacheMemoryManager): try: cacheMemoryManager.disable() sampleData = np.random.randint(0, 256, size=(50, 30, 10)) sampleData = sampleData.astype(np.uint8) sampleData = vigra.taggedView(sampleData, axistags="xyz") graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.setValue(sampleData) op = OpLabelVolume(graph=graph) op.Input.connect(opData.Output) x = op.Output[...].wait() op.Input.disconnect() op.cleanUp() r = weakref.ref(op) del op gc.collect() ref = r() if ref is not None: for i, o in enumerate(gc.get_referrers(ref)): print("Object", i, ":", type(o), ":", o) assert r( ) is None, "OpBlockedArrayCache was not cleaned up correctly" finally: cacheMemoryManager.enable()
def testBasic(self): data = numpy.random.random((100, 100)).astype(numpy.float32) data = vigra.taggedView(data, vigra.defaultAxistags('xy')) graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue(data) opWriter = OpNpyWriter(graph=graph) opWriter.Input.connect(opPiper.Output) opWriter.Filepath.setValue(self._tmpdir + '/npy_writer_test_output.npy') # Write it... opWriter.write() opRead = OpInputDataReader(graph=graph) try: opRead.FilePath.setValue(opWriter.Filepath.value) expected_data = data.view(numpy.ndarray) read_data = opRead.Output[:].wait() assert (read_data == expected_data ).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def test_via_OpExportSlot(self): data = 255 * numpy.random.random((64, 128, 128, 1)) data = data.astype(numpy.uint8) data = vigra.taggedView(data, vigra.defaultAxistags("zyxc")) graph = Graph() opPiper = OpArrayPiper(graph=graph) opPiper.Input.setValue(data) opExport = OpExportSlot(graph=graph) opExport.Input.connect(opPiper.Output) opExport.OutputFormat.setValue("dvid") url = "http://{hostname}/api/node/{data_uuid}/{data_name}".format( **self.__dict__) opExport.OutputFilenameFormat.setValue(url) assert opExport.ExportPath.ready() assert opExport.ExportPath.value == url opExport.run_export() opRead = OpInputDataReader(graph=graph) try: opRead.FilePath.setValue(opExport.ExportPath.value) expected_data = data.view(numpy.ndarray) read_data = opRead.Output(*roiFromShape(data.shape)).wait() assert (read_data == expected_data ).all(), "Read data didn't match exported data!" finally: opRead.cleanUp()
def setup_method(self, method): self.testVol = vigra.VigraArray((200, 200, 200)) self.testVol[:] = numpy.random.rand(200, 200, 200) self.graph = Graph() self.op = OpArrayPiper(graph=self.graph) self.op.inputs["Input"].setValue(self.testVol) self.roiOp = OpRoiTest(graph=self.graph) self.roiOp.inputs["input"].setValue(self.testVol)
def test_OpNansheDictionaryLearning(self): p = numpy.array([[27, 51], [66, 85], [77, 45]]) space = numpy.array((100, 100)) radii = numpy.array((5, 6, 7)) g = nanshe.syn.data.generate_hypersphere_masks(space, p, radii) gv = g[..., None] gv = gv.astype(float) gv = vigra.taggedView(gv, "tyxc") graph = Graph() op = OpNansheDictionaryLearning(graph=graph) opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(gv) op.Input.connect(opPrep.Output) op.K.setValue(len(g)) op.Gamma1.setValue(0) op.Gamma2.setValue(0) op.NumThreads.setValue(1) op.Batchsize.setValue(256) op.NumIter.setValue(10) op.Lambda1.setValue(0.2) op.Lambda2.setValue(0) op.PosAlpha.setValue(True) op.PosD.setValue(True) op.Clean.setValue(True) op.Mode.setValue(2) op.ModeD.setValue(0) d = op.Output[...].wait() d = (d != 0) assert (g.shape == d.shape) assert ((g.astype(bool).max(axis=0) == d.astype(bool).max( axis=0)).all()) unmatched_g = range(len(g)) matched = dict() for i in xrange(len(d)): new_unmatched_g = [] for j in unmatched_g: if not (d[i] == g[j]).all(): new_unmatched_g.append(j) else: matched[i] = j unmatched_g = new_unmatched_g print(unmatched_g) assert (len(unmatched_g) == 0)
def testChangeBlockshape_masked(self): logger.info("Generating sample data...") sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0) sampleData = sampleData.view(numpy.ma.masked_array) sampleData.set_fill_value(numpy.float32(numpy.nan)) sampleData[0] = numpy.ma.masked graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.meta.has_mask = True opData.Input.meta.axistags = vigra.defaultAxistags('xyz') opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) #logger.debug("Setting block shape...") op.BlockShape.setValue([100, 75, 50]) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:100, 50:150, 75:150] expectedData = sampleData[slicing] #logger.debug("Requesting data...") readData = op.Output[slicing].wait() #logger.debug("Checking data...") assert (readData == expectedData).all() and \ (readData.mask == expectedData.mask).all() and \ ((readData.fill_value == expectedData.fill_value) | (numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\ "Incorrect output!" # Now change the blockshape and the input and try again... sampleDataWithChannel = sampleData[..., None] opData.Input.meta.axistags = vigra.defaultAxistags('xyzc') opData.Input.setValue(sampleDataWithChannel) op.BlockShape.setValue([45, 33, 40, 1]) assert op.Output.ready() slicing = numpy.s_[60:70, 50:110, 60:120, 0:1] expectedData = sampleDataWithChannel[slicing] #logger.debug("Requesting data...") readData = op.Output[slicing].wait() #logger.debug("Checking data...") assert (readData == expectedData).all() and \ (readData.mask == expectedData.mask).all() and \ ((readData.fill_value == expectedData.fill_value) | (numpy.isnan(readData.fill_value) & numpy.isnan(expectedData.fill_value))).all(),\ "Incorrect output!"
def setup_method(self, method): graph = Graph() self.graph = graph # Data is tagged by channel data = numpy.indices((10,10,10,3))[3] data = data.view(vigra.VigraArray) data.axistags = vigra.defaultAxistags('xyzc') self.opProvider = OpArrayPiper(graph=graph) self.opProvider.Input.setValue(data) self.opSlicer = OpMultiArraySlicer2(graph=graph) self.opSlicer.AxisFlag.setValue('c') self.opSlicer.Input.connect(self.opProvider.Output)
def testMultiThread_masked(self): logger.info("Generating sample data...") sampleData = numpy.indices((3, 100, 200, 150, 2), dtype=numpy.float32).sum(0) sampleData = sampleData.view(numpy.ma.masked_array) sampleData.set_fill_value(numpy.float32(numpy.nan)) sampleData[0] = numpy.ma.masked graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.meta.has_mask = True opData.Input.meta.axistags = vigra.defaultAxistags("txyzc") opData.Input.setValue(sampleData) op = OpCompressedCache(parent=None, graph=graph) # logger.debug("Setting block shape...") op.BlockShape.setValue([1, 100, 75, 50, 2]) op.Input.connect(opData.Output) assert op.Output.ready() slicing = numpy.s_[0:2, 0:100, 50:150, 75:150, 0:1] expectedData = sampleData[slicing] results = {} def readData(resultIndex): results[resultIndex] = op.Output[slicing].wait() threads = [] for i in range(10): threads.append( threading.Thread(target=functools.partial(readData, i))) for th in threads: th.start() for th in threads: th.join() assert len(results) == len(threads), "Didn't get all results." # logger.debug("Checking data...") for i, data in list(results.items()): assert ((data == expectedData).all() and (data.mask == expectedData.mask).all() and ((data.fill_value == expectedData.fill_value) | (numpy.isnan(data.fill_value) & numpy.isnan(expectedData.fill_value))).all() ), "Incorrect output for index {}".format(i)
def testBasic(self): op = OpArrayPiper(graph=Graph()) inputData = numpy.indices((100, 100)).sum(0) op.Input.setValue(inputData) roiList = [] block_starts = getIntersectingBlocks([10, 10], ([0, 0], [100, 100])) for block_start in block_starts: roiList.append(getBlockBounds([100, 100], [10, 10], block_start)) results = numpy.zeros((100, 100), dtype=numpy.int32) resultslock = threading.Lock() resultsCount = [0] def handleResult(roi, result): acquired = resultslock.acquire(False) assert acquired, "resultslock is contested! Access to callback is supposed to be automatically serialized." results[roiToSlice(*roi)] = result logger.debug("Got result for {}".format(roi)) resultslock.release() resultsCount[0] += 1 progressList = [] def handleProgress(progress): progressList.append(progress) logger.debug("Progress update: {}".format(progress)) totalVolume = numpy.prod(inputData.shape) batch = RoiRequestBatch(op.Output, roiList.__iter__(), totalVolume, batchSize=10, allowParallelResults=False) batch.resultSignal.subscribe(handleResult) batch.progressSignal.subscribe(handleProgress) batch.execute() logger.debug("Got {} results".format(resultsCount[0])) assert (results == inputData).all() # Progress reporting MUST start with 0 and end with 100 assert progressList[0] == 0, "Invalid progress reporting." assert progressList[-1] == 100, "Invalid progress reporting." # There should be some intermediate progress reporting, but exactly how much is unspecified. assert len(progressList) >= 10 logger.debug("FINISHED")
def testOutput(self): graph = Graph() data = numpy.random.random((1, 100, 100, 10, 1)) opProvider = OpArrayPiper(graph=graph) opProvider.Input.setValue(data) opSubRegion = OpSubRegion(graph=graph) opSubRegion.Input.connect(opProvider.Output) opSubRegion.Start.setValue((0, 20, 30, 5, 0)) opSubRegion.Stop.setValue((1, 30, 50, 8, 1)) subData = opSubRegion.Output(start=(0, 5, 10, 1, 0), stop=(1, 10, 20, 3, 1)).wait() assert (subData == data[0:1, 25:30, 40:50, 6:8, 0:1]).all()
def testCleanup(self): sampleData = numpy.indices((100, 200, 150), dtype=numpy.float32).sum(0) sampleData = vigra.taggedView(sampleData, axistags='xyz') graph = Graph() opData = OpArrayPiper(graph=graph) opData.Input.setValue(sampleData) op = OpCompressedCache(graph=graph) #logger.debug("Setting block shape...") op.BlockShape.setValue([100, 75, 50]) op.Input.connect(opData.Output) x = op.Output[...].wait() op.Input.disconnect() r = weakref.ref(op) del op gc.collect() assert r() is None, "OpBlockedArrayCache was not cleaned up correctly"
def testBasic2(self): a = numpy.arange(256) a = expand_view(a, 256) a = a[..., None] a = vigra.taggedView(a, "xyc") graph = Graph() op = OpColorizeLabelImageCached(graph=graph) opPrep = OpArrayPiper(graph=graph) opPrep.Input.setValue(a) op.Input.connect(opPrep.Output) b = op.Output[...].wait() b_colors = set([tuple(_) for _ in list(array_to_matrix(b.T).T)]) assert (len(b_colors) == 256)
data_unknown_meta_tag = op.Output.meta.whatsthis assert data_shape == input_array.shape assert data_dtype == input_array.dtype assert data_unknown_meta_tag is None # we could use dict queries, but it's not that comfortable assert op.Output.meta.shape is op.Output.meta["shape"] """ Now that we know how to query a single operator, it's time for building a chain of operators. """ graph = Graph() op1 = OpArrayPiper(graph=graph) op1.name = "OpArrayPiper1" op2 = OpArrayPiper(graph=graph) op2.name = "OpArrayPiper2" op3 = OpArrayPiper(graph=graph) op3.name = "OpArrayPiper3" op3.Input.connect(op2.Output) op2.Input.connect(op1.Output) op1.Input.setValue(input_array) """ We connected three operators to a chain. Three times no operation still equals no operation, but we just wanted to see how the connections are done. Operators are always connected from end point to start point (downstream to upstream).