def test_special_z_translation(self): """ This tests the special """ tiled_volume = TiledVolume( self.data_setup.SPECIAL_Z_VOLUME_DESCRIPTION_FILE) tiled_volume.TEST_MODE = True reference_roi = numpy.array([(20, 150, 100), (40, 550, 550)]) result_out = numpy.zeros(reference_roi[1] - reference_roi[0], dtype=tiled_volume.description.dtype) roi_translated = reference_roi - [11, 0, 0] tiled_volume.read(roi_translated, result_out) ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH) with h5py.File(ref_path_comp.externalPath, 'r') as f: ref_data = f[ref_path_comp.internalPath][:] expected = ref_data[roiToSlice(*reference_roi)] #numpy.save('/tmp/expected.npy', expected) #numpy.save('/tmp/result_out.npy', result_out) assert (expected == result_out).all()
def setupOutputs(self): if self.tiled_volume: self.tiled_volume.close() # Create a TiledVolume object to read the description file and do the downloads. self.tiled_volume = TiledVolume(self.DescriptionFilePath.value) self.Output.meta.shape = tuple(self.tiled_volume.output_shape) self.Output.meta.dtype = self.tiled_volume.description.dtype self.Output.meta.axistags = vigra.defaultAxistags( self.tiled_volume.description.output_axes) self.Output.meta.prefer_2d = True self.Output.meta.nickname = self.tiled_volume.description.name
def testBasic(self): tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE ) tiled_volume.TEST_MODE = True roi = numpy.array( [(10, 150, 100), (30, 550, 550)] ) result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype ) tiled_volume.read( roi, result_out ) ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH) with h5py.File(ref_path_comp.externalPath, 'r') as f: ref_data = f[ref_path_comp.internalPath][:] expected = ref_data[roiToSlice(*roi)] #numpy.save('/tmp/expected.npy', expected) #numpy.save('/tmp/result_out.npy', result_out) assert (expected == result_out).all()
def testBasic(self): tiled_volume = TiledVolume( self.data_setup.LOCAL_VOLUME_DESCRIPTION_FILE ) tiled_volume.TEST_MODE = True roi = numpy.array( [(10, 150, 100), (30, 550, 550)] ) result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype ) tiled_volume.read( roi, result_out ) ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH) with h5py.File(ref_path_comp.externalPath, 'r') as f: ref_data = f[ref_path_comp.internalPath][:] expected = ref_data[roiToSlice(*roi)] #numpy.save('/tmp/expected.npy', expected) #numpy.save('/tmp/result_out.npy', result_out) assert (expected == result_out).all()
def testMissingTiles(self): tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE ) tiled_volume.TEST_MODE = True # The test data should be missing slice 2, and the config doesn't remap the data. roi = numpy.array( [(0, 150, 100), (10, 550, 550)] ) result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype ) tiled_volume.read( roi, result_out ) ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH) with h5py.File(ref_path_comp.externalPath, 'r') as f: ref_data = f[ref_path_comp.internalPath][:] # Slice 2 is missing from the server data expected = ref_data[roiToSlice(*roi)] expected[2] = 0 #numpy.save('/tmp/expected.npy', expected) #numpy.save('/tmp/result_out.npy', result_out) assert (expected == result_out).all()
def setupOutputs(self): if self.tiled_volume: self.tiled_volume.close() # Create a TiledVolume object to read the description file and do the downloads. self.tiled_volume = TiledVolume(self.DescriptionFilePath.value) self.Output.meta.shape = tuple(self.tiled_volume.output_shape) self.Output.meta.dtype = self.tiled_volume.description.dtype self.Output.meta.axistags = vigra.defaultAxistags(self.tiled_volume.description.output_axes) self.Output.meta.prefer_2d = True self.Output.meta.nickname = self.tiled_volume.description.name
class OpTiledVolumeReader(Operator): """ An operator to retrieve volumes from a remote server that provides volumes via image tiles. The operator requires a LOCAL json config file that describes the remote dataset and interface. (See tiledVolume.py) """ DescriptionFilePath = InputSlot(stype="filestring") Output = OutputSlot() def __init__(self, *args, **kwargs): super(OpTiledVolumeReader, self).__init__(*args, **kwargs) self.tiled_volume = None def setupOutputs(self): if self.tiled_volume: self.tiled_volume.close() # Create a TiledVolume object to read the description file and do the downloads. self.tiled_volume = TiledVolume(self.DescriptionFilePath.value) self.Output.meta.shape = tuple(self.tiled_volume.output_shape) self.Output.meta.dtype = self.tiled_volume.description.dtype self.Output.meta.axistags = vigra.defaultAxistags(self.tiled_volume.description.output_axes) self.Output.meta.prefer_2d = True self.Output.meta.nickname = self.tiled_volume.description.name def execute(self, slot, subindex, roi, result): self.tiled_volume.read((roi.start, roi.stop), result) return result def propagateDirty(self, slot, subindex, roi): assert slot == self.DescriptionFilePath, "Unknown input slot." self.Output.setDirty(slice(None)) def cleanUp(self): if self.tiled_volume: self.tiled_volume.close() super(OpTiledVolumeReader, self).cleanUp()
class OpTiledVolumeReader(Operator): """ An operator to retrieve volumes from a remote server that provides volumes via image tiles. The operator requires a LOCAL json config file that describes the remote dataset and interface. (See tiledVolume.py) """ DescriptionFilePath = InputSlot(stype='filestring') Output = OutputSlot() def __init__(self, *args, **kwargs): super(OpTiledVolumeReader, self).__init__(*args, **kwargs) self.tiled_volume = None def setupOutputs(self): if self.tiled_volume: self.tiled_volume.close() # Create a TiledVolume object to read the description file and do the downloads. self.tiled_volume = TiledVolume(self.DescriptionFilePath.value) self.Output.meta.shape = tuple(self.tiled_volume.output_shape) self.Output.meta.dtype = self.tiled_volume.description.dtype self.Output.meta.axistags = vigra.defaultAxistags( self.tiled_volume.description.output_axes) self.Output.meta.prefer_2d = True self.Output.meta.nickname = self.tiled_volume.description.name def execute(self, slot, subindex, roi, result): self.tiled_volume.read((roi.start, roi.stop), result) return result def propagateDirty(self, slot, subindex, roi): assert slot == self.DescriptionFilePath, "Unknown input slot." self.Output.setDirty(slice(None)) def cleanUp(self): if self.tiled_volume: self.tiled_volume.close() super(OpTiledVolumeReader, self).cleanUp()
def testRemappedTiles(self): # The config above specifies that slices 45:47 get their data from slice 44, # and slice 41 is the same as 40 tiled_volume = TiledVolume( self.data_setup.VOLUME_DESCRIPTION_FILE ) tiled_volume.TEST_MODE = True roi = numpy.array( [(40, 150, 100), (50, 550, 550)] ) result_out = numpy.zeros( roi[1] - roi[0], dtype=tiled_volume.description.dtype ) tiled_volume.read( roi, result_out ) ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH) with h5py.File(ref_path_comp.externalPath, 'r') as f: ref_data = f[ref_path_comp.internalPath][:] # Slices 5,6,7 are missing from the server data, and 'filled in' with slice 4 # Similarly, slice 1 is missing and filled in with slice 0. expected = ref_data[roiToSlice(*roi)] expected[5:8] = expected[4] expected[1] = expected[0] #numpy.save('/tmp/expected.npy', expected) #numpy.save('/tmp/result_out.npy', result_out) assert (expected == result_out).all()
def test_special_z_translation(self): """ This tests the special """ tiled_volume = TiledVolume( self.data_setup.SPECIAL_Z_VOLUME_DESCRIPTION_FILE ) tiled_volume.TEST_MODE = True reference_roi = numpy.array( [(20, 150, 100), (40, 550, 550)] ) result_out = numpy.zeros( reference_roi[1] - reference_roi[0], dtype=tiled_volume.description.dtype ) roi_translated = reference_roi - [11,0,0] tiled_volume.read( roi_translated, result_out ) ref_path_comp = PathComponents(self.data_setup.REFERENCE_VOL_PATH) with h5py.File(ref_path_comp.externalPath, 'r') as f: ref_data = f[ref_path_comp.internalPath][:] expected = ref_data[roiToSlice(*reference_roi)] #numpy.save('/tmp/expected.npy', expected) #numpy.save('/tmp/result_out.npy', result_out) assert (expected == result_out).all()
def setup(self): """ Generate a directory with all the files needed for this test. We use the same temporary directory every time, so we don't waste time regenerating the data if the test has already been run recently. The directory consists of the following files: - reference_volume.h5 - volume_description.json - transposed_volume_description.json - [lots of png tiles..] """ tmp = tempfile.gettempdir() self.TILE_DIRECTORY = os.path.join( tmp, 'testTiledVolume_data' ) logger.debug("Using test directory: {}".format( self.TILE_DIRECTORY )) self.REFERENCE_VOL_PATH = os.path.join( self.TILE_DIRECTORY, 'reference_volume.h5/data' ) ref_vol_path_comp = PathComponents(self.REFERENCE_VOL_PATH) self.REFERENCE_VOL_FILE = ref_vol_path_comp.externalPath self.VOLUME_DESCRIPTION_FILE = os.path.join( self.TILE_DIRECTORY, 'volume_description.json' ) self.TRANSPOSED_VOLUME_DESCRIPTION_FILE = os.path.join( self.TILE_DIRECTORY, 'transposed_volume_description.json' ) if not os.path.exists(self.TILE_DIRECTORY): print "Creating new tile directory: {}".format( self.TILE_DIRECTORY ) os.mkdir(self.TILE_DIRECTORY) if not os.path.exists(self.REFERENCE_VOL_FILE): ref_vol = numpy.random.randint(0,255, (100,600,600) ).astype(numpy.uint8) with h5py.File(self.REFERENCE_VOL_FILE, 'w') as ref_file: ref_file[ref_vol_path_comp.internalPath] = ref_vol else: with h5py.File(self.REFERENCE_VOL_FILE, 'r') as ref_file: ref_vol = ref_file[ref_vol_path_comp.internalPath][:] need_rewrite = False if not os.path.exists( self.VOLUME_DESCRIPTION_FILE ): need_rewrite = True else: with open(self.VOLUME_DESCRIPTION_FILE, 'r') as f: if f.read() != volume_description_text: need_rewrite = True if need_rewrite: with open(self.VOLUME_DESCRIPTION_FILE, 'w') as f: f.write(volume_description_text) # Read the volume description as a JsonConfig Namespace volume_description = TiledVolume.readDescription(self.VOLUME_DESCRIPTION_FILE) # Write out a copy of the description, but with custom output axes config_helper = JsonConfigParser( TiledVolume.DescriptionFields ) transposed_description = copy.copy(volume_description) transposed_description.output_axes = "xyz" config_helper.writeConfigFile(self.TRANSPOSED_VOLUME_DESCRIPTION_FILE, transposed_description) # Remove all old image tiles in the tile directory files = os.listdir(self.TILE_DIRECTORY) for name in files: if os.path.splitext(name)[1] == '.' + volume_description.format: os.remove( os.path.join(self.TILE_DIRECTORY, name) ) # Write the new tiles export_to_tiles( ref_vol, volume_description.tile_shape_2d_yx[0], self.TILE_DIRECTORY, print_progress=False ) # To support testMissingTiles (below), remove slice 2 files = os.listdir(self.TILE_DIRECTORY) for name in files: if name.startswith("tile_z00002"): p = os.path.join(self.TILE_DIRECTORY, name) print "removing:", p os.remove( p ) # lastly, start the server self._start_server()
def setup(self): """ Generate a directory with all the files needed for this test. We use the same temporary directory every time, so we don't waste time regenerating the data if the test has already been run recently. The directory consists of the following files: - reference_volume.h5 - volume_description.json - transposed_volume_description.json - [lots of png tiles..] """ tmp = tempfile.gettempdir() self.TILE_DIRECTORY = os.path.join(tmp, 'testTiledVolume_data') logger.debug("Using test directory: {}".format(self.TILE_DIRECTORY)) self.REFERENCE_VOL_PATH = os.path.join(self.TILE_DIRECTORY, 'reference_volume.h5/data') ref_vol_path_comp = PathComponents(self.REFERENCE_VOL_PATH) self.REFERENCE_VOL_FILE = ref_vol_path_comp.externalPath self.VOLUME_DESCRIPTION_FILE = os.path.join(self.TILE_DIRECTORY, 'volume_description.json') self.TRANSPOSED_VOLUME_DESCRIPTION_FILE = os.path.join( self.TILE_DIRECTORY, 'transposed_volume_description.json') if not os.path.exists(self.TILE_DIRECTORY): print "Creating new tile directory: {}".format(self.TILE_DIRECTORY) os.mkdir(self.TILE_DIRECTORY) if not os.path.exists(self.REFERENCE_VOL_FILE): ref_vol = numpy.random.randint(0, 255, (100, 600, 600)).astype(numpy.uint8) with h5py.File(self.REFERENCE_VOL_FILE, 'w') as ref_file: ref_file[ref_vol_path_comp.internalPath] = ref_vol else: with h5py.File(self.REFERENCE_VOL_FILE, 'r') as ref_file: ref_vol = ref_file[ref_vol_path_comp.internalPath][:] need_rewrite = False if not os.path.exists(self.VOLUME_DESCRIPTION_FILE): need_rewrite = True else: with open(self.VOLUME_DESCRIPTION_FILE, 'r') as f: if f.read() != volume_description_text: need_rewrite = True if need_rewrite: with open(self.VOLUME_DESCRIPTION_FILE, 'w') as f: f.write(volume_description_text) # Read the volume description as a JsonConfig Namespace volume_description = TiledVolume.readDescription( self.VOLUME_DESCRIPTION_FILE) # Write out a copy of the description, but with custom output axes config_helper = JsonConfigParser(TiledVolume.DescriptionFields) transposed_description = copy.copy(volume_description) transposed_description.output_axes = "xyz" config_helper.writeConfigFile( self.TRANSPOSED_VOLUME_DESCRIPTION_FILE, transposed_description) # Remove all old image tiles in the tile directory files = os.listdir(self.TILE_DIRECTORY) for name in files: if os.path.splitext( name)[1] == '.' + volume_description.format: os.remove(os.path.join(self.TILE_DIRECTORY, name)) # Write the new tiles export_to_tiles(ref_vol, volume_description.tile_shape_2d_yx[0], self.TILE_DIRECTORY, print_progress=False) # To support testMissingTiles (below), remove slice 2 files = os.listdir(self.TILE_DIRECTORY) for name in files: if name.startswith("tile_z00002"): p = os.path.join(self.TILE_DIRECTORY, name) print "removing:", p os.remove(p) # lastly, start the server self._start_server()