def testBasic(self): """ Requires access to the Internet... """ testConfig0 = """ { "_schema_name" : "RESTful-volume-description", "_schema_version" : 1.0, "name" : "Bock11-level0", "format" : "hdf5", "axes" : "zyx", "##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917", "origin_offset" : [2917, 50000, 50000], "bounds" : [4156, 135424, 119808], "dtype" : "numpy.uint8", "url_format" : "http://openconnecto.me/emca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/", "hdf5_dataset" : "cube" } """ testConfig4 = """ { "_schema_name" : "RESTful-volume-description", "_schema_version" : 1.0, "name" : "Bock11-level4", "format" : "hdf5", "axes" : "zyx", "##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917", "origin_offset" : [2917, 50000, 50000], "bounds" : [4156, 8704, 7680], "dtype" : "numpy.uint8", "url_format" : "http://openconnecto.me/emca/bock11/hdf5/4/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/", "hdf5_dataset" : "cube" } """ # Create the description file. tempDir = tempfile.mkdtemp() descriptionFilePath = os.path.join(tempDir, 'desc.json') with open(descriptionFilePath, 'w') as descFile: descFile.write( testConfig0 ) # Create the volume object volume = RESTfulVolume( descriptionFilePath ) #slicing = numpy.s_[0:100, 4000:4200, 4000:4200] slicing = numpy.s_[0:25, 50000:50050, 50000:50075] roi = sliceToRoi( slicing, volume.description.shape ) outputFile = os.path.join(tempDir, 'volume.h5') datasetPath = outputFile + '/cube' logger.debug("Downloading subvolume to: {}".format( datasetPath )) volume.downloadSubVolume(roi, datasetPath) with h5py.File(outputFile, 'r') as hdf5File: data = hdf5File['cube'] assert data.shape == ( 25, 50, 75 ) shutil.rmtree(tempDir)
class OpRESTfulVolumeReader(Operator): """ An operator to retrieve hdf5 volumes from a remote server that provides a RESTful interface. The operator requires a LOCAL json config file that describes the remote dataset and interface. """ name = "OpRESTfulVolumeReader" DescriptionFilePath = InputSlot(stype='filestring') Output = OutputSlot() def __init__(self, *args, **kwargs): super(OpRESTfulVolumeReader, self).__init__(*args, **kwargs) self._axes = None self._volumeObject = None def setupOutputs(self): # Create a RESTfulVolume object to read the description file and do the downloads. self._volumeObject = RESTfulVolume( self.DescriptionFilePath.value ) self._axes = self._volumeObject.description.axes outputShape = tuple( self._volumeObject.description.shape ) # If the dataset has no channel axis, add one. if 'c' not in self._axes: outputShape += (1,) self._axes += 'c' self.Output.meta.shape = outputShape self.Output.meta.dtype = self._volumeObject.description.dtype self.Output.meta.axistags = vigra.defaultAxistags(self._axes) def execute(self, slot, subindex, roi, result): roi = copy.copy(roi) # If we are artificially adding a channel index, remove it from the roi for the download. if len(self.Output.meta.shape) > len(self._volumeObject.description.shape): roi.start.pop( self.Output.meta.axistags.index('c') ) roi.stop.pop( self.Output.meta.axistags.index('c') ) # Write the data from the url out to disk (in a temporary file) hdf5FilePath = os.path.join(tempfile.mkdtemp(), 'cube.h5') hdf5DatasetPath = hdf5FilePath + self._volumeObject.description.hdf5_dataset self._volumeObject.downloadSubVolume( (roi.start, roi.stop), hdf5DatasetPath ) # Open the file we just created using h5py with h5py.File( hdf5FilePath, 'r' ) as hdf5File: dataset = hdf5File[self._volumeObject.description.hdf5_dataset] if len(result.shape) > len(dataset.shape): # We appended a channel axis to Output, but the dataset doesn't have that. result[...,0] = dataset[...] else: result[...] = dataset[...] return result def propagateDirty(self, slot, subindex, roi): assert slot == self.DescriptionFilePath, "Unknown input slot." self.Output.setDirty( slice(None) )
def setupOutputs(self): # Create a RESTfulVolume object to read the description file and do the downloads. self._volumeObject = RESTfulVolume( self.DescriptionFilePath.value ) self._axes = self._volumeObject.description.axes outputShape = tuple( self._volumeObject.description.shape ) # If the dataset has no channel axis, add one. if 'c' not in self._axes: outputShape += (1,) self._axes += 'c' self.Output.meta.shape = outputShape self.Output.meta.dtype = self._volumeObject.description.dtype self.Output.meta.axistags = vigra.defaultAxistags(self._axes)
def readDescription(cls, descriptionFilePath): description = RESTfulBlockwiseFileset.DescriptionSchema.parseConfigFile( descriptionFilePath ) RESTfulVolume.updateDescription(description.remote_description) return description