def setupOutputs(self):
if not os.path.exists(self.DescriptionFilePath.value):
raise OpBlockwiseFilesetReader.MissingDatasetError(
"Dataset description not found: {}".format(
self.DescriptionFilePath.value))
# Load up the class that does the real work
self._blockwiseFileset = BlockwiseFileset(
self.DescriptionFilePath.value)
# Check for errors in the description file
descriptionFields = self._blockwiseFileset.description
axes = descriptionFields.axes
assert False not in map(
lambda a: a in 'txyzc',
axes), "Unknown axis type. Known axes: txyzc Your axes:".format(
axes)
self.Output.meta.shape = tuple(descriptionFields.view_shape)
self.Output.meta.dtype = descriptionFields.dtype
self.Output.meta.axistags = vigra.defaultAxistags(
descriptionFields.axes)
drange = descriptionFields.drange
if drange is not None:
self.Output.meta.drange = drange
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1, )
# Check for errors
primaryOutputDescription = BlockwiseFileset.readDescription(
self.OutputDatasetDescription.value)
primary_block_shape = primaryOutputDescription.block_shape
primary_sub_block_shape = primaryOutputDescription.sub_block_shape
assert primary_sub_block_shape is not None, "Primary output description file must specify a sub_block_shape"
# Ratio of blocks to sub-blocks for all secondaries must match the primary.
primary_sub_block_factor = (primary_block_shape +
primary_sub_block_shape -
1) / primary_sub_block_shape
for i, slot in enumerate(self.SecondaryOutputDescriptions):
descriptionPath = slot.value
secondaryDescription = BlockwiseFileset.readDescription(
descriptionPath)
block_shape = secondaryDescription.block_shape
sub_block_shape = secondaryDescription.sub_block_shape
assert sub_block_shape is not None, "Secondary output description #{} doesn't specify a sub_block_shape".format(
i)
sub_block_factor = (block_shape + sub_block_shape -
1) / sub_block_shape
if (tuple(primary_sub_block_factor) != tuple(sub_block_factor)):
msg = "Error: Ratio of sub_block_shape to block_shape must be the same in the primary output dataset and in all secondary datasets.\n"
msg += "Secondary dataset {} has a factor of {}, which doesn't match primary factor of {}".format(
i, sub_block_factor, primary_sub_block_factor)
raise RuntimeError(msg)
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1,)
# Check for errors
primaryOutputDescription = BlockwiseFileset.readDescription(self.OutputDatasetDescription.value)
primary_block_shape = primaryOutputDescription.block_shape
primary_sub_block_shape = primaryOutputDescription.sub_block_shape
assert primary_sub_block_shape is not None, "Primary output description file must specify a sub_block_shape"
# Ratio of blocks to sub-blocks for all secondaries must match the primary.
primary_sub_block_factor = (primary_block_shape + primary_sub_block_shape - 1) / primary_sub_block_shape
for i, slot in enumerate( self.SecondaryOutputDescriptions ):
descriptionPath = slot.value
secondaryDescription = BlockwiseFileset.readDescription(descriptionPath)
block_shape = secondaryDescription.block_shape
sub_block_shape = secondaryDescription.sub_block_shape
assert sub_block_shape is not None, "Secondary output description #{} doesn't specify a sub_block_shape".format( i )
sub_block_factor = (block_shape + sub_block_shape - 1) / sub_block_shape
if (tuple(primary_sub_block_factor) != tuple(sub_block_factor)):
msg = "Error: Ratio of sub_block_shape to block_shape must be the same in the primary output dataset and in all secondary datasets.\n"
msg += "Secondary dataset {} has a factor of {}, which doesn't match primary factor of {}".format( i, sub_block_factor, primary_sub_block_factor )
raise RuntimeError(msg)
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1, )
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset(
self.OutputFilesetDescription.value, 'a')
class OpBlockwiseFilesetReader(Operator):
"""
Adapter that provides an operator interface to the BlockwiseFileset class for reading ONLY.
"""
name = "OpBlockwiseFilesetReader"
DescriptionFilePath = InputSlot(stype='filestring')
Output = OutputSlot()
class MissingDatasetError(Exception):
pass
def __init__(self, *args, **kwargs):
super(OpBlockwiseFilesetReader, self).__init__(*args, **kwargs)
self._blockwiseFileset = None
self._opDummyData = OpDummyData(parent=self)
def setupOutputs(self):
if not os.path.exists(self.DescriptionFilePath.value):
raise OpBlockwiseFilesetReader.MissingDatasetError(
"Dataset description not found: {}".format(
self.DescriptionFilePath.value))
# Load up the class that does the real work
self._blockwiseFileset = BlockwiseFileset(
self.DescriptionFilePath.value)
# Check for errors in the description file
descriptionFields = self._blockwiseFileset.description
axes = descriptionFields.axes
assert False not in map(
lambda a: a in 'txyzc',
axes), "Unknown axis type. Known axes: txyzc Your axes:".format(
axes)
self.Output.meta.shape = tuple(descriptionFields.view_shape)
self.Output.meta.dtype = descriptionFields.dtype
self.Output.meta.axistags = vigra.defaultAxistags(
descriptionFields.axes)
drange = descriptionFields.drange
if drange is not None:
self.Output.meta.drange = drange
def execute(self, slot, subindex, roi, result):
assert slot == self.Output, "Unknown output slot"
try:
self._blockwiseFileset.readData((roi.start, roi.stop), result)
except BlockwiseFileset.BlockNotReadyError:
result[:] = self._opDummyData.execute(slot, subindex, roi, result)
return result
def propagateDirty(self, slot, subindex, roi):
assert slot == self.DescriptionFilePath, "Unknown input slot."
self.Output.setDirty(slice(None))
def cleanUp(self):
if self._blockwiseFileset is not None:
self._blockwiseFileset.close()
super(OpBlockwiseFilesetReader, self).cleanUp()
class OpBlockwiseFilesetReader(Operator):
"""
Adapter that provides an operator interface to the BlockwiseFileset class for reading ONLY.
"""
name = "OpBlockwiseFilesetReader"
DescriptionFilePath = InputSlot(stype="filestring")
Output = OutputSlot()
class MissingDatasetError(Exception):
pass
def __init__(self, *args, **kwargs):
super(OpBlockwiseFilesetReader, self).__init__(*args, **kwargs)
self._blockwiseFileset = None
self._opDummyData = OpDummyData(parent=self)
def setupOutputs(self):
if not os.path.exists(self.DescriptionFilePath.value):
raise OpBlockwiseFilesetReader.MissingDatasetError(
"Dataset description not found: {}".format(self.DescriptionFilePath.value)
)
# Load up the class that does the real work
self._blockwiseFileset = BlockwiseFileset(self.DescriptionFilePath.value)
# Check for errors in the description file
descriptionFields = self._blockwiseFileset.description
axes = descriptionFields.axes
assert False not in map(
lambda a: a in "txyzc", axes
), "Unknown axis type. Known axes: txyzc Your axes:".format(axes)
self.Output.meta.shape = tuple(descriptionFields.view_shape)
self.Output.meta.dtype = descriptionFields.dtype
self.Output.meta.axistags = vigra.defaultAxistags(descriptionFields.axes)
drange = descriptionFields.drange
if drange is not None:
self.Output.meta.drange = drange
def execute(self, slot, subindex, roi, result):
assert slot == self.Output, "Unknown output slot"
try:
self._blockwiseFileset.readData((roi.start, roi.stop), result)
except BlockwiseFileset.BlockNotReadyError:
result[:] = self._opDummyData.execute(slot, subindex, roi, result)
return result
def propagateDirty(self, slot, subindex, roi):
assert slot == self.DescriptionFilePath, "Unknown input slot."
self.Output.setDirty(slice(None))
def cleanUp(self):
if self._blockwiseFileset is not None:
self._blockwiseFileset.close()
super(OpBlockwiseFilesetReader, self).cleanUp()
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1,)
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset( self.OutputFilesetDescription.value, 'a' )
self._secondaryBlockwiseFilesets = []
for slot in self.SecondaryOutputDescriptions:
descriptionPath = slot.value
self._secondaryBlockwiseFilesets.append( BlockwiseFileset( descriptionPath, 'a' ) )
def test_9_TestView(self):
"""
Load some of the dataset again; this time with an offset view.
Note: The original blockwise fileset must be closed before this test starts.
"""
# Create a copy of the original description, but specify a translated (and smaller) view
desc = BlockwiseFileset.readDescription(self.description_path)
desc.view_origin = [0, 300, 200, 100, 0]
desc.view_shape = [1, 50, 50, 50, 1]
offsetConfigPath = self.description_path + '_offset'
BlockwiseFileset.writeDescription(offsetConfigPath, desc)
# Open the fileset using the special description file
bfs = BlockwiseFileset( offsetConfigPath, 'r' )
try:
assert (bfs.description.view_origin == desc.view_origin).all()
assert (bfs.description.view_shape == desc.view_shape).all()
# Read some data
logger.debug( "Reading data..." )
disk_slicing = numpy.s_[:, 300:350, 200:250, 100:150, :]
view_slicing = numpy.s_[:, 0:50, 0:50, 0:50, :]
roi = sliceToRoi( view_slicing, self.dataShape )
roiShape = roi[1] - roi[0]
read_data = numpy.zeros( tuple(roiShape), dtype=numpy.uint8 )
bfs.readData( roi, read_data )
# The data we read should match the correct part of the original dataset.
logger.debug( "Checking data..." )
assert self.data[disk_slicing].shape == read_data.shape
assert (self.data[disk_slicing] == read_data).all(), "Data didn't match."
finally:
bfs.close()
def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug("Loading config file...")
with open(self.configpath, 'w') as f:
f.write(testConfig)
logger.debug("Creating random test data...")
bfs = BlockwiseFileset(self.configpath, 'a')
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint(255,
size=dataShape).astype(numpy.uint8)
logger.debug("Writing test data...")
datasetRoi = ([0, 0, 0, 0, 0], dataShape)
bfs.writeData(datasetRoi, self.data)
block_starts = getIntersectingBlocks(bfs.description.block_shape,
datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def setupClass(cls):
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,200,1],
"dtype" : "numpy.uint8",
"compression" : "lzf",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "description.json")
with open(cls.description_path, 'w') as f:
f.write(testConfig)
logger.debug("Loading config file...")
cls.bfs = BlockwiseFileset(cls.description_path, 'a')
cls.dataShape = tuple(cls.bfs.description.shape)
logger.debug("Creating random test data...")
cls.data = numpy.random.randint(255,
size=cls.dataShape).astype(numpy.uint8)
def setupClass(cls):
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,10,20,5,1],
"dtype" : "object",
"block_shape" : [1, 5, 4, 1, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "config.json")
with open(cls.description_path, 'w') as f:
f.write(testConfig)
logger.debug("Loading config file...")
cls.bfs = BlockwiseFileset(cls.description_path, 'a')
cls.dataShape = tuple(cls.bfs.description.shape)
def make_dummy_dict(x):
return {str(x): numpy.array([x, x])}
vec_make_dummy_dict = numpy.vectorize(make_dummy_dict)
int_data = numpy.random.randint(255,
size=cls.dataShape).astype(numpy.uint8)
dict_data = vec_make_dummy_dict(int_data)
cls.data = dict_data
def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug( "Loading config file..." )
with open(self.configpath, 'w') as f:
f.write(testConfig)
logger.debug( "Creating random test data..." )
bfs = BlockwiseFileset( self.configpath, 'a' )
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint( 255, size=dataShape ).astype(numpy.uint8)
logger.debug( "Writing test data..." )
datasetRoi = ([0,0,0,0,0], dataShape)
bfs.writeData( datasetRoi, self.data )
block_starts = getIntersectingBlocks(bfs.description.block_shape, datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1,)
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset( self.OutputFilesetDescription.value, 'a' )
self._secondaryBlockwiseFilesets = []
for slot in self.SecondaryOutputDescriptions:
descriptionPath = slot.value
self._secondaryBlockwiseFilesets.append( BlockwiseFileset( descriptionPath, 'a' ) )
def test_6_TestExportSubset(self):
roi = ( (0, 0, 50, 100, 0), (1, 100, 200, 200, 1) )
exportDir = tempfile.mkdtemp()
self.bfs.close()
self.bfs.reopen( 'r' )
exported_description_path = self.bfs.exportSubset( roi, exportDir )
try:
exported_bfs = BlockwiseFileset( exported_description_path, 'r' )
assert os.path.exists( exported_description_path ), "Couldn't even find the exported description file."
read_data = exported_bfs.readData( roi )
expected_data = self.data[ roiToSlice(*roi) ]
assert read_data.shape == expected_data.shape, "Exported data had wrong shape"
assert read_data.dtype == expected_data.dtype, "Exported data had wrong dtype"
assert (read_data == expected_data).all(), "Exported data did not match expected data"
finally:
shutil.rmtree(exportDir)
def setupClass(cls):
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,10,20,5,1],
"dtype" : "object",
"block_shape" : [1, 5, 4, 1, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "config.json")
with open(cls.description_path, 'w') as f:
f.write(testConfig)
logger.debug( "Loading config file..." )
cls.bfs = BlockwiseFileset( cls.description_path, 'a' )
cls.dataShape = tuple(cls.bfs.description.shape)
def make_dummy_dict(x):
return {str(x) : numpy.array([x,x])}
vec_make_dummy_dict = numpy.vectorize(make_dummy_dict)
int_data = numpy.random.randint(255, size=cls.dataShape ).astype(numpy.uint8)
dict_data = vec_make_dummy_dict(int_data)
cls.data = dict_data
def test_6_TestExportSubset(self):
roi = ((0, 0, 50, 100, 0), (1, 100, 200, 200, 1))
exportDir = tempfile.mkdtemp()
self.bfs.close()
self.bfs.reopen('r')
exported_description_path = self.bfs.exportSubset(roi, exportDir)
try:
exported_bfs = BlockwiseFileset(exported_description_path, 'r')
assert os.path.exists(
exported_description_path
), "Couldn't even find the exported description file."
read_data = exported_bfs.readData(roi)
expected_data = self.data[roiToSlice(*roi)]
assert read_data.shape == expected_data.shape, "Exported data had wrong shape"
assert read_data.dtype == expected_data.dtype, "Exported data had wrong dtype"
assert (read_data == expected_data
).all(), "Exported data did not match expected data"
finally:
shutil.rmtree(exportDir)
def setupOutputs(self):
if not os.path.exists(self.DescriptionFilePath.value):
raise OpBlockwiseFilesetReader.MissingDatasetError("Dataset description not found: {}".format( self.DescriptionFilePath.value ) )
# Load up the class that does the real work
self._blockwiseFileset = BlockwiseFileset( self.DescriptionFilePath.value )
# Check for errors in the description file
descriptionFields = self._blockwiseFileset.description
axes = descriptionFields.axes
assert False not in map(lambda a: a in 'txyzc', axes), "Unknown axis type. Known axes: txyzc Your axes:".format(axes)
self.Output.meta.shape = tuple(descriptionFields.view_shape)
self.Output.meta.dtype = descriptionFields.dtype
self.Output.meta.axistags = vigra.defaultAxistags(descriptionFields.axes)
drange = descriptionFields.drange
if drange is not None:
self.Output.meta.drange = drange
def test_9_TestView(self):
"""
Load some of the dataset again; this time with an offset view.
Note: The original blockwise fileset must be closed before this test starts.
"""
# Create a copy of the original description, but specify a translated (and smaller) view
desc = BlockwiseFileset.readDescription(self.description_path)
desc.view_origin = [0, 300, 200, 100, 0]
desc.view_shape = [1, 50, 50, 50, 1]
offsetConfigPath = self.description_path + '_offset'
BlockwiseFileset.writeDescription(offsetConfigPath, desc)
# Open the fileset using the special description file
bfs = BlockwiseFileset(offsetConfigPath, 'r')
try:
assert (bfs.description.view_origin == desc.view_origin).all()
assert (bfs.description.view_shape == desc.view_shape).all()
# Read some data
logger.debug("Reading data...")
disk_slicing = numpy.s_[:, 300:350, 200:250, 100:150, :]
view_slicing = numpy.s_[:, 0:50, 0:50, 0:50, :]
roi = sliceToRoi(view_slicing, self.dataShape)
roiShape = roi[1] - roi[0]
read_data = numpy.zeros(tuple(roiShape), dtype=numpy.uint8)
bfs.readData(roi, read_data)
# The data we read should match the correct part of the original dataset.
logger.debug("Checking data...")
assert self.data[disk_slicing].shape == read_data.shape
assert (self.data[disk_slicing] == read_data
).all(), "Data didn't match."
finally:
bfs.close()
def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if "Darwin" in platform.platform():
# For unknown reasons, blockwise fileset tests fail due to strange "too many files" errors on mac
raise nose.SkipTest
testConfig = """
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug("Loading config file...")
with open(self.configpath, "w") as f:
f.write(testConfig)
logger.debug("Creating random test data...")
bfs = BlockwiseFileset(self.configpath, "a")
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint(255, size=dataShape).astype(numpy.uint8)
logger.debug("Writing test data...")
datasetRoi = ([0, 0, 0, 0, 0], dataShape)
bfs.writeData(datasetRoi, self.data)
block_starts = getIntersectingBlocks(bfs.description.block_shape, datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def setupClass(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
from lazyflow.utility.io.blockwiseFileset import BlockwiseFileset
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format( cls.tempDir ))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"## NOTE 1": "The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"## NOTE 2": "The website says that the data goes up to plane 4156, but it actually errors out past 4150",
"origin_offset" : [2917, 0, 0],
"bounds" : [4150, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = \
"""
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format( remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription )
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, 'w') as f:
f.write(compositeDescription)
def setupClass(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
from lazyflow.utility.io.blockwiseFileset import BlockwiseFileset
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format(cls.tempDir))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"## NOTE 1": "The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"## NOTE 2": "The website says that the data goes up to plane 4156, but it actually errors out past 4150",
"origin_offset" : [2917, 0, 0],
"bounds" : [4150, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = \
"""
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format( remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription )
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, 'w') as f:
f.write(compositeDescription)
class OpTaskWorker(Operator):
Input = InputSlot()
RoiString = InputSlot(stype='string')
TaskName = InputSlot(stype='string')
ConfigFilePath = InputSlot(stype='filestring')
OutputFilesetDescription = InputSlot(stype='filestring')
SecondaryInputs = InputSlot(level=1, optional=True)
SecondaryOutputDescriptions = InputSlot(level=1, optional=True)
ReturnCode = OutputSlot()
def __init__(self, *args, **kwargs):
super( OpTaskWorker, self ).__init__( *args, **kwargs )
self.progressSignal = OrderedSignal()
self._primaryBlockwiseFileset = None
self._secondaryBlockwiseFilesets = []
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1,)
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset( self.OutputFilesetDescription.value, 'a' )
self._secondaryBlockwiseFilesets = []
for slot in self.SecondaryOutputDescriptions:
descriptionPath = slot.value
self._secondaryBlockwiseFilesets.append( BlockwiseFileset( descriptionPath, 'a' ) )
def cleanUp(self):
self._closeFiles()
super( OpTaskWorker, self ).cleanUp()
def _closeFiles(self):
if self._primaryBlockwiseFileset is not None:
self._primaryBlockwiseFileset.close()
for fileset in self._secondaryBlockwiseFilesets:
fileset.close()
self._primaryBlockwiseFileset = None
self._secondaryBlockwiseFilesets = []
def execute(self, slot, subindex, ignored_roi, result):
configFilePath = self.ConfigFilePath.value
config = parseClusterConfigFile( configFilePath )
blockwiseFileset = self._primaryBlockwiseFileset
# Check axis compatibility
inputAxes = self.Input.meta.getTaggedShape().keys()
outputAxes = list(blockwiseFileset.description.axes)
assert set(inputAxes) == set(outputAxes), \
"Output dataset has the wrong set of axes. Input axes: {}, Output axes: {}".format( "".join(inputAxes), "".join(outputAxes) )
roiString = self.RoiString.value
roi = Roi.loads(roiString)
if len( roi.start ) != len( self.Input.meta.shape ):
assert False, "Task roi: {} is not valid for this input. Did the master launch this task correctly?".format( roiString )
logger.info( "Executing for roi: {}".format(roi) )
if config.use_node_local_scratch:
assert False, "FIXME."
assert (blockwiseFileset.getEntireBlockRoi( roi.start )[1] == roi.stop).all(), "Each task must execute exactly one full block. ({},{}) is not a valid block roi.".format( roi.start, roi.stop )
assert self.Input.ready()
# Convert the task subrequest shape dict into a shape for this dataset (and axisordering)
subrequest_shape = map( lambda tag: config.task_subrequest_shape[tag.key], self.Input.meta.axistags )
primary_subrequest_shape = self._primaryBlockwiseFileset.description.sub_block_shape
if primary_subrequest_shape is not None:
# If the output dataset specified a sub_block_shape, override the cluster config
subrequest_shape = primary_subrequest_shape
with Timer() as computeTimer:
# Stream the data out to disk.
streamer = BigRequestStreamer(self.Input, (roi.start, roi.stop), subrequest_shape, config.task_parallel_subrequests )
streamer.progressSignal.subscribe( self.progressSignal )
streamer.resultSignal.subscribe( self._handlePrimaryResultBlock )
streamer.execute()
# Now the block is ready. Update the status.
blockwiseFileset.setBlockStatus( roi.start, BlockwiseFileset.BLOCK_AVAILABLE )
logger.info( "Finished task in {} seconds".format( computeTimer.seconds() ) )
result[0] = True
return result
def propagateDirty(self, slot, subindex, roi):
self.ReturnCode.setDirty( slice(None) )
def _handlePrimaryResultBlock(self, roi, result):
# First write the primary
self._primaryBlockwiseFileset.writeData(roi, result)
# Get this block's index with respect to the primary dataset
sub_block_index = roi[0] / self._primaryBlockwiseFileset.description.sub_block_shape
# Now request the secondaries
for slot, fileset in zip(self.SecondaryInputs, self._secondaryBlockwiseFilesets):
# Compute the corresponding sub_block in this output dataset
sub_block_shape = fileset.description.sub_block_shape
sub_block_start = sub_block_index * sub_block_shape
sub_block_stop = sub_block_start + sub_block_shape
sub_block_stop = numpy.minimum( sub_block_stop, fileset.description.shape )
sub_block_roi = (sub_block_start, sub_block_stop)
secondary_result = slot( *sub_block_roi ).wait()
fileset.writeData( sub_block_roi, secondary_result )
def _prepareDestination(self):
"""
- If the result file doesn't exist yet, create it (and the dataset)
- If the result file already exists, return a list of the rois that
are NOT needed (their data already exists in the final output)
"""
originalDescription = BlockwiseFileset.readDescription(self.OutputDatasetDescription.value)
datasetDescription = copy.deepcopy(originalDescription)
# Modify description fields as needed
# -- axes
datasetDescription.axes = "".join( self.Input.meta.getTaggedShape().keys() )
assert set(originalDescription.axes) == set( datasetDescription.axes ), \
"Can't prepare destination dataset: original dataset description listed " \
"axes as {}, but actual output axes are {}".format( originalDescription.axes, datasetDescription.axes )
# -- shape
datasetDescription.view_shape = list(self.Input.meta.shape)
# -- block_shape
assert originalDescription.block_shape is not None
originalBlockDims = collections.OrderedDict( zip( originalDescription.axes, originalDescription.block_shape ) )
datasetDescription.block_shape = map( lambda a: originalBlockDims[a], datasetDescription.axes )
datasetDescription.block_shape = map( min, zip( datasetDescription.block_shape, self.Input.meta.shape ) )
# -- chunks
if originalDescription.chunks is not None:
originalChunkDims = collections.OrderedDict( zip( originalDescription.axes, originalDescription.chunks ) )
datasetDescription.chunks = map( lambda a: originalChunkDims[a], datasetDescription.axes )
datasetDescription.chunks = map( min, zip( datasetDescription.chunks, self.Input.meta.shape ) )
# -- dtype
if datasetDescription.dtype != self.Input.meta.dtype:
dtype = self.Input.meta.dtype
if type(dtype) is numpy.dtype:
dtype = dtype.type
datasetDescription.dtype = dtype().__class__.__name__
# Create a unique hash for this blocking scheme.
# If it changes, we can't use any previous data.
sha = hashlib.sha1()
sha.update( str( tuple( datasetDescription.block_shape) ) )
sha.update( datasetDescription.axes )
sha.update( datasetDescription.block_file_name_format )
datasetDescription.hash_id = sha.hexdigest()
if datasetDescription != originalDescription:
descriptionFilePath = self.OutputDatasetDescription.value
logger.info( "Overwriting dataset description: {}".format( descriptionFilePath ) )
BlockwiseFileset.writeDescription(descriptionFilePath, datasetDescription)
with open( descriptionFilePath, 'r' ) as f:
logger.info( f.read() )
# Now open the dataset
blockwiseFileset = BlockwiseFileset( self.OutputDatasetDescription.value )
taskInfos = self._prepareTaskInfos( blockwiseFileset.getAllBlockRois() )
if blockwiseFileset.description.hash_id != originalDescription.hash_id:
# Something about our blocking scheme changed.
# Make sure all blocks are marked as NOT available.
# (Just in case some were left over from a previous run.)
for roi in taskInfos.keys():
blockwiseFileset.setBlockStatus( roi[0], BlockwiseFileset.BLOCK_NOT_AVAILABLE )
return blockwiseFileset, taskInfos
class OpTaskWorker(Operator):
Input = InputSlot()
RoiString = InputSlot(stype='string')
TaskName = InputSlot(stype='string')
ConfigFilePath = InputSlot(stype='filestring')
OutputFilesetDescription = InputSlot(stype='filestring')
ReturnCode = OutputSlot()
def __init__(self, *args, **kwargs):
super(OpTaskWorker, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._primaryBlockwiseFileset = None
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1, )
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset(
self.OutputFilesetDescription.value, 'a')
def cleanUp(self):
self._closeFiles()
super(OpTaskWorker, self).cleanUp()
def _closeFiles(self):
if self._primaryBlockwiseFileset is not None:
self._primaryBlockwiseFileset.close()
self._primaryBlockwiseFileset = None
def execute(self, slot, subindex, ignored_roi, result):
configFilePath = self.ConfigFilePath.value
config = parseClusterConfigFile(configFilePath)
blockwiseFileset = self._primaryBlockwiseFileset
# Check axis compatibility
inputAxes = self.Input.meta.getTaggedShape().keys()
outputAxes = list(blockwiseFileset.description.axes)
assert set(inputAxes) == set(outputAxes), \
"Output dataset has the wrong set of axes. Input axes: {}, Output axes: {}".format( "".join(inputAxes), "".join(outputAxes) )
roiString = self.RoiString.value
roi = Roi.loads(roiString)
if len(roi.start) != len(self.Input.meta.shape):
assert False, "Task roi: {} is not valid for this input. Did the master launch this task correctly?".format(
roiString)
logger.info("Executing for roi: {}".format(roi))
if config.use_node_local_scratch:
assert False, "FIXME."
assert (
blockwiseFileset.getEntireBlockRoi(roi.start)[1] == roi.stop
).all(
), "Each task must execute exactly one full block. ({},{}) is not a valid block roi.".format(
roi.start, roi.stop)
assert self.Input.ready()
with Timer() as computeTimer:
# Stream the data out to disk.
request_blockshape = self._primaryBlockwiseFileset.description.sub_block_shape # Could be None. That's okay.
streamer = BigRequestStreamer(self.Input, (roi.start, roi.stop),
request_blockshape)
streamer.progressSignal.subscribe(self.progressSignal)
streamer.resultSignal.subscribe(self._handlePrimaryResultBlock)
streamer.execute()
# Now the block is ready. Update the status.
blockwiseFileset.setBlockStatus(roi.start,
BlockwiseFileset.BLOCK_AVAILABLE)
logger.info("Finished task in {} seconds".format(
computeTimer.seconds()))
result[0] = True
return result
def propagateDirty(self, slot, subindex, roi):
self.ReturnCode.setDirty(slice(None))
def _handlePrimaryResultBlock(self, roi, result):
# First write the primary
self._primaryBlockwiseFileset.writeData(roi, result)
# Ask the workflow if there is any special post-processing to do...
self.get_workflow().postprocessClusterSubResult(
roi, result, self._primaryBlockwiseFileset)
def get_workflow(self):
op = self
while not isinstance(op, Workflow):
op = op.parent
return op
def _prepareDestination(self):
"""
- If the result file doesn't exist yet, create it (and the dataset)
- If the result file already exists, return a list of the rois that
are NOT needed (their data already exists in the final output)
"""
originalDescription = BlockwiseFileset.readDescription(
self.OutputDatasetDescription.value)
datasetDescription = copy.deepcopy(originalDescription)
# Modify description fields as needed
# -- axes
datasetDescription.axes = "".join(
self.Input.meta.getTaggedShape().keys())
assert set(originalDescription.axes) == set( datasetDescription.axes ), \
"Can't prepare destination dataset: original dataset description listed " \
"axes as {}, but actual output axes are {}".format( originalDescription.axes, datasetDescription.axes )
# -- shape
datasetDescription.view_shape = list(self.Input.meta.shape)
# -- block_shape
assert originalDescription.block_shape is not None
originalBlockDims = collections.OrderedDict(
zip(originalDescription.axes, originalDescription.block_shape))
datasetDescription.block_shape = map(lambda a: originalBlockDims[a],
datasetDescription.axes)
datasetDescription.block_shape = map(
min, zip(datasetDescription.block_shape, self.Input.meta.shape))
# -- chunks
if originalDescription.chunks is not None:
originalChunkDims = collections.OrderedDict(
zip(originalDescription.axes, originalDescription.chunks))
datasetDescription.chunks = map(lambda a: originalChunkDims[a],
datasetDescription.axes)
datasetDescription.chunks = map(
min, zip(datasetDescription.chunks, self.Input.meta.shape))
# -- dtype
if datasetDescription.dtype != self.Input.meta.dtype:
dtype = self.Input.meta.dtype
if type(dtype) is numpy.dtype:
dtype = dtype.type
datasetDescription.dtype = dtype().__class__.__name__
# Create a unique hash for this blocking scheme.
# If it changes, we can't use any previous data.
sha = hashlib.sha1()
sha.update(str(tuple(datasetDescription.block_shape)))
sha.update(datasetDescription.axes)
sha.update(datasetDescription.block_file_name_format)
datasetDescription.hash_id = sha.hexdigest()
if datasetDescription != originalDescription:
descriptionFilePath = self.OutputDatasetDescription.value
logger.info("Overwriting dataset description: {}".format(
descriptionFilePath))
BlockwiseFileset.writeDescription(descriptionFilePath,
datasetDescription)
with open(descriptionFilePath, 'r') as f:
logger.info(f.read())
# Now open the dataset
blockwiseFileset = BlockwiseFileset(
self.OutputDatasetDescription.value)
taskInfos = self._prepareTaskInfos(blockwiseFileset.getAllBlockRois())
if blockwiseFileset.description.hash_id != originalDescription.hash_id:
# Something about our blocking scheme changed.
# Make sure all blocks are marked as NOT available.
# (Just in case some were left over from a previous run.)
for roi in taskInfos.keys():
blockwiseFileset.setBlockStatus(
roi[0], BlockwiseFileset.BLOCK_NOT_AVAILABLE)
return blockwiseFileset, taskInfos
class OpTaskWorker(Operator):
Input = InputSlot()
RoiString = InputSlot(stype='string')
TaskName = InputSlot(stype='string')
ConfigFilePath = InputSlot(stype='filestring')
OutputFilesetDescription = InputSlot(stype='filestring')
SecondaryInputs = InputSlot(level=1, optional=True)
SecondaryOutputDescriptions = InputSlot(level=1, optional=True)
ReturnCode = OutputSlot()
def __init__(self, *args, **kwargs):
super(OpTaskWorker, self).__init__(*args, **kwargs)
self.progressSignal = OrderedSignal()
self._primaryBlockwiseFileset = None
self._secondaryBlockwiseFilesets = []
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1, )
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset(
self.OutputFilesetDescription.value, 'a')
self._secondaryBlockwiseFilesets = []
for slot in self.SecondaryOutputDescriptions:
descriptionPath = slot.value
self._secondaryBlockwiseFilesets.append(
BlockwiseFileset(descriptionPath, 'a'))
def cleanUp(self):
self._closeFiles()
super(OpTaskWorker, self).cleanUp()
def _closeFiles(self):
if self._primaryBlockwiseFileset is not None:
self._primaryBlockwiseFileset.close()
for fileset in self._secondaryBlockwiseFilesets:
fileset.close()
self._primaryBlockwiseFileset = None
self._secondaryBlockwiseFilesets = []
def execute(self, slot, subindex, ignored_roi, result):
configFilePath = self.ConfigFilePath.value
config = parseClusterConfigFile(configFilePath)
blockwiseFileset = self._primaryBlockwiseFileset
# Check axis compatibility
inputAxes = self.Input.meta.getTaggedShape().keys()
outputAxes = list(blockwiseFileset.description.axes)
assert set(inputAxes) == set(outputAxes), \
"Output dataset has the wrong set of axes. Input axes: {}, Output axes: {}".format( "".join(inputAxes), "".join(outputAxes) )
roiString = self.RoiString.value
roi = Roi.loads(roiString)
if len(roi.start) != len(self.Input.meta.shape):
assert False, "Task roi: {} is not valid for this input. Did the master launch this task correctly?".format(
roiString)
logger.info("Executing for roi: {}".format(roi))
if config.use_node_local_scratch:
assert False, "FIXME."
assert (
blockwiseFileset.getEntireBlockRoi(roi.start)[1] == roi.stop
).all(
), "Each task must execute exactly one full block. ({},{}) is not a valid block roi.".format(
roi.start, roi.stop)
assert self.Input.ready()
# Convert the task subrequest shape dict into a shape for this dataset (and axisordering)
subrequest_shape = map(
lambda tag: config.task_subrequest_shape[tag.key],
self.Input.meta.axistags)
primary_subrequest_shape = self._primaryBlockwiseFileset.description.sub_block_shape
if primary_subrequest_shape is not None:
# If the output dataset specified a sub_block_shape, override the cluster config
subrequest_shape = primary_subrequest_shape
with Timer() as computeTimer:
# Stream the data out to disk.
streamer = BigRequestStreamer(self.Input, (roi.start, roi.stop),
subrequest_shape,
config.task_parallel_subrequests)
streamer.progressSignal.subscribe(self.progressSignal)
streamer.resultSignal.subscribe(self._handlePrimaryResultBlock)
streamer.execute()
# Now the block is ready. Update the status.
blockwiseFileset.setBlockStatus(roi.start,
BlockwiseFileset.BLOCK_AVAILABLE)
logger.info("Finished task in {} seconds".format(
computeTimer.seconds()))
result[0] = True
return result
def propagateDirty(self, slot, subindex, roi):
self.ReturnCode.setDirty(slice(None))
def _handlePrimaryResultBlock(self, roi, result):
# First write the primary
self._primaryBlockwiseFileset.writeData(roi, result)
# Get this block's index with respect to the primary dataset
sub_block_index = roi[
0] / self._primaryBlockwiseFileset.description.sub_block_shape
# Now request the secondaries
for slot, fileset in zip(self.SecondaryInputs,
self._secondaryBlockwiseFilesets):
# Compute the corresponding sub_block in this output dataset
sub_block_shape = fileset.description.sub_block_shape
sub_block_start = sub_block_index * sub_block_shape
sub_block_stop = sub_block_start + sub_block_shape
sub_block_stop = numpy.minimum(sub_block_stop,
fileset.description.shape)
sub_block_roi = (sub_block_start, sub_block_stop)
secondary_result = slot(*sub_block_roi).wait()
fileset.writeData(sub_block_roi, secondary_result)
class OpTaskWorker(Operator):
Input = InputSlot()
RoiString = InputSlot(stype='string')
TaskName = InputSlot(stype='string')
ConfigFilePath = InputSlot(stype='filestring')
OutputFilesetDescription = InputSlot(stype='filestring')
ReturnCode = OutputSlot()
def __init__(self, *args, **kwargs):
super( OpTaskWorker, self ).__init__( *args, **kwargs )
self.progressSignal = OrderedSignal()
self._primaryBlockwiseFileset = None
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1,)
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset( self.OutputFilesetDescription.value, 'a' )
def cleanUp(self):
self._closeFiles()
super( OpTaskWorker, self ).cleanUp()
def _closeFiles(self):
if self._primaryBlockwiseFileset is not None:
self._primaryBlockwiseFileset.close()
self._primaryBlockwiseFileset = None
def execute(self, slot, subindex, ignored_roi, result):
configFilePath = self.ConfigFilePath.value
config = parseClusterConfigFile( configFilePath )
blockwiseFileset = self._primaryBlockwiseFileset
# Check axis compatibility
inputAxes = self.Input.meta.getTaggedShape().keys()
outputAxes = list(blockwiseFileset.description.axes)
assert set(inputAxes) == set(outputAxes), \
"Output dataset has the wrong set of axes. Input axes: {}, Output axes: {}".format( "".join(inputAxes), "".join(outputAxes) )
roiString = self.RoiString.value
roi = Roi.loads(roiString)
if len( roi.start ) != len( self.Input.meta.shape ):
assert False, "Task roi: {} is not valid for this input. Did the master launch this task correctly?".format( roiString )
logger.info( "Executing for roi: {}".format(roi) )
if config.use_node_local_scratch:
assert False, "FIXME."
assert (blockwiseFileset.getEntireBlockRoi( roi.start )[1] == roi.stop).all(), "Each task must execute exactly one full block. ({},{}) is not a valid block roi.".format( roi.start, roi.stop )
assert self.Input.ready()
with Timer() as computeTimer:
# Stream the data out to disk.
request_blockshape = self._primaryBlockwiseFileset.description.sub_block_shape # Could be None. That's okay.
streamer = BigRequestStreamer(self.Input, (roi.start, roi.stop), request_blockshape )
streamer.progressSignal.subscribe( self.progressSignal )
streamer.resultSignal.subscribe( self._handlePrimaryResultBlock )
streamer.execute()
# Now the block is ready. Update the status.
blockwiseFileset.setBlockStatus( roi.start, BlockwiseFileset.BLOCK_AVAILABLE )
logger.info( "Finished task in {} seconds".format( computeTimer.seconds() ) )
result[0] = True
return result
def propagateDirty(self, slot, subindex, roi):
self.ReturnCode.setDirty( slice(None) )
def _handlePrimaryResultBlock(self, roi, result):
# First write the primary
self._primaryBlockwiseFileset.writeData(roi, result)
# Ask the workflow if there is any special post-processing to do...
self.get_workflow().postprocessClusterSubResult(roi, result, self._primaryBlockwiseFileset)
def get_workflow(self):
op = self
while not isinstance(op, Workflow):
op = op.parent
return op
def setupOutputs(self):
self.ReturnCode.meta.dtype = bool
self.ReturnCode.meta.shape = (1,)
self._closeFiles()
self._primaryBlockwiseFileset = BlockwiseFileset( self.OutputFilesetDescription.value, 'a' )
def setupClass(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format(cls.tempDir))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT",
"dataset_root_dir" : "blocks"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = \
"""
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format( remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription )
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, 'w') as f:
f.write(compositeDescription)
# Create a new fileset that views the same data and stores it the
# same way locally, but this time we'll use an offset 'view'
# Start with a copy of the non-offset description
offsetDescription = RESTfulBlockwiseFileset.readDescription(
cls.descriptionFilePath)
offsetDescription.local_description.view_origin = numpy.array(
[0, 20, 0])
offsetDescription.local_description.dataset_root_dir = "offset_blocks"
cls.descriptionFilePath_offset = os.path.join(
cls.tempDir, "description_offset.json")
RESTfulBlockwiseFileset.writeDescription(
cls.descriptionFilePath_offset, offsetDescription)
def setupClass(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format( cls.tempDir ))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT",
"dataset_root_dir" : "blocks"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = \
"""
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format( remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription )
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, 'w') as f:
f.write(compositeDescription)
# Create a new fileset that views the same data and stores it the
# same way locally, but this time we'll use an offset 'view'
# Start with a copy of the non-offset description
offsetDescription = RESTfulBlockwiseFileset.readDescription(cls.descriptionFilePath)
offsetDescription.local_description.view_origin = numpy.array([0,20,0])
offsetDescription.local_description.dataset_root_dir = "offset_blocks"
cls.descriptionFilePath_offset = os.path.join(cls.tempDir, "description_offset.json")
RESTfulBlockwiseFileset.writeDescription(cls.descriptionFilePath_offset, offsetDescription)
