def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug("Loading config file...")
with open(self.configpath, 'w') as f:
f.write(testConfig)
logger.debug("Creating random test data...")
bfs = BlockwiseFileset(self.configpath, 'a')
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint(255,
size=dataShape).astype(numpy.uint8)
logger.debug("Writing test data...")
datasetRoi = ([0, 0, 0, 0, 0], dataShape)
bfs.writeData(datasetRoi, self.data)
block_starts = getIntersectingBlocks(bfs.description.block_shape,
datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def setUp(self):
"""
Create a blockwise fileset to test with.
"""
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,100,1],
"dtype" : "numpy.uint8",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
self.tempDir = tempfile.mkdtemp()
self.configpath = os.path.join(self.tempDir, "config.json")
logger.debug( "Loading config file..." )
with open(self.configpath, 'w') as f:
f.write(testConfig)
logger.debug( "Creating random test data..." )
bfs = BlockwiseFileset( self.configpath, 'a' )
dataShape = tuple(bfs.description.shape)
self.data = numpy.random.randint( 255, size=dataShape ).astype(numpy.uint8)
logger.debug( "Writing test data..." )
datasetRoi = ([0,0,0,0,0], dataShape)
bfs.writeData( datasetRoi, self.data )
block_starts = getIntersectingBlocks(bfs.description.block_shape, datasetRoi)
for block_start in block_starts:
bfs.setBlockStatus(block_start, BlockwiseFileset.BLOCK_AVAILABLE)
bfs.close()
def setup_class(cls):
if platform.system() == "Windows":
# On windows, there are errors, and we make no attempt to solve them (at the moment).
pytest.skip("Windows")
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
pytest.skip(
"System is not configured to allow opening a lot of files")
testConfig = """
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,10,20,5,1],
"dtype" : "object",
"block_shape" : [1, 5, 4, 1, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "config.json")
with open(cls.description_path, "w") as f:
f.write(testConfig)
logger.debug("Loading config file...")
cls.bfs = BlockwiseFileset(cls.description_path, "a")
cls.dataShape = tuple(cls.bfs.description.shape)
def make_dummy_dict(x):
return {str(x): numpy.array([x, x])}
vec_make_dummy_dict = numpy.vectorize(make_dummy_dict)
int_data = numpy.random.randint(255,
size=cls.dataShape).astype(numpy.uint8)
dict_data = vec_make_dummy_dict(int_data)
cls.data = dict_data
def setup_class(cls):
if platform.system() == "Windows":
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = """
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,10,20,5,1],
"dtype" : "object",
"block_shape" : [1, 5, 4, 1, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "config.json")
with open(cls.description_path, "w") as f:
f.write(testConfig)
logger.debug("Loading config file...")
cls.bfs = BlockwiseFileset(cls.description_path, "a")
cls.dataShape = tuple(cls.bfs.description.shape)
def make_dummy_dict(x):
return {str(x): numpy.array([x, x])}
vec_make_dummy_dict = numpy.vectorize(make_dummy_dict)
int_data = numpy.random.randint(255, size=cls.dataShape).astype(numpy.uint8)
dict_data = vec_make_dummy_dict(int_data)
cls.data = dict_data
def setup_class(cls):
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,200,1],
"dtype" : "numpy.uint8",
"compression" : "lzf",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "description.json")
with open(cls.description_path, 'w') as f:
f.write(testConfig)
logger.debug("Loading config file...")
cls.bfs = BlockwiseFileset(cls.description_path, 'a')
cls.dataShape = tuple(cls.bfs.description.shape)
logger.debug("Creating random test data...")
cls.data = numpy.random.randint(255,
size=cls.dataShape).astype(numpy.uint8)
def setupClass(cls):
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
testConfig = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "synapse_small",
"format" : "hdf5",
"axes" : "txyzc",
"shape" : [1,400,400,200,1],
"dtype" : "numpy.uint8",
"compression" : "lzf",
"block_shape" : [1, 50, 50, 50, 100],
"block_file_name_format" : "cube{roiString}.h5/volume/data"
}
"""
cls.tempDir = tempfile.mkdtemp()
cls.description_path = os.path.join(cls.tempDir, "description.json")
with open(cls.description_path, 'w') as f:
f.write(testConfig)
logger.debug( "Loading config file..." )
cls.bfs = BlockwiseFileset( cls.description_path, 'a' )
cls.dataShape = tuple(cls.bfs.description.shape)
logger.debug( "Creating random test data..." )
cls.data = numpy.random.randint(255, size=cls.dataShape ).astype(numpy.uint8)
def setup_class(cls):
if platform.system() == "Windows":
pytest.skip("Windows")
try:
from lazyflow.utility.io_util.blockwiseFileset import BlockwiseFileset
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
pytest.skip(
"System is not configured to allow opening a lot of files")
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format(cls.tempDir))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"## NOTE 1": "The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"## NOTE 2": "The website says that the data goes up to plane 4156, but it actually errors out past 4150",
"origin_offset" : [2917, 0, 0],
"bounds" : [4150, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = """
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = """
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format(remote_description=Bock11VolumeDescription,
local_description=blockwiseFilesetDescription)
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, "w") as f:
f.write(compositeDescription)
def setupClass(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == 'Windows':
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format(cls.tempDir))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = \
"""
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT",
"dataset_root_dir" : "blocks"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = \
"""
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format( remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription )
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, 'w') as f:
f.write(compositeDescription)
# Create a new fileset that views the same data and stores it the
# same way locally, but this time we'll use an offset 'view'
# Start with a copy of the non-offset description
offsetDescription = RESTfulBlockwiseFileset.readDescription(
cls.descriptionFilePath)
offsetDescription.local_description.view_origin = numpy.array(
[0, 20, 0])
offsetDescription.local_description.dataset_root_dir = "offset_blocks"
cls.descriptionFilePath_offset = os.path.join(
cls.tempDir, "description_offset.json")
RESTfulBlockwiseFileset.writeDescription(
cls.descriptionFilePath_offset, offsetDescription)
def setup_class(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == "Windows":
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
from lazyflow.utility.io_util.blockwiseFileset import BlockwiseFileset
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format(cls.tempDir))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"## NOTE 1": "The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"## NOTE 2": "The website says that the data goes up to plane 4156, but it actually errors out past 4150",
"origin_offset" : [2917, 0, 0],
"bounds" : [4150, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = """
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = """
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format(
remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription
)
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, "w") as f:
f.write(compositeDescription)
def setup_class(cls):
# The openconnectome site appears to be down at the moment.
# This test fails when that happens...
raise nose.SkipTest
if platform.system() == "Windows":
# On windows, there are errors, and we make no attempt to solve them (at the moment).
raise nose.SkipTest
try:
BlockwiseFileset._prepare_system()
except ValueError:
# If the system isn't configured to allow lots of open files, we can't run this test.
raise nose.SkipTest
cls.tempDir = tempfile.mkdtemp()
logger.debug("Working in {}".format(cls.tempDir))
# Create the two sub-descriptions
Bock11VolumeDescription = """
{
"_schema_name" : "RESTful-volume-description",
"_schema_version" : 1.0,
"name" : "Bock11-level0",
"format" : "hdf5",
"axes" : "zyx",
"##NOTE":"The first z-slice of the bock dataset is 2917, so the origin_offset must be at least 2917",
"origin_offset" : [2917, 50000, 50000],
"bounds" : [4156, 135424, 119808],
"dtype" : "numpy.uint8",
"url_format" : "http://openconnecto.me/ocp/ca/bock11/hdf5/0/{x_start},{x_stop}/{y_start},{y_stop}/{z_start},{z_stop}/",
"hdf5_dataset" : "CUTOUT"
}
"""
blockwiseFilesetDescription = """
{
"_schema_name" : "blockwise-fileset-description",
"_schema_version" : 1.0,
"name" : "bock11-blocks",
"format" : "hdf5",
"axes" : "zyx",
"shape" : [40,40,40],
"dtype" : "numpy.uint8",
"block_shape" : [20, 20, 20],
"block_file_name_format" : "block-{roiString}.h5/CUTOUT",
"dataset_root_dir" : "blocks"
}
"""
# Combine them into the composite description (see RESTfulBlockwiseFileset.DescriptionFields)
compositeDescription = """
{{
"_schema_name" : "RESTful-blockwise-fileset-description",
"_schema_version" : 1.0,
"remote_description" : {remote_description},
"local_description" : {local_description}
}}
""".format(
remote_description=Bock11VolumeDescription, local_description=blockwiseFilesetDescription
)
# Create the description file
cls.descriptionFilePath = os.path.join(cls.tempDir, "description.json")
with open(cls.descriptionFilePath, "w") as f:
f.write(compositeDescription)
# Create a new fileset that views the same data and stores it the
# same way locally, but this time we'll use an offset 'view'
# Start with a copy of the non-offset description
offsetDescription = RESTfulBlockwiseFileset.readDescription(cls.descriptionFilePath)
offsetDescription.local_description.view_origin = numpy.array([0, 20, 0])
offsetDescription.local_description.dataset_root_dir = "offset_blocks"
cls.descriptionFilePath_offset = os.path.join(cls.tempDir, "description_offset.json")
RESTfulBlockwiseFileset.writeDescription(cls.descriptionFilePath_offset, offsetDescription)