def preload_source(t_query):
"""load info from example tile (image)
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
* :data:`OUTPUT.INFO`.``TYPE.NAME`` -- \
numpy datatype of any given tile
* :data:`RUNTIME.IMAGE`.``BLOCK.NAME`` -- \
numpy 3x1 array of any given tile shape
* :data:`OUTPUT.INFO`.``SIZE.NAME`` -- \
numpy 3x1 array of full volume shape
"""
# read all tifs in tifs folder
search = os.path.join(t_query.path, '*')
depth = len(list(glob.glob(search)))
# Should count files on filesystem
N_FILES = np.uint32([depth, 1, 1])
tile_0 = ImageStack.load_tile(t_query)
# Return empty if can't load first tile
if not len(tile_0):
return {}
# Get properties from example tile
FILE_SIZE = tile_0.shape
FULL_SIZE = FILE_SIZE * N_FILES
DATA_TYPE = str(tile_0.dtype)
# 'block-size', 'dimensions', and 'data-type'
k_block = t_query.RUNTIME.IMAGE.BLOCK.NAME
k_size = t_query.OUTPUT.INFO.SIZE.NAME
k_type = t_query.OUTPUT.INFO.TYPE.NAME
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(
common, **{
k_block: np.uint32([FILE_SIZE]),
k_size: np.uint32(FULL_SIZE),
k_type: DATA_TYPE,
})
def preload_source(t_query):
"""load info from example tile (image)
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
* :data:`OUTPUT.INFO`.``TYPE.NAME`` -- \
numpy datatype of any given tile
* :data:`RUNTIME.IMAGE`.``BLOCK.NAME`` -- \
numpy 3x1 array of any given tile shape
* :data:`OUTPUT.INFO`.``SIZE.NAME`` -- \
numpy 3x1 array of full volume shape
"""
# read all tifs in tifs folder
search = os.path.join(t_query.path, '*')
depth = len(list(glob.glob(search)))
# Should count files on filesystem
N_FILES = np.uint32([depth, 1, 1])
tile_0 = ImageStack.load_tile(t_query)
# Return empty if can't load first tile
if not len(tile_0):
return {}
# Get properties from example tile
FILE_SIZE = tile_0.shape
FULL_SIZE = FILE_SIZE * N_FILES
DATA_TYPE = str(tile_0.dtype)
# 'block-size', 'dimensions', and 'data-type'
k_block = t_query.RUNTIME.IMAGE.BLOCK.NAME
k_size = t_query.OUTPUT.INFO.SIZE.NAME
k_type = t_query.OUTPUT.INFO.TYPE.NAME
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(common, **{
k_block: np.uint32([FILE_SIZE]),
k_size: np.uint32(FULL_SIZE),
k_type: DATA_TYPE,
})
def preload_source(t_query):
"""load info from example tile (image)
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
* :data:`OUTPUT.INFO`.``TYPE.NAME`` -- \
numpy datatype of any given tile
* :data:`RUNTIME.IMAGE`.``BLOCK.NAME`` -- \
numpy 3x1 array of any given tile shape
* :data:`OUTPUT.INFO`.``SIZE.NAME`` -- \
numpy 3x1 array of full volume shape
"""
common = Datasource.preload_source(t_query)
return common
def preload_source(t_query):
"""load info from example tile (image)
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
Will be empty if filename does not give \
a valid mojo directory.
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Keyword names
output = t_query.OUTPUT.INFO
runtime = t_query.RUNTIME.IMAGE
k_format = runtime.SOURCE.MOJO.FORMAT.NAME
# Get the name and ending of the target folder
path_name = t_query.OUTPUT.INFO.PATH.VALUE
meta_file = os.path.join(path_name, Mojo._meta)
# Return if no meta file for mojo
if not os.path.exists(meta_file):
return {}
# Load the meta info
meta_info = ET.parse(meta_file).getroot().attrib
# Estimate the data type
n_bytes = int(meta_info['numBytesPerVoxel'])
dtype = 'uint{}'.format(8 * n_bytes)
# Get the data file exension
file_ext = meta_info['fileExtension']
# Get the block shape and full size
block_z = meta_info['numVoxelsPerTileZ']
block_y = meta_info['numVoxelsPerTileY']
block_x = meta_info['numVoxelsPerTileX']
full_z = meta_info['numVoxelsZ']
full_y = meta_info['numVoxelsY']
full_x = meta_info['numVoxelsX']
####
# Get max blocksizes for different resolutions
####
lo_res = int(meta_info['numTilesW'])
block_size = [block_z, block_y, block_x]
# Specify block_size for all resolutions
block_array = [block_size for res in range(lo_res)]
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(common, **{
runtime.BLOCK.NAME: np.uint32(block_array),
output.SIZE.NAME: np.uint32([full_z, full_y, full_x]),
output.TYPE.NAME: dtype,
k_format: file_ext,
})
def preload_source(t_query):
"""load info from example tile (image)
Then gets three needed values from the given \
path from the :class:`TileQuery` t_query
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
Will be empty if filename does not give \
a valid json file pointing to the tiff grid.
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Keyword names
output = t_query.OUTPUT.INFO
runtime = t_query.RUNTIME.IMAGE
boss_field = runtime.SOURCE.BOSS
info_field = boss_field.INFO
block_field = info_field.BLOCK
full_field = info_field.EXTENT
start_field = info_field.START
# Get the name and ending of the target file
filename = t_query.OUTPUT.INFO.PATH.VALUE
ending = os.path.splitext(filename)[1]
# Return if the ending is not json
if ending not in BossGrid._meta_files:
return {}
# Return if the path does not exist
if not os.path.exists(filename):
return {}
# Get function to read the metainfo file
order = BossGrid._meta_files.index(ending)
reader = BossGrid._read[order]
# Get information from json file
with open(filename, 'r') as jd:
# Get all the filenames
all_info = reader(jd)
boss = all_info.get(boss_field.ALL, [])
info = all_info.get(info_field.NAME, {})
# Return if no metadata
if not len(info):
return {}
# All the paths
path_dict = {}
any_path = None
# Origin of first tile
start_info = info.get(start_field.NAME, {})
start_list = map(start_info.get, start_field.ZYX)
# Set default first tile origin
if any([s is None for s in start_list]):
start_list = start_field.VALUE
# Extract offset of first tile
tile_start = np.uint64(start_list)
any_y, any_x = tile_start[1:]
# Shape of one tile
block_info = info.get(block_field.NAME, {})
block_list = map(block_info.get, block_field.ZYX)
# Return if no block shape
if not all(block_list):
return {}
# Shape of full volume
full_info = info.get(full_field.NAME)
full_extent = map(full_info.get, full_field.ZYX)
# Return if no full extent shape
if not all(full_extent):
return {}
# Block shape as a numpy array
block_shape = np.uint64(block_list)
if block_shape.shape != (3,):
return {}
# Finally, list all the mip levels
block_shapes = block_shape[np.newaxis]
# Full shape as a numpy array
full_bounds = np.uint64(full_extent)
if full_bounds.shape != (3,2):
return {}
# Finally, get the full shape from extent
full_shape = np.diff(full_bounds).T[0]
# All paths in dictionary
for d in boss:
path = d.get(boss_field.PATH, '')
# Update the maximum value
z,y,x = map(d.get, boss_field.ZYX)
z_format = x is None or y is None
# Set any path
if not any_path:
any_path = path
if z_format:
any_path = path.format(column=any_x, row=any_y)
if not os.path.exists(any_path):
any_path = None
# Allow for simple section formats
if z_format:
path_dict[z] = path
continue
# Allow for specific paths per tile
if z not in path_dict:
path_dict[z] = {
y: {
x: path
}
}
continue
# Add column to dictionary
if y not in path_dict[z]:
path_dict[z][y] = {
x: path
}
continue
# Add row to dictionary
path_dict[z][y][x] = path
# Return if no paths
if not any_path:
return {}
# Get the tile size from a tile
any_tile = BossGrid.imread(any_path)
any_dtype = str(any_tile.dtype)
# All keys to follow API
keywords = {
start_field.NAME: tile_start,
boss_field.PATHS.NAME: path_dict,
runtime.BLOCK.NAME: block_shapes,
output.SIZE.NAME: full_shape,
output.TYPE.NAME: any_dtype,
}
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(common, **keywords)
def preload_source(t_query):
"""load info from example tile (image)
Calls :meth:`valid_path` to get filename and \
inner dataset path for the full h5 image volume.
Then gets three needed values from the given \
path from the :class:`TileQuery` t_query
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
Will be empty if :meth:`valid_path` finds\
this filname to not give a valid h5 volume.
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Keyword names
output = t_query.OUTPUT.INFO
runtime = t_query.RUNTIME.IMAGE
k_h5 = runtime.SOURCE.HDF5.NAME
# Get the max block size in bytes for a single tile
max_bytes = t_query.RUNTIME.CACHE.MAX_BLOCK.VALUE
max_bytes = int(max_bytes / 64)
# Check if path is valid
keywords = HDF5.valid_path(t_query)
if not keywords:
return {}
# Validate highest in z file name and dataset
filename = keywords[k_h5][-1][0]
dataset = keywords[k_h5][-1][1]
offset = keywords[k_h5][-1][2]
# Load properties from H5 dataset
with h5py.File(filename, 'r') as fd:
# Get the volume
vol = fd[dataset]
# Get a shape for all the files
shape = np.uint32(vol.shape)
shape[0] += offset
####
# Get a blockshape as a flat section
####
# Get the bytes for a full slice
voxel_bytes = np.uint32(vol.dtype.itemsize)
slice_bytes = voxel_bytes * np.prod(shape[1:])
# Get the nearest tile size under cache limit
square_overage = np.ceil(slice_bytes / max_bytes)
side_scalar = np.ceil(np.sqrt(square_overage))
# Set the actual blocksize to be under the cache limit
plane_shape = np.ceil(shape[1:] / side_scalar)
max_block = np.r_[[64], plane_shape]
####
# Get max blocksizes for different resolutions
####
lo_res = 1
# Get all block sizes by halving the max block size
all_blocks = [shape / (2**res) for res in range(lo_res)]
block_array = np.clip(np.ceil(all_blocks), 1, max_block)
# return named keywords
keywords.update({
runtime.BLOCK.NAME: np.uint32(block_array),
output.SIZE.NAME: np.uint32(shape),
output.TYPE.NAME: str(HDF5.dtype(vol)),
})
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(common, **keywords)
def preload_source(t_query):
"""load info from example tile (image)
Then gets three needed values from the given \
path from the :class:`TileQuery` t_query
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
Will be empty if filename does not give \
a valid json file pointing to the tiff grid.
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Keyword names
output = t_query.OUTPUT.INFO
runtime = t_query.RUNTIME.IMAGE
boss_field = runtime.SOURCE.BOSS
info_field = boss_field.INFO
block_field = info_field.BLOCK
full_field = info_field.EXTENT
start_field = info_field.START
# Get the name and ending of the target file
filename = t_query.OUTPUT.INFO.PATH.VALUE
ending = os.path.splitext(filename)[1]
# Return if the ending is not json
if ending not in BossGrid._meta_files:
return {}
# Return if the path does not exist
if not os.path.exists(filename):
return {}
# Get function to read the metainfo file
order = BossGrid._meta_files.index(ending)
reader = BossGrid._read[order]
# Get information from json file
with open(filename, 'r') as jd:
# Get all the filenames
all_info = reader(jd)
boss = all_info.get(boss_field.ALL, [])
info = all_info.get(info_field.NAME, {})
# Return if no metadata
if not len(info):
return {}
# All the paths
path_dict = {}
any_path = None
# Origin of first tile
start_info = info.get(start_field.NAME, {})
start_list = map(start_info.get, start_field.ZYX)
# Set default first tile origin
if any([s is None for s in start_list]):
start_list = start_field.VALUE
# Extract offset of first tile
tile_start = np.uint64(start_list)
any_y, any_x = tile_start[1:]
# Shape of one tile
block_info = info.get(block_field.NAME, {})
block_list = map(block_info.get, block_field.ZYX)
# Return if no block shape
if not all(block_list):
return {}
# Shape of full volume
full_info = info.get(full_field.NAME)
full_extent = map(full_info.get, full_field.ZYX)
# Return if no full extent shape
if not all(full_extent):
return {}
# Block shape as a numpy array
block_shape = np.uint64(block_list)
if block_shape.shape != (3, ):
return {}
# Finally, list all the mip levels
block_shapes = block_shape[np.newaxis]
# Full shape as a numpy array
full_bounds = np.uint64(full_extent)
if full_bounds.shape != (3, 2):
return {}
# Finally, get the full shape from extent
full_shape = np.diff(full_bounds).T[0]
# All paths in dictionary
for d in boss:
path = d.get(boss_field.PATH, '')
# Update the maximum value
z, y, x = map(d.get, boss_field.ZYX)
z_format = x is None or y is None
# Set any path
if not any_path:
any_path = path
if z_format:
any_path = path.format(column=any_x, row=any_y)
if not os.path.exists(any_path):
any_path = None
# Allow for simple section formats
if z_format:
path_dict[z] = path
continue
# Allow for specific paths per tile
if z not in path_dict:
path_dict[z] = {y: {x: path}}
continue
# Add column to dictionary
if y not in path_dict[z]:
path_dict[z][y] = {x: path}
continue
# Add row to dictionary
path_dict[z][y][x] = path
# Return if no paths
if not any_path:
return {}
# Get the tile size from a tile
any_tile = BossGrid.imread(any_path)
any_dtype = str(any_tile.dtype)
# All keys to follow API
keywords = {
start_field.NAME: tile_start,
boss_field.PATHS.NAME: path_dict,
runtime.BLOCK.NAME: block_shapes,
output.SIZE.NAME: full_shape,
output.TYPE.NAME: any_dtype,
}
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(common, **keywords)
def preload_source(t_query):
"""load info from example tile (image)
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
Will be empty if filename does not give \
a valid mojo directory.
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Keyword names
output = t_query.OUTPUT.INFO
runtime = t_query.RUNTIME.IMAGE
k_format = runtime.SOURCE.MOJO.FORMAT.NAME
# Get the name and ending of the target folder
path_name = t_query.OUTPUT.INFO.PATH.VALUE
meta_file = os.path.join(path_name, Mojo._meta)
# Return if no meta file for mojo
if not os.path.exists(meta_file):
return {}
# Load the meta info
meta_info = ET.parse(meta_file).getroot().attrib
# Estimate the data type
n_bytes = int(meta_info['numBytesPerVoxel'])
dtype = 'uint{}'.format(8 * n_bytes)
# Get the data file exension
file_ext = meta_info['fileExtension']
# Get the block shape and full size
block_z = meta_info['numVoxelsPerTileZ']
block_y = meta_info['numVoxelsPerTileY']
block_x = meta_info['numVoxelsPerTileX']
full_z = meta_info['numVoxelsZ']
full_y = meta_info['numVoxelsY']
full_x = meta_info['numVoxelsX']
####
# Get max blocksizes for different resolutions
####
lo_res = int(meta_info['numTilesW'])
block_size = [block_z, block_y, block_x]
# Specify block_size for all resolutions
block_array = [block_size for res in range(lo_res)]
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(
common, **{
runtime.BLOCK.NAME: np.uint32(block_array),
output.SIZE.NAME: np.uint32([full_z, full_y, full_x]),
output.TYPE.NAME: dtype,
k_format: file_ext,
})
def preload_source(t_query):
"""load info from example tile (image)
Calls :meth:`valid_path` to get filename and \
inner dataset path for the full h5 image volume.
Then gets three needed values from the given \
path from the :class:`TileQuery` t_query
Arguments
-----------
t_query: :class:`TileQuery`
Only the file path is needed
Returns
--------
dict
Will be empty if :meth:`valid_path` finds\
this filname to not give a valid h5 volume.
* :class:`RUNTIME` ``.IMAGE.BLOCK.NAME``
(numpy.ndarray) -- 3x1 for any give tile shape
* :class:`OUTPUT` ``.INFO.TYPE.NAME``
(str) -- numpy dtype of any given tile
* :class:`OUTPUT` ``.INFO.SIZE.NAME``
(numpy.ndarray) -- 3x1 for full volume shape
"""
# Keyword names
output = t_query.OUTPUT.INFO
runtime = t_query.RUNTIME.IMAGE
k_h5 = runtime.SOURCE.HDF5.NAME
# Get the max block size in bytes for a single tile
max_bytes = t_query.RUNTIME.CACHE.MAX_BLOCK.VALUE
max_bytes = int(max_bytes/64)
# Check if path is valid
keywords = HDF5.valid_path(t_query)
if not keywords:
return {}
# Validate highest in z file name and dataset
filename = keywords[k_h5][-1][0]
dataset = keywords[k_h5][-1][1]
offset = keywords[k_h5][-1][2]
# Load properties from H5 dataset
with h5py.File(filename,'r') as fd:
# Get the volume
vol = fd[dataset]
# Get a shape for all the files
shape = np.uint32(vol.shape)
shape[0] += offset
####
# Get a blockshape as a flat section
####
# Get the bytes for a full slice
voxel_bytes = np.uint32(vol.dtype.itemsize)
slice_bytes = voxel_bytes * np.prod(shape[1:])
# Get the nearest tile size under cache limit
square_overage = np.ceil(slice_bytes / max_bytes)
side_scalar = np.ceil(np.sqrt(square_overage))
# Set the actual blocksize to be under the cache limit
plane_shape = np.ceil(shape[1:] / side_scalar)
max_block = np.r_[[64], plane_shape]
####
# Get max blocksizes for different resolutions
####
lo_res = 1
# Get all block sizes by halving the max block size
all_blocks = [shape/(2**res) for res in range(lo_res)]
block_array = np.clip(np.ceil(all_blocks), 1, max_block)
# return named keywords
keywords.update({
runtime.BLOCK.NAME: np.uint32(block_array),
output.SIZE.NAME: np.uint32(shape),
output.TYPE.NAME: str(HDF5.dtype(vol)),
})
# Combine results with parent method
common = Datasource.preload_source(t_query)
return dict(common, **keywords)
