def preload_source(t_query): """load info from example tile (image) Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict * :data:`OUTPUT.INFO`.``TYPE.NAME`` -- \ numpy datatype of any given tile * :data:`RUNTIME.IMAGE`.``BLOCK.NAME`` -- \ numpy 3x1 array of any given tile shape * :data:`OUTPUT.INFO`.``SIZE.NAME`` -- \ numpy 3x1 array of full volume shape """ # read all tifs in tifs folder search = os.path.join(t_query.path, '*') depth = len(list(glob.glob(search))) # Should count files on filesystem N_FILES = np.uint32([depth, 1, 1]) tile_0 = ImageStack.load_tile(t_query) # Return empty if can't load first tile if not len(tile_0): return {} # Get properties from example tile FILE_SIZE = tile_0.shape FULL_SIZE = FILE_SIZE * N_FILES DATA_TYPE = str(tile_0.dtype) # 'block-size', 'dimensions', and 'data-type' k_block = t_query.RUNTIME.IMAGE.BLOCK.NAME k_size = t_query.OUTPUT.INFO.SIZE.NAME k_type = t_query.OUTPUT.INFO.TYPE.NAME # Combine results with parent method common = Datasource.preload_source(t_query) return dict( common, **{ k_block: np.uint32([FILE_SIZE]), k_size: np.uint32(FULL_SIZE), k_type: DATA_TYPE, })
def preload_source(t_query): """load info from example tile (image) Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict * :data:`OUTPUT.INFO`.``TYPE.NAME`` -- \ numpy datatype of any given tile * :data:`RUNTIME.IMAGE`.``BLOCK.NAME`` -- \ numpy 3x1 array of any given tile shape * :data:`OUTPUT.INFO`.``SIZE.NAME`` -- \ numpy 3x1 array of full volume shape """ # read all tifs in tifs folder search = os.path.join(t_query.path, '*') depth = len(list(glob.glob(search))) # Should count files on filesystem N_FILES = np.uint32([depth, 1, 1]) tile_0 = ImageStack.load_tile(t_query) # Return empty if can't load first tile if not len(tile_0): return {} # Get properties from example tile FILE_SIZE = tile_0.shape FULL_SIZE = FILE_SIZE * N_FILES DATA_TYPE = str(tile_0.dtype) # 'block-size', 'dimensions', and 'data-type' k_block = t_query.RUNTIME.IMAGE.BLOCK.NAME k_size = t_query.OUTPUT.INFO.SIZE.NAME k_type = t_query.OUTPUT.INFO.TYPE.NAME # Combine results with parent method common = Datasource.preload_source(t_query) return dict(common, **{ k_block: np.uint32([FILE_SIZE]), k_size: np.uint32(FULL_SIZE), k_type: DATA_TYPE, })
def preload_source(t_query): """load info from example tile (image) Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict * :data:`OUTPUT.INFO`.``TYPE.NAME`` -- \ numpy datatype of any given tile * :data:`RUNTIME.IMAGE`.``BLOCK.NAME`` -- \ numpy 3x1 array of any given tile shape * :data:`OUTPUT.INFO`.``SIZE.NAME`` -- \ numpy 3x1 array of full volume shape """ common = Datasource.preload_source(t_query) return common
def preload_source(t_query): """load info from example tile (image) Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict Will be empty if filename does not give \ a valid mojo directory. * :class:`RUNTIME` ``.IMAGE.BLOCK.NAME`` (numpy.ndarray) -- 3x1 for any give tile shape * :class:`OUTPUT` ``.INFO.TYPE.NAME`` (str) -- numpy dtype of any given tile * :class:`OUTPUT` ``.INFO.SIZE.NAME`` (numpy.ndarray) -- 3x1 for full volume shape """ # Keyword names output = t_query.OUTPUT.INFO runtime = t_query.RUNTIME.IMAGE k_format = runtime.SOURCE.MOJO.FORMAT.NAME # Get the name and ending of the target folder path_name = t_query.OUTPUT.INFO.PATH.VALUE meta_file = os.path.join(path_name, Mojo._meta) # Return if no meta file for mojo if not os.path.exists(meta_file): return {} # Load the meta info meta_info = ET.parse(meta_file).getroot().attrib # Estimate the data type n_bytes = int(meta_info['numBytesPerVoxel']) dtype = 'uint{}'.format(8 * n_bytes) # Get the data file exension file_ext = meta_info['fileExtension'] # Get the block shape and full size block_z = meta_info['numVoxelsPerTileZ'] block_y = meta_info['numVoxelsPerTileY'] block_x = meta_info['numVoxelsPerTileX'] full_z = meta_info['numVoxelsZ'] full_y = meta_info['numVoxelsY'] full_x = meta_info['numVoxelsX'] #### # Get max blocksizes for different resolutions #### lo_res = int(meta_info['numTilesW']) block_size = [block_z, block_y, block_x] # Specify block_size for all resolutions block_array = [block_size for res in range(lo_res)] # Combine results with parent method common = Datasource.preload_source(t_query) return dict(common, **{ runtime.BLOCK.NAME: np.uint32(block_array), output.SIZE.NAME: np.uint32([full_z, full_y, full_x]), output.TYPE.NAME: dtype, k_format: file_ext, })
def preload_source(t_query): """load info from example tile (image) Then gets three needed values from the given \ path from the :class:`TileQuery` t_query Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict Will be empty if filename does not give \ a valid json file pointing to the tiff grid. * :class:`RUNTIME` ``.IMAGE.BLOCK.NAME`` (numpy.ndarray) -- 3x1 for any give tile shape * :class:`OUTPUT` ``.INFO.TYPE.NAME`` (str) -- numpy dtype of any given tile * :class:`OUTPUT` ``.INFO.SIZE.NAME`` (numpy.ndarray) -- 3x1 for full volume shape """ # Keyword names output = t_query.OUTPUT.INFO runtime = t_query.RUNTIME.IMAGE boss_field = runtime.SOURCE.BOSS info_field = boss_field.INFO block_field = info_field.BLOCK full_field = info_field.EXTENT start_field = info_field.START # Get the name and ending of the target file filename = t_query.OUTPUT.INFO.PATH.VALUE ending = os.path.splitext(filename)[1] # Return if the ending is not json if ending not in BossGrid._meta_files: return {} # Return if the path does not exist if not os.path.exists(filename): return {} # Get function to read the metainfo file order = BossGrid._meta_files.index(ending) reader = BossGrid._read[order] # Get information from json file with open(filename, 'r') as jd: # Get all the filenames all_info = reader(jd) boss = all_info.get(boss_field.ALL, []) info = all_info.get(info_field.NAME, {}) # Return if no metadata if not len(info): return {} # All the paths path_dict = {} any_path = None # Origin of first tile start_info = info.get(start_field.NAME, {}) start_list = map(start_info.get, start_field.ZYX) # Set default first tile origin if any([s is None for s in start_list]): start_list = start_field.VALUE # Extract offset of first tile tile_start = np.uint64(start_list) any_y, any_x = tile_start[1:] # Shape of one tile block_info = info.get(block_field.NAME, {}) block_list = map(block_info.get, block_field.ZYX) # Return if no block shape if not all(block_list): return {} # Shape of full volume full_info = info.get(full_field.NAME) full_extent = map(full_info.get, full_field.ZYX) # Return if no full extent shape if not all(full_extent): return {} # Block shape as a numpy array block_shape = np.uint64(block_list) if block_shape.shape != (3,): return {} # Finally, list all the mip levels block_shapes = block_shape[np.newaxis] # Full shape as a numpy array full_bounds = np.uint64(full_extent) if full_bounds.shape != (3,2): return {} # Finally, get the full shape from extent full_shape = np.diff(full_bounds).T[0] # All paths in dictionary for d in boss: path = d.get(boss_field.PATH, '') # Update the maximum value z,y,x = map(d.get, boss_field.ZYX) z_format = x is None or y is None # Set any path if not any_path: any_path = path if z_format: any_path = path.format(column=any_x, row=any_y) if not os.path.exists(any_path): any_path = None # Allow for simple section formats if z_format: path_dict[z] = path continue # Allow for specific paths per tile if z not in path_dict: path_dict[z] = { y: { x: path } } continue # Add column to dictionary if y not in path_dict[z]: path_dict[z][y] = { x: path } continue # Add row to dictionary path_dict[z][y][x] = path # Return if no paths if not any_path: return {} # Get the tile size from a tile any_tile = BossGrid.imread(any_path) any_dtype = str(any_tile.dtype) # All keys to follow API keywords = { start_field.NAME: tile_start, boss_field.PATHS.NAME: path_dict, runtime.BLOCK.NAME: block_shapes, output.SIZE.NAME: full_shape, output.TYPE.NAME: any_dtype, } # Combine results with parent method common = Datasource.preload_source(t_query) return dict(common, **keywords)
def preload_source(t_query): """load info from example tile (image) Calls :meth:`valid_path` to get filename and \ inner dataset path for the full h5 image volume. Then gets three needed values from the given \ path from the :class:`TileQuery` t_query Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict Will be empty if :meth:`valid_path` finds\ this filname to not give a valid h5 volume. * :class:`RUNTIME` ``.IMAGE.BLOCK.NAME`` (numpy.ndarray) -- 3x1 for any give tile shape * :class:`OUTPUT` ``.INFO.TYPE.NAME`` (str) -- numpy dtype of any given tile * :class:`OUTPUT` ``.INFO.SIZE.NAME`` (numpy.ndarray) -- 3x1 for full volume shape """ # Keyword names output = t_query.OUTPUT.INFO runtime = t_query.RUNTIME.IMAGE k_h5 = runtime.SOURCE.HDF5.NAME # Get the max block size in bytes for a single tile max_bytes = t_query.RUNTIME.CACHE.MAX_BLOCK.VALUE max_bytes = int(max_bytes / 64) # Check if path is valid keywords = HDF5.valid_path(t_query) if not keywords: return {} # Validate highest in z file name and dataset filename = keywords[k_h5][-1][0] dataset = keywords[k_h5][-1][1] offset = keywords[k_h5][-1][2] # Load properties from H5 dataset with h5py.File(filename, 'r') as fd: # Get the volume vol = fd[dataset] # Get a shape for all the files shape = np.uint32(vol.shape) shape[0] += offset #### # Get a blockshape as a flat section #### # Get the bytes for a full slice voxel_bytes = np.uint32(vol.dtype.itemsize) slice_bytes = voxel_bytes * np.prod(shape[1:]) # Get the nearest tile size under cache limit square_overage = np.ceil(slice_bytes / max_bytes) side_scalar = np.ceil(np.sqrt(square_overage)) # Set the actual blocksize to be under the cache limit plane_shape = np.ceil(shape[1:] / side_scalar) max_block = np.r_[[64], plane_shape] #### # Get max blocksizes for different resolutions #### lo_res = 1 # Get all block sizes by halving the max block size all_blocks = [shape / (2**res) for res in range(lo_res)] block_array = np.clip(np.ceil(all_blocks), 1, max_block) # return named keywords keywords.update({ runtime.BLOCK.NAME: np.uint32(block_array), output.SIZE.NAME: np.uint32(shape), output.TYPE.NAME: str(HDF5.dtype(vol)), }) # Combine results with parent method common = Datasource.preload_source(t_query) return dict(common, **keywords)
def preload_source(t_query): """load info from example tile (image) Then gets three needed values from the given \ path from the :class:`TileQuery` t_query Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict Will be empty if filename does not give \ a valid json file pointing to the tiff grid. * :class:`RUNTIME` ``.IMAGE.BLOCK.NAME`` (numpy.ndarray) -- 3x1 for any give tile shape * :class:`OUTPUT` ``.INFO.TYPE.NAME`` (str) -- numpy dtype of any given tile * :class:`OUTPUT` ``.INFO.SIZE.NAME`` (numpy.ndarray) -- 3x1 for full volume shape """ # Keyword names output = t_query.OUTPUT.INFO runtime = t_query.RUNTIME.IMAGE boss_field = runtime.SOURCE.BOSS info_field = boss_field.INFO block_field = info_field.BLOCK full_field = info_field.EXTENT start_field = info_field.START # Get the name and ending of the target file filename = t_query.OUTPUT.INFO.PATH.VALUE ending = os.path.splitext(filename)[1] # Return if the ending is not json if ending not in BossGrid._meta_files: return {} # Return if the path does not exist if not os.path.exists(filename): return {} # Get function to read the metainfo file order = BossGrid._meta_files.index(ending) reader = BossGrid._read[order] # Get information from json file with open(filename, 'r') as jd: # Get all the filenames all_info = reader(jd) boss = all_info.get(boss_field.ALL, []) info = all_info.get(info_field.NAME, {}) # Return if no metadata if not len(info): return {} # All the paths path_dict = {} any_path = None # Origin of first tile start_info = info.get(start_field.NAME, {}) start_list = map(start_info.get, start_field.ZYX) # Set default first tile origin if any([s is None for s in start_list]): start_list = start_field.VALUE # Extract offset of first tile tile_start = np.uint64(start_list) any_y, any_x = tile_start[1:] # Shape of one tile block_info = info.get(block_field.NAME, {}) block_list = map(block_info.get, block_field.ZYX) # Return if no block shape if not all(block_list): return {} # Shape of full volume full_info = info.get(full_field.NAME) full_extent = map(full_info.get, full_field.ZYX) # Return if no full extent shape if not all(full_extent): return {} # Block shape as a numpy array block_shape = np.uint64(block_list) if block_shape.shape != (3, ): return {} # Finally, list all the mip levels block_shapes = block_shape[np.newaxis] # Full shape as a numpy array full_bounds = np.uint64(full_extent) if full_bounds.shape != (3, 2): return {} # Finally, get the full shape from extent full_shape = np.diff(full_bounds).T[0] # All paths in dictionary for d in boss: path = d.get(boss_field.PATH, '') # Update the maximum value z, y, x = map(d.get, boss_field.ZYX) z_format = x is None or y is None # Set any path if not any_path: any_path = path if z_format: any_path = path.format(column=any_x, row=any_y) if not os.path.exists(any_path): any_path = None # Allow for simple section formats if z_format: path_dict[z] = path continue # Allow for specific paths per tile if z not in path_dict: path_dict[z] = {y: {x: path}} continue # Add column to dictionary if y not in path_dict[z]: path_dict[z][y] = {x: path} continue # Add row to dictionary path_dict[z][y][x] = path # Return if no paths if not any_path: return {} # Get the tile size from a tile any_tile = BossGrid.imread(any_path) any_dtype = str(any_tile.dtype) # All keys to follow API keywords = { start_field.NAME: tile_start, boss_field.PATHS.NAME: path_dict, runtime.BLOCK.NAME: block_shapes, output.SIZE.NAME: full_shape, output.TYPE.NAME: any_dtype, } # Combine results with parent method common = Datasource.preload_source(t_query) return dict(common, **keywords)
def preload_source(t_query): """load info from example tile (image) Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict Will be empty if filename does not give \ a valid mojo directory. * :class:`RUNTIME` ``.IMAGE.BLOCK.NAME`` (numpy.ndarray) -- 3x1 for any give tile shape * :class:`OUTPUT` ``.INFO.TYPE.NAME`` (str) -- numpy dtype of any given tile * :class:`OUTPUT` ``.INFO.SIZE.NAME`` (numpy.ndarray) -- 3x1 for full volume shape """ # Keyword names output = t_query.OUTPUT.INFO runtime = t_query.RUNTIME.IMAGE k_format = runtime.SOURCE.MOJO.FORMAT.NAME # Get the name and ending of the target folder path_name = t_query.OUTPUT.INFO.PATH.VALUE meta_file = os.path.join(path_name, Mojo._meta) # Return if no meta file for mojo if not os.path.exists(meta_file): return {} # Load the meta info meta_info = ET.parse(meta_file).getroot().attrib # Estimate the data type n_bytes = int(meta_info['numBytesPerVoxel']) dtype = 'uint{}'.format(8 * n_bytes) # Get the data file exension file_ext = meta_info['fileExtension'] # Get the block shape and full size block_z = meta_info['numVoxelsPerTileZ'] block_y = meta_info['numVoxelsPerTileY'] block_x = meta_info['numVoxelsPerTileX'] full_z = meta_info['numVoxelsZ'] full_y = meta_info['numVoxelsY'] full_x = meta_info['numVoxelsX'] #### # Get max blocksizes for different resolutions #### lo_res = int(meta_info['numTilesW']) block_size = [block_z, block_y, block_x] # Specify block_size for all resolutions block_array = [block_size for res in range(lo_res)] # Combine results with parent method common = Datasource.preload_source(t_query) return dict( common, **{ runtime.BLOCK.NAME: np.uint32(block_array), output.SIZE.NAME: np.uint32([full_z, full_y, full_x]), output.TYPE.NAME: dtype, k_format: file_ext, })
def preload_source(t_query): """load info from example tile (image) Calls :meth:`valid_path` to get filename and \ inner dataset path for the full h5 image volume. Then gets three needed values from the given \ path from the :class:`TileQuery` t_query Arguments ----------- t_query: :class:`TileQuery` Only the file path is needed Returns -------- dict Will be empty if :meth:`valid_path` finds\ this filname to not give a valid h5 volume. * :class:`RUNTIME` ``.IMAGE.BLOCK.NAME`` (numpy.ndarray) -- 3x1 for any give tile shape * :class:`OUTPUT` ``.INFO.TYPE.NAME`` (str) -- numpy dtype of any given tile * :class:`OUTPUT` ``.INFO.SIZE.NAME`` (numpy.ndarray) -- 3x1 for full volume shape """ # Keyword names output = t_query.OUTPUT.INFO runtime = t_query.RUNTIME.IMAGE k_h5 = runtime.SOURCE.HDF5.NAME # Get the max block size in bytes for a single tile max_bytes = t_query.RUNTIME.CACHE.MAX_BLOCK.VALUE max_bytes = int(max_bytes/64) # Check if path is valid keywords = HDF5.valid_path(t_query) if not keywords: return {} # Validate highest in z file name and dataset filename = keywords[k_h5][-1][0] dataset = keywords[k_h5][-1][1] offset = keywords[k_h5][-1][2] # Load properties from H5 dataset with h5py.File(filename,'r') as fd: # Get the volume vol = fd[dataset] # Get a shape for all the files shape = np.uint32(vol.shape) shape[0] += offset #### # Get a blockshape as a flat section #### # Get the bytes for a full slice voxel_bytes = np.uint32(vol.dtype.itemsize) slice_bytes = voxel_bytes * np.prod(shape[1:]) # Get the nearest tile size under cache limit square_overage = np.ceil(slice_bytes / max_bytes) side_scalar = np.ceil(np.sqrt(square_overage)) # Set the actual blocksize to be under the cache limit plane_shape = np.ceil(shape[1:] / side_scalar) max_block = np.r_[[64], plane_shape] #### # Get max blocksizes for different resolutions #### lo_res = 1 # Get all block sizes by halving the max block size all_blocks = [shape/(2**res) for res in range(lo_res)] block_array = np.clip(np.ceil(all_blocks), 1, max_block) # return named keywords keywords.update({ runtime.BLOCK.NAME: np.uint32(block_array), output.SIZE.NAME: np.uint32(shape), output.TYPE.NAME: str(HDF5.dtype(vol)), }) # Combine results with parent method common = Datasource.preload_source(t_query) return dict(common, **keywords)