def get_cutout(self, token, channel, x_start, x_stop, y_start, y_stop, z_start, z_stop, resolution=1, block_size=(256, 256, 16)): """ Get volumetric cutout data from the neurodata server. Arguments: token (str): Token to identify data to download channel (str): Channel resolution (int): Resolution level Q_start (int):` The lower bound of dimension 'Q' Q_stop (int): The upper bound of dimension 'Q' block_size (int[3]): Block size of this dataset Returns: numpy.ndarray: Downloaded data. """ size = (x_stop-x_start)*(y_stop-y_start)*(z_stop-z_start) # For now, max out at 512MB if size < 1E9 / 2: return self._get_cutout_no_chunking(token, channel, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop) else: # Get an array-of-tuples of blocks to request. from ndio.utils.parallel import block_compute, snap_to_cube blocks = block_compute(x_start, x_stop, y_start, y_stop, z_start, z_stop) vol = numpy.zeros(((y_stop - y_start) + 1, (x_stop - x_start) + 1, (z_stop - z_start) + 1)) for b in blocks: data = self._get_cutout_no_chunking( token, channel, resolution, b[0][0], b[0][1], b[1][0], b[1][1], b[2][0], b[2][1]) data = numpy.rollaxis(data, 0, 3) vol[b[1][0]:b[1][1], b[0][0]:b[0][1], b[2][0]:b[2][1]] = data return vol
def _post_cutout_with_chunking(self, token, channel, x_start, y_start, z_start, data, resolution, ul_func): # must chunk first from ndio.utils.parallel import block_compute blocks = block_compute(x_start, x_start + data.shape[2], y_start, y_start + data.shape[1], z_start, z_start + data.shape[0]) for b in blocks: subvol = data[b[2][0]-z_start: b[2][1]-z_start, b[1][0]-y_start: b[1][1]-y_start, b[0][0]-x_start: b[0][1]-x_start] # upload the chunk: ul_func(token, channel, x_start, y_start, z_start, subvol, resolution) return True
def _post_cutout_with_chunking(self, token, channel, x_start, y_start, z_start, data, resolution, ul_func): # must chunk first from ndio.utils.parallel import block_compute blocks = block_compute(x_start, x_start + data.shape[2], y_start, y_start + data.shape[1], z_start, z_start + data.shape[0]) for b in blocks: # data coordinate relative to the size of the arra subvol = data[b[2][0]-z_start: b[2][1]-z_start, b[1][0]-y_start: b[1][1]-y_start, b[0][0]-x_start: b[0][1]-x_start] # upload the chunk: # upload coordinate relative to x_start, y_start, z_start ul_func(token, channel, b[0][0], b[1][0], b[2][0], subvol, resolution) return True
def get_cutout(self, token, channel, x_start, x_stop, y_start, y_stop, z_start, z_stop, resolution=1, block_size=DEFAULT_BLOCK_SIZE, neariso=False): """ Get volumetric cutout data from the neurodata server. Arguments: token (str): Token to identify data to download channel (str): Channel resolution (int): Resolution level Q_start (int): The lower bound of dimension 'Q' Q_stop (int): The upper bound of dimension 'Q' block_size (int[3]): Block size of this dataset. If not provided, ndio uses the metadata of this tokenchannel to set. If you find that your downloads are timing out or otherwise failing, it may be wise to start off by making this smaller. neariso (bool : False): Passes the 'neariso' param to the cutout. If you don't know what this means, ignore it! Returns: numpy.ndarray: Downloaded data. """ if block_size is None: # look up block size from metadata block_size = self.get_block_size(token, resolution) origin = self.get_image_offset(token, resolution) # If z_stop - z_start is < 16, backend still pulls minimum 16 slices if (z_stop - z_start) < 16: z_slices = 16 else: z_slices = z_stop - z_start # Calculate size of the data to be downloaded. size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4 # Switch which download function to use based on which libraries are # available in this version of python. if six.PY2: dl_func = self._get_cutout_blosc_no_chunking elif six.PY3: dl_func = self._get_cutout_no_chunking else: raise ValueError("Invalid Python version.") if size < self._chunk_threshold: vol = dl_func(token, channel, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop, neariso=neariso) vol = numpy.rollaxis(vol, 1) vol = numpy.rollaxis(vol, 2) return vol else: from ndio.utils.parallel import block_compute blocks = block_compute(x_start, x_stop, y_start, y_stop, z_start, z_stop, origin, block_size) vol = numpy.zeros(((z_stop - z_start), (y_stop - y_start), (x_stop - x_start))) for b in blocks: data = dl_func(token, channel, resolution, b[0][0], b[0][1], b[1][0], b[1][1], b[2][0], b[2][1], neariso=neariso) if b == blocks[0]: # first block vol = numpy.zeros(((z_stop - z_start), (y_stop - y_start), (x_stop - x_start)), dtype=data.dtype) vol[b[2][0]-z_start: b[2][1]-z_start, b[1][0]-y_start: b[1][1]-y_start, b[0][0]-x_start: b[0][1]-x_start] = data vol = numpy.rollaxis(vol, 1) vol = numpy.rollaxis(vol, 2) return vol
def get_cutout( self, token, channel, x_start, x_stop, y_start, y_stop, z_start, z_stop, resolution=1, block_size=(256, 256, 16), crop=False, ): """ Get volumetric cutout data from the OCP server. Arguments: token (str): Token to identify data to download channel (str): Channel resolution (int): Resolution level Q_start (int):` The lower bound of dimension 'Q' Q_stop (int): The upper bound of dimension 'Q' block_size (int[3]): Block size of this dataset crop (bool): whether or not to crop the volume before returning it Returns: numpy.ndarray: Downloaded data. Raises: NotImplementedError: If you try to crop... Sorry :( """ if crop is True: raise NotImplementedError("Can't handle crops yet, sorry! :(") size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start) # For now, max out at 1GB # if size < 1E9: if True: return self._get_cutout_no_chunking( token, channel, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop ) else: # Get an array-of-tuples of blocks to request. from ndio.utils.parallel import block_compute, snap_to_cube small_chunks = block_compute(x_start, x_stop, y_start, y_stop, z_start, z_stop) # Each time we download a chunk, we'll store it in # this, in the format (block_origin, data) downloaded_chunks = [] for c in small_chunks: downloaded_chunks.append( ( c, self._get_cutout_no_chunking( token, channel, resolution, c[0][0], c[0][1], c[1][0], c[1][1], c[2][0], c[2][1] ), ) ) x_bounds = snap_to_cube(x_start, x_stop, chunk_depth=256, q_index=0) y_bounds = snap_to_cube(y_start, y_stop, chunk_depth=256, q_index=0) z_bounds = snap_to_cube(z_start, z_stop, chunk_depth=16, q_index=1) volume = numpy.zeros((x_bounds[1] - x_bounds[0], y_bounds[1] - y_bounds[0], z_bounds[1] - z_bounds[0])) # TODO: Optimize. for chunk in downloaded_chunks: x_range, y_range, z_range = chunk[0] xi = 0 for x in range(x_range[0], x_range[1]): yi = 0 for y in range(y_range[0], y_range[1]): zi = 0 for z in range(z_range[0], z_range[1]): volume[x - x_range[0]][y - y_range[0]][z - z_range[0]] = chunk[1][zi][xi][yi] zi += 1 yi += 1 xi += 1 return volume
def get_cutout(self, token, channel, x_start, x_stop, y_start, y_stop, z_start, z_stop, resolution=1, block_size=DEFAULT_BLOCK_SIZE, neariso=False): """ Get volumetric cutout data from the neurodata server. Arguments: token (str): Token to identify data to download channel (str): Channel resolution (int): Resolution level Q_start (int): The lower bound of dimension 'Q' Q_stop (int): The upper bound of dimension 'Q' block_size (int[3]): Block size of this dataset. If not provided, ndio uses the metadata of this tokenchannel to set. If you find that your downloads are timing out or otherwise failing, it may be wise to start off by making this smaller. neariso (bool : False): Passes the 'neariso' param to the cutout. If you don't know what this means, ignore it! Returns: numpy.ndarray: Downloaded data. """ if block_size is None: # look up block size from metadata block_size = self.get_block_size(token, resolution) origin = self.get_image_offset(token, resolution) # If z_stop - z_start is < 16, backend still pulls minimum 16 slices if (z_stop - z_start) < 16: z_slices = 16 else: z_slices = z_stop - z_start # Calculate size of the data to be downloaded. # TODO Multiply size by number of bytes - This should not be >4 size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4 # Switch which download function to use based on which libraries are # available in this version of python. if six.PY2: dl_func = self._get_cutout_blosc_no_chunking elif six.PY3: dl_func = self._get_cutout_no_chunking else: raise ValueError("Invalid Python version.") if size < self._chunk_threshold: vol = dl_func(token, channel, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop, neariso=neariso) vol = numpy.rollaxis(vol, 1) vol = numpy.rollaxis(vol, 2) return vol else: from ndio.utils.parallel import block_compute blocks = block_compute(x_start, x_stop, y_start, y_stop, z_start, z_stop, origin, block_size) vol = numpy.zeros(((z_stop - z_start), (y_stop - y_start), (x_stop - x_start))) for b in blocks: data = dl_func(token, channel, resolution, b[0][0], b[0][1], b[1][0], b[1][1], b[2][0], b[2][1], neariso=neariso) if b == blocks[0]: # first block TODO -update if parallelized vol = numpy.zeros(((z_stop - z_start), (y_stop - y_start), (x_stop - x_start)), dtype=data.dtype) vol[b[2][0]-z_start: b[2][1]-z_start, b[1][0]-y_start: b[1][1]-y_start, b[0][0]-x_start: b[0][1]-x_start] = data vol = numpy.rollaxis(vol, 1) vol = numpy.rollaxis(vol, 2) return vol