def get_cutout(self, token, channel,
x_start, x_stop,
y_start, y_stop,
z_start, z_stop,
resolution=1,
block_size=(256, 256, 16)):
"""
Get volumetric cutout data from the neurodata server.
Arguments:
token (str): Token to identify data to download
channel (str): Channel
resolution (int): Resolution level
Q_start (int):` The lower bound of dimension 'Q'
Q_stop (int): The upper bound of dimension 'Q'
block_size (int[3]): Block size of this dataset
Returns:
numpy.ndarray: Downloaded data.
"""
size = (x_stop-x_start)*(y_stop-y_start)*(z_stop-z_start)
# For now, max out at 512MB
if size < 1E9 / 2:
return self._get_cutout_no_chunking(token, channel, resolution,
x_start, x_stop,
y_start, y_stop,
z_start, z_stop)
else:
# Get an array-of-tuples of blocks to request.
from ndio.utils.parallel import block_compute, snap_to_cube
blocks = block_compute(x_start, x_stop,
y_start, y_stop,
z_start, z_stop)
vol = numpy.zeros(((y_stop - y_start) + 1,
(x_stop - x_start) + 1,
(z_stop - z_start) + 1))
for b in blocks:
data = self._get_cutout_no_chunking(
token, channel, resolution,
b[0][0], b[0][1],
b[1][0], b[1][1],
b[2][0], b[2][1])
data = numpy.rollaxis(data, 0, 3)
vol[b[1][0]:b[1][1], b[0][0]:b[0][1], b[2][0]:b[2][1]] = data
return vol
def _post_cutout_with_chunking(self, token, channel, x_start,
y_start, z_start, data,
resolution, ul_func):
# must chunk first
from ndio.utils.parallel import block_compute
blocks = block_compute(x_start, x_start + data.shape[2],
y_start, y_start + data.shape[1],
z_start, z_start + data.shape[0])
for b in blocks:
subvol = data[b[2][0]-z_start: b[2][1]-z_start,
b[1][0]-y_start: b[1][1]-y_start,
b[0][0]-x_start: b[0][1]-x_start]
# upload the chunk:
ul_func(token, channel, x_start,
y_start, z_start, subvol,
resolution)
return True
def _post_cutout_with_chunking(self, token, channel, x_start,
y_start, z_start, data,
resolution, ul_func):
# must chunk first
from ndio.utils.parallel import block_compute
blocks = block_compute(x_start, x_start + data.shape[2],
y_start, y_start + data.shape[1],
z_start, z_start + data.shape[0])
for b in blocks:
# data coordinate relative to the size of the arra
subvol = data[b[2][0]-z_start: b[2][1]-z_start,
b[1][0]-y_start: b[1][1]-y_start,
b[0][0]-x_start: b[0][1]-x_start]
# upload the chunk:
# upload coordinate relative to x_start, y_start, z_start
ul_func(token, channel, b[0][0],
b[1][0], b[2][0], subvol,
resolution)
return True
def get_cutout(self, token, channel,
x_start, x_stop,
y_start, y_stop,
z_start, z_stop,
resolution=1,
block_size=DEFAULT_BLOCK_SIZE,
neariso=False):
"""
Get volumetric cutout data from the neurodata server.
Arguments:
token (str): Token to identify data to download
channel (str): Channel
resolution (int): Resolution level
Q_start (int): The lower bound of dimension 'Q'
Q_stop (int): The upper bound of dimension 'Q'
block_size (int[3]): Block size of this dataset. If not provided,
ndio uses the metadata of this tokenchannel to set. If you find
that your downloads are timing out or otherwise failing, it may
be wise to start off by making this smaller.
neariso (bool : False): Passes the 'neariso' param to the cutout.
If you don't know what this means, ignore it!
Returns:
numpy.ndarray: Downloaded data.
"""
if block_size is None:
# look up block size from metadata
block_size = self.get_block_size(token, resolution)
origin = self.get_image_offset(token, resolution)
# If z_stop - z_start is < 16, backend still pulls minimum 16 slices
if (z_stop - z_start) < 16:
z_slices = 16
else:
z_slices = z_stop - z_start
# Calculate size of the data to be downloaded.
size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4
# Switch which download function to use based on which libraries are
# available in this version of python.
if six.PY2:
dl_func = self._get_cutout_blosc_no_chunking
elif six.PY3:
dl_func = self._get_cutout_no_chunking
else:
raise ValueError("Invalid Python version.")
if size < self._chunk_threshold:
vol = dl_func(token, channel, resolution,
x_start, x_stop,
y_start, y_stop,
z_start, z_stop, neariso=neariso)
vol = numpy.rollaxis(vol, 1)
vol = numpy.rollaxis(vol, 2)
return vol
else:
from ndio.utils.parallel import block_compute
blocks = block_compute(x_start, x_stop,
y_start, y_stop,
z_start, z_stop,
origin, block_size)
vol = numpy.zeros(((z_stop - z_start),
(y_stop - y_start),
(x_stop - x_start)))
for b in blocks:
data = dl_func(token, channel, resolution,
b[0][0], b[0][1],
b[1][0], b[1][1],
b[2][0], b[2][1], neariso=neariso)
if b == blocks[0]: # first block
vol = numpy.zeros(((z_stop - z_start),
(y_stop - y_start),
(x_stop - x_start)), dtype=data.dtype)
vol[b[2][0]-z_start: b[2][1]-z_start,
b[1][0]-y_start: b[1][1]-y_start,
b[0][0]-x_start: b[0][1]-x_start] = data
vol = numpy.rollaxis(vol, 1)
vol = numpy.rollaxis(vol, 2)
return vol
def get_cutout(
self,
token,
channel,
x_start,
x_stop,
y_start,
y_stop,
z_start,
z_stop,
resolution=1,
block_size=(256, 256, 16),
crop=False,
):
"""
Get volumetric cutout data from the OCP server.
Arguments:
token (str): Token to identify data to download
channel (str): Channel
resolution (int): Resolution level
Q_start (int):` The lower bound of dimension 'Q'
Q_stop (int): The upper bound of dimension 'Q'
block_size (int[3]): Block size of this dataset
crop (bool): whether or not to crop the volume before returning it
Returns:
numpy.ndarray: Downloaded data.
Raises:
NotImplementedError: If you try to crop... Sorry :(
"""
if crop is True:
raise NotImplementedError("Can't handle crops yet, sorry! :(")
size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start)
# For now, max out at 1GB
# if size < 1E9:
if True:
return self._get_cutout_no_chunking(
token, channel, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop
)
else:
# Get an array-of-tuples of blocks to request.
from ndio.utils.parallel import block_compute, snap_to_cube
small_chunks = block_compute(x_start, x_stop, y_start, y_stop, z_start, z_stop)
# Each time we download a chunk, we'll store it in
# this, in the format (block_origin, data)
downloaded_chunks = []
for c in small_chunks:
downloaded_chunks.append(
(
c,
self._get_cutout_no_chunking(
token, channel, resolution, c[0][0], c[0][1], c[1][0], c[1][1], c[2][0], c[2][1]
),
)
)
x_bounds = snap_to_cube(x_start, x_stop, chunk_depth=256, q_index=0)
y_bounds = snap_to_cube(y_start, y_stop, chunk_depth=256, q_index=0)
z_bounds = snap_to_cube(z_start, z_stop, chunk_depth=16, q_index=1)
volume = numpy.zeros((x_bounds[1] - x_bounds[0], y_bounds[1] - y_bounds[0], z_bounds[1] - z_bounds[0]))
# TODO: Optimize.
for chunk in downloaded_chunks:
x_range, y_range, z_range = chunk[0]
xi = 0
for x in range(x_range[0], x_range[1]):
yi = 0
for y in range(y_range[0], y_range[1]):
zi = 0
for z in range(z_range[0], z_range[1]):
volume[x - x_range[0]][y - y_range[0]][z - z_range[0]] = chunk[1][zi][xi][yi]
zi += 1
yi += 1
xi += 1
return volume
def get_cutout(self, token, channel,
x_start, x_stop,
y_start, y_stop,
z_start, z_stop,
resolution=1,
block_size=DEFAULT_BLOCK_SIZE,
neariso=False):
"""
Get volumetric cutout data from the neurodata server.
Arguments:
token (str): Token to identify data to download
channel (str): Channel
resolution (int): Resolution level
Q_start (int): The lower bound of dimension 'Q'
Q_stop (int): The upper bound of dimension 'Q'
block_size (int[3]): Block size of this dataset. If not provided,
ndio uses the metadata of this tokenchannel to set. If you find
that your downloads are timing out or otherwise failing, it may
be wise to start off by making this smaller.
neariso (bool : False): Passes the 'neariso' param to the cutout.
If you don't know what this means, ignore it!
Returns:
numpy.ndarray: Downloaded data.
"""
if block_size is None:
# look up block size from metadata
block_size = self.get_block_size(token, resolution)
origin = self.get_image_offset(token, resolution)
# If z_stop - z_start is < 16, backend still pulls minimum 16 slices
if (z_stop - z_start) < 16:
z_slices = 16
else:
z_slices = z_stop - z_start
# Calculate size of the data to be downloaded.
# TODO Multiply size by number of bytes - This should not be >4
size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4
# Switch which download function to use based on which libraries are
# available in this version of python.
if six.PY2:
dl_func = self._get_cutout_blosc_no_chunking
elif six.PY3:
dl_func = self._get_cutout_no_chunking
else:
raise ValueError("Invalid Python version.")
if size < self._chunk_threshold:
vol = dl_func(token, channel, resolution,
x_start, x_stop,
y_start, y_stop,
z_start, z_stop, neariso=neariso)
vol = numpy.rollaxis(vol, 1)
vol = numpy.rollaxis(vol, 2)
return vol
else:
from ndio.utils.parallel import block_compute
blocks = block_compute(x_start, x_stop,
y_start, y_stop,
z_start, z_stop,
origin, block_size)
vol = numpy.zeros(((z_stop - z_start),
(y_stop - y_start),
(x_stop - x_start)))
for b in blocks:
data = dl_func(token, channel, resolution,
b[0][0], b[0][1],
b[1][0], b[1][1],
b[2][0], b[2][1], neariso=neariso)
if b == blocks[0]: # first block TODO -update if parallelized
vol = numpy.zeros(((z_stop - z_start),
(y_stop - y_start),
(x_stop - x_start)), dtype=data.dtype)
vol[b[2][0]-z_start: b[2][1]-z_start,
b[1][0]-y_start: b[1][1]-y_start,
b[0][0]-x_start: b[0][1]-x_start] = data
vol = numpy.rollaxis(vol, 1)
vol = numpy.rollaxis(vol, 2)
return vol
