Esempio n. 1
0
    def get_cutout(self, token, channel,
                   x_start, x_stop,
                   y_start, y_stop,
                   z_start, z_stop,
                   resolution=1,
                   block_size=(256, 256, 16)):
        """
        Get volumetric cutout data from the neurodata server.

        Arguments:
            token (str): Token to identify data to download
            channel (str): Channel
            resolution (int): Resolution level
            Q_start (int):` The lower bound of dimension 'Q'
            Q_stop (int): The upper bound of dimension 'Q'
            block_size (int[3]): Block size of this dataset

        Returns:
            numpy.ndarray: Downloaded data.
        """

        size = (x_stop-x_start)*(y_stop-y_start)*(z_stop-z_start)

        # For now, max out at 512MB
        if size < 1E9 / 2:
            return self._get_cutout_no_chunking(token, channel, resolution,
                                                x_start, x_stop,
                                                y_start, y_stop,
                                                z_start, z_stop)

        else:
            # Get an array-of-tuples of blocks to request.
            from ndio.utils.parallel import block_compute, snap_to_cube
            blocks = block_compute(x_start, x_stop,
                                   y_start, y_stop,
                                   z_start, z_stop)

            vol = numpy.zeros(((y_stop - y_start) + 1,
                              (x_stop - x_start) + 1,
                              (z_stop - z_start) + 1))
            for b in blocks:
                data = self._get_cutout_no_chunking(
                                     token, channel, resolution,
                                     b[0][0], b[0][1],
                                     b[1][0], b[1][1],
                                     b[2][0], b[2][1])
                data = numpy.rollaxis(data, 0, 3)
                vol[b[1][0]:b[1][1], b[0][0]:b[0][1], b[2][0]:b[2][1]] = data

            return vol
Esempio n. 2
0
    def _post_cutout_with_chunking(self, token, channel, x_start,
                                   y_start, z_start, data,
                                   resolution, ul_func):
        # must chunk first
        from ndio.utils.parallel import block_compute
        blocks = block_compute(x_start, x_start + data.shape[2],
                               y_start, y_start + data.shape[1],
                               z_start, z_start + data.shape[0])

        for b in blocks:
            subvol = data[b[2][0]-z_start: b[2][1]-z_start,
                          b[1][0]-y_start: b[1][1]-y_start,
                          b[0][0]-x_start: b[0][1]-x_start]
            # upload the chunk:
            ul_func(token, channel, x_start,
                    y_start, z_start, subvol,
                    resolution)
        return True
Esempio n. 3
0
 def _post_cutout_with_chunking(self, token, channel, x_start,
                                y_start, z_start, data,
                                resolution, ul_func):
     # must chunk first
     from ndio.utils.parallel import block_compute
     blocks = block_compute(x_start, x_start + data.shape[2],
                            y_start, y_start + data.shape[1],
                            z_start, z_start + data.shape[0])
     for b in blocks:
         # data coordinate relative to the size of the arra
         subvol = data[b[2][0]-z_start: b[2][1]-z_start,
                       b[1][0]-y_start: b[1][1]-y_start,
                       b[0][0]-x_start: b[0][1]-x_start]
         # upload the chunk:
         # upload coordinate relative to x_start, y_start, z_start
         ul_func(token, channel, b[0][0],
                 b[1][0], b[2][0], subvol,
                 resolution)
     return True
Esempio n. 4
0
    def get_cutout(self, token, channel,
                   x_start, x_stop,
                   y_start, y_stop,
                   z_start, z_stop,
                   resolution=1,
                   block_size=DEFAULT_BLOCK_SIZE,
                   neariso=False):
        """
        Get volumetric cutout data from the neurodata server.

        Arguments:
            token (str): Token to identify data to download
            channel (str): Channel
            resolution (int): Resolution level
            Q_start (int): The lower bound of dimension 'Q'
            Q_stop (int): The upper bound of dimension 'Q'
            block_size (int[3]): Block size of this dataset. If not provided,
                ndio uses the metadata of this tokenchannel to set. If you find
                that your downloads are timing out or otherwise failing, it may
                be wise to start off by making this smaller.
            neariso (bool : False): Passes the 'neariso' param to the cutout.
                If you don't know what this means, ignore it!

        Returns:
            numpy.ndarray: Downloaded data.
        """
        if block_size is None:
            # look up block size from metadata
            block_size = self.get_block_size(token, resolution)

        origin = self.get_image_offset(token, resolution)

        # If z_stop - z_start is < 16, backend still pulls minimum 16 slices
        if (z_stop - z_start) < 16:
            z_slices = 16
        else:
            z_slices = z_stop - z_start

        # Calculate size of the data to be downloaded.
        size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4

        # Switch which download function to use based on which libraries are
        # available in this version of python.
        if six.PY2:
            dl_func = self._get_cutout_blosc_no_chunking
        elif six.PY3:
            dl_func = self._get_cutout_no_chunking
        else:
            raise ValueError("Invalid Python version.")

        if size < self._chunk_threshold:
            vol = dl_func(token, channel, resolution,
                          x_start, x_stop,
                          y_start, y_stop,
                          z_start, z_stop, neariso=neariso)
            vol = numpy.rollaxis(vol, 1)
            vol = numpy.rollaxis(vol, 2)
            return vol
        else:
            from ndio.utils.parallel import block_compute
            blocks = block_compute(x_start, x_stop,
                                   y_start, y_stop,
                                   z_start, z_stop,
                                   origin, block_size)

            vol = numpy.zeros(((z_stop - z_start),
                              (y_stop - y_start),
                              (x_stop - x_start)))
            for b in blocks:

                data = dl_func(token, channel, resolution,
                               b[0][0], b[0][1],
                               b[1][0], b[1][1],
                               b[2][0], b[2][1], neariso=neariso)

                if b == blocks[0]:  # first block
                    vol = numpy.zeros(((z_stop - z_start),
                                       (y_stop - y_start),
                                       (x_stop - x_start)), dtype=data.dtype)

                vol[b[2][0]-z_start: b[2][1]-z_start,
                    b[1][0]-y_start: b[1][1]-y_start,
                    b[0][0]-x_start: b[0][1]-x_start] = data

            vol = numpy.rollaxis(vol, 1)
            vol = numpy.rollaxis(vol, 2)
            return vol
Esempio n. 5
0
    def get_cutout(
        self,
        token,
        channel,
        x_start,
        x_stop,
        y_start,
        y_stop,
        z_start,
        z_stop,
        resolution=1,
        block_size=(256, 256, 16),
        crop=False,
    ):
        """
        Get volumetric cutout data from the OCP server.

        Arguments:
            token (str): Token to identify data to download
            channel (str): Channel
            resolution (int): Resolution level
            Q_start (int):` The lower bound of dimension 'Q'
            Q_stop (int): The upper bound of dimension 'Q'
            block_size (int[3]): Block size of this dataset
            crop (bool): whether or not to crop the volume before returning it

        Returns:
            numpy.ndarray: Downloaded data.

        Raises:
            NotImplementedError: If you try to crop... Sorry :(
        """

        if crop is True:
            raise NotImplementedError("Can't handle crops yet, sorry! :(")

        size = (x_stop - x_start) * (y_stop - y_start) * (z_stop - z_start)

        # For now, max out at 1GB
        # if size < 1E9:
        if True:
            return self._get_cutout_no_chunking(
                token, channel, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop
            )

        else:
            # Get an array-of-tuples of blocks to request.
            from ndio.utils.parallel import block_compute, snap_to_cube

            small_chunks = block_compute(x_start, x_stop, y_start, y_stop, z_start, z_stop)

            # Each time we download a chunk, we'll store it in
            # this, in the format (block_origin, data)
            downloaded_chunks = []
            for c in small_chunks:
                downloaded_chunks.append(
                    (
                        c,
                        self._get_cutout_no_chunking(
                            token, channel, resolution, c[0][0], c[0][1], c[1][0], c[1][1], c[2][0], c[2][1]
                        ),
                    )
                )

            x_bounds = snap_to_cube(x_start, x_stop, chunk_depth=256, q_index=0)
            y_bounds = snap_to_cube(y_start, y_stop, chunk_depth=256, q_index=0)
            z_bounds = snap_to_cube(z_start, z_stop, chunk_depth=16, q_index=1)

            volume = numpy.zeros((x_bounds[1] - x_bounds[0], y_bounds[1] - y_bounds[0], z_bounds[1] - z_bounds[0]))

            # TODO: Optimize.
            for chunk in downloaded_chunks:
                x_range, y_range, z_range = chunk[0]
                xi = 0
                for x in range(x_range[0], x_range[1]):
                    yi = 0
                    for y in range(y_range[0], y_range[1]):
                        zi = 0
                        for z in range(z_range[0], z_range[1]):
                            volume[x - x_range[0]][y - y_range[0]][z - z_range[0]] = chunk[1][zi][xi][yi]
                            zi += 1
                        yi += 1
                    xi += 1
            return volume
Esempio n. 6
0
    def get_cutout(self, token, channel,
                   x_start, x_stop,
                   y_start, y_stop,
                   z_start, z_stop,
                   resolution=1,
                   block_size=DEFAULT_BLOCK_SIZE,
                   neariso=False):
        """
        Get volumetric cutout data from the neurodata server.

        Arguments:
            token (str): Token to identify data to download
            channel (str): Channel
            resolution (int): Resolution level
            Q_start (int): The lower bound of dimension 'Q'
            Q_stop (int): The upper bound of dimension 'Q'
            block_size (int[3]): Block size of this dataset. If not provided,
                ndio uses the metadata of this tokenchannel to set. If you find
                that your downloads are timing out or otherwise failing, it may
                be wise to start off by making this smaller.
            neariso (bool : False): Passes the 'neariso' param to the cutout.
                If you don't know what this means, ignore it!

        Returns:
            numpy.ndarray: Downloaded data.
        """
        if block_size is None:
            # look up block size from metadata
            block_size = self.get_block_size(token, resolution)

        origin = self.get_image_offset(token, resolution)

        # If z_stop - z_start is < 16, backend still pulls minimum 16 slices
        if (z_stop - z_start) < 16:
            z_slices = 16
        else:
            z_slices = z_stop - z_start

        # Calculate size of the data to be downloaded.
        # TODO Multiply size by number of bytes - This should not be >4
        size = (x_stop - x_start) * (y_stop - y_start) * z_slices * 4

        # Switch which download function to use based on which libraries are
        # available in this version of python.
        if six.PY2:
            dl_func = self._get_cutout_blosc_no_chunking
        elif six.PY3:
            dl_func = self._get_cutout_no_chunking
        else:
            raise ValueError("Invalid Python version.")

        if size < self._chunk_threshold:
            vol = dl_func(token, channel, resolution,
                          x_start, x_stop,
                          y_start, y_stop,
                          z_start, z_stop, neariso=neariso)
            vol = numpy.rollaxis(vol, 1)
            vol = numpy.rollaxis(vol, 2)
            return vol
        else:
            from ndio.utils.parallel import block_compute
            blocks = block_compute(x_start, x_stop,
                                   y_start, y_stop,
                                   z_start, z_stop,
                                   origin, block_size)

            vol = numpy.zeros(((z_stop - z_start),
                              (y_stop - y_start),
                              (x_stop - x_start)))
            for b in blocks:

                data = dl_func(token, channel, resolution,
                               b[0][0], b[0][1],
                               b[1][0], b[1][1],
                               b[2][0], b[2][1], neariso=neariso)

                if b == blocks[0]:  # first block  TODO -update if parallelized
                    vol = numpy.zeros(((z_stop - z_start),
                                       (y_stop - y_start),
                                       (x_stop - x_start)), dtype=data.dtype)

                vol[b[2][0]-z_start: b[2][1]-z_start,
                    b[1][0]-y_start: b[1][1]-y_start,
                    b[0][0]-x_start: b[0][1]-x_start] = data

            vol = numpy.rollaxis(vol, 1)
            vol = numpy.rollaxis(vol, 2)
            return vol