示例#1
0
    def _map_iterate(self,
                     function,
                     iterating_kwargs=(),
                     show_progressbar=None,
                     parallel=None,
                     max_workers=None,
                     ragged=None,
                     inplace=True,
                     **kwargs):
        if ragged not in (True, False):
            raise ValueError('"ragged" kwarg has to be bool for lazy signals')
        _logger.debug("Entering '_map_iterate'")

        size = max(1, self.axes_manager.navigation_size)
        from hyperspy.misc.utils import (create_map_objects,
                                         map_result_construction)
        func, iterators = create_map_objects(function, size, iterating_kwargs,
                                             **kwargs)
        iterators = (self._iterate_signal(), ) + iterators
        res_shape = self.axes_manager._navigation_shape_in_array
        # no navigation
        if not len(res_shape) and ragged:
            res_shape = (1, )

        all_delayed = [dd(func)(data) for data in zip(*iterators)]

        if ragged:
            sig_shape = ()
            sig_dtype = np.dtype('O')
        else:
            one_compute = all_delayed[0].compute()
            sig_shape = one_compute.shape
            sig_dtype = one_compute.dtype
        pixels = [
            da.from_delayed(res, shape=sig_shape, dtype=sig_dtype)
            for res in all_delayed
        ]

        for step in reversed(res_shape):
            _len = len(pixels)
            starts = range(0, _len, step)
            ends = range(step, _len + step, step)
            pixels = [
                da.stack(pixels[s:e], axis=0) for s, e in zip(starts, ends)
            ]
        result = pixels[0]
        res = map_result_construction(self,
                                      inplace,
                                      result,
                                      ragged,
                                      sig_shape,
                                      lazy=True)
        return res
示例#2
0
文件: lazy.py 项目: mwalls/hyperspy
    def _map_iterate(self,
                     function,
                     iterating_kwargs=(),
                     show_progressbar=None,
                     parallel=None,
                     ragged=None,
                     inplace=True,
                     **kwargs):
        if ragged not in (True, False):
            raise ValueError('"ragged" kwarg has to be bool for lazy signals')
        _logger.debug("Entering '_map_iterate'")

        size = max(1, self.axes_manager.navigation_size)
        from hyperspy.misc.utils import (create_map_objects,
                                         map_result_construction)
        func, iterators = create_map_objects(function, size, iterating_kwargs,
                                             **kwargs)
        iterators = (self._iterate_signal(), ) + iterators
        res_shape = self.axes_manager._navigation_shape_in_array
        # no navigation
        if not len(res_shape) and ragged:
            res_shape = (1,)

        all_delayed = [dd(func)(data) for data in zip(*iterators)]

        if ragged:
            sig_shape = ()
            sig_dtype = np.dtype('O')
        else:
            one_compute = all_delayed[0].compute()
            sig_shape = one_compute.shape
            sig_dtype = one_compute.dtype
        pixels = [
            da.from_delayed(
                res, shape=sig_shape, dtype=sig_dtype) for res in all_delayed
        ]

        for step in reversed(res_shape):
            _len = len(pixels)
            starts = range(0, _len, step)
            ends = range(step, _len + step, step)
            pixels = [
                da.stack(
                    pixels[s:e], axis=0) for s, e in zip(starts, ends)
            ]
        result = pixels[0]
        res = map_result_construction(
            self, inplace, result, ragged, sig_shape, lazy=True)
        return res
示例#3
0
文件: lazy.py 项目: pc494/hyperspy
    def _map_iterate(self,
                     function,
                     iterating_kwargs=(),
                     show_progressbar=None,
                     parallel=None,
                     max_workers=None,
                     ragged=None,
                     inplace=True,
                     **kwargs):
        if ragged not in (True, False):
            raise ValueError('"ragged" kwarg has to be bool for lazy signals')
        _logger.debug("Entering '_map_iterate'")

        size = max(1, self.axes_manager.navigation_size)
        from hyperspy.misc.utils import (create_map_objects,
                                         map_result_construction)
        func, iterators = create_map_objects(function, size, iterating_kwargs,
                                             **kwargs)
        iterators = (self._iterate_signal(), ) + iterators
        res_shape = self.axes_manager._navigation_shape_in_array
        # no navigation
        if not len(res_shape) and ragged:
            res_shape = (1, )

        all_delayed = [dd(func)(data) for data in zip(*iterators)]

        if ragged:
            if inplace:
                raise ValueError("In place computation is not compatible with "
                                 "ragged array for lazy signal.")
            # Shape of the signal dimension will change for the each nav.
            # index, which means we can't predict the shape and the dtype needs
            # to be python object to support numpy ragged array
            sig_shape = ()
            sig_dtype = np.dtype('O')
        else:
            one_compute = all_delayed[0].compute()
            # No signal dimension for scalar
            if np.isscalar(one_compute):
                sig_shape = ()
                sig_dtype = type(one_compute)
            else:
                sig_shape = one_compute.shape
                sig_dtype = one_compute.dtype
        pixels = [
            da.from_delayed(res, shape=sig_shape, dtype=sig_dtype)
            for res in all_delayed
        ]
        if ragged:
            if show_progressbar is None:
                from hyperspy.defaults_parser import preferences
                show_progressbar = preferences.General.show_progressbar
            # We compute here because this is not sure if this is possible
            # to make a ragged dask array: we need to provide a chunk size...
            res_data = np.empty(res_shape, dtype=sig_dtype)
            _logger.info("Lazy signal is computed to make the ragged array.")
            if show_progressbar:
                cm = ProgressBar
            else:
                cm = dummy_context_manager
            with cm():
                try:
                    for i, pixel in enumerate(pixels):
                        res_data.flat[i] = pixel.compute()
                except MemoryError:
                    raise MemoryError("The use of 'ragged' array requires the "
                                      "computation of the lazy signal.")
        else:
            if len(pixels) > 0:
                for step in reversed(res_shape):
                    _len = len(pixels)
                    starts = range(0, _len, step)
                    ends = range(step, _len + step, step)
                    pixels = [
                        da.stack(pixels[s:e], axis=0)
                        for s, e in zip(starts, ends)
                    ]
            res_data = pixels[0]

        res = map_result_construction(self,
                                      inplace,
                                      res_data,
                                      ragged,
                                      sig_shape,
                                      lazy=not ragged)

        return res