示例#1
0
def _test():
    import numpy as np
    # analysis:ignore
    import ClearMap.IO.Slice as slc
    from importlib import reload
    reload(slc)

    s1 = (slice(1, 4), [1, 2, 3, 4, 5], None, Ellipsis)
    ss = slc.simplify_slicing(s1, ndim=5)
    print(ss)

    shape = (7, 6, 2, 3, 5)

    d1 = slc.sliced_ndim(s1, 5, allow_index_arrays=True)
    shape1 = slc.sliced_shape(s1, shape, allow_index_arrays=True)
    print(d1, shape1)

    x = np.random.rand(*shape)
    x1 = x[s1]
    x1.shape == shape1

    s2 = (slice(None, None, 2), slice(3, 4), slice(None), 1, [0, 2, 1])
    s12 = slc.sliced_slicing(s2, s1, shape, allow_index_arrays=True)

    np.all(x[s12] == x[s1][s2])

    slc.is_view(s1)

    s1s = slc.simplify_slicing(s1)
    slc.is_view(s1s)

    s2 = (slice(None, None, 2), slice(3, 4), slice(None), 1, slice(0, 2))

    y = x[s1s][s2]
    y.base is x

    s12 = slc.sliced_slicing(s2, s1s, shape)
    slc.is_view(s12)
    x[s12].base is x

    x = slc.src.VirtualSource(shape=(5, 10, 15),
                              dtype=float,
                              order='F',
                              location='/home/test.src')

    s = slc.Slice(source=x, slicing=(Ellipsis, 1))
    print(s)
    print(s.source)
    print(s.unpacked_slicing)

    reload(slc)
    shape = (50, 100, 200)
    s1 = (slice(None), slice(None), slice(0, 38))
    s2 = (slice(None), slice(None), slice(None, -10))

    slc.sliced_slicing(s2, s1, shape)
示例#2
0
    def __getitem__(self, slicing, processes=None, order=None):
        e = self.expression

        shape = self.shape
        ndim = self.ndim
        ndim_list = e.ntags()

        slicing = slc.unpack_slicing(slicing, ndim)

        slicing_file = slicing[:-ndim_list]
        slicing_list = slicing[-ndim_list:]

        shape_file = shape[:-ndim_list]
        shape_list = shape[-ndim_list:]

        sliced_shape_file = slc.sliced_shape(slicing=slicing_file,
                                             shape=shape_file)
        #sliced_shape_list = slc.sliced_shape(slicing=slicing_list, shape=shape_list);

        #start indices
        indices_start = self.expression.indices(self.file_list[0])
        #print(indices_start)
        #TODO: steps in file list

        #genereate file list to read
        #Note: indices increase according to the axes order but their own order is in tag order
        indices = []
        slicing_list_indices = []
        shape_list_keep_dims = ()
        slicing_keep_dims_to_final = (Ellipsis, )
        for sl, s, i in zip(slicing_list, shape_list, indices_start):
            if isinstance(sl, slice):
                slice_indices = sl.indices(s)
                slice_indices = (slice_indices[0] + i, slice_indices[1] + i,
                                 slice_indices[2])
                indices.append(range(*slice_indices))
                n = len(indices[-1])
                slicing_list_indices.append(range(n))
                shape_list_keep_dims += (n, )
                slicing_keep_dims_to_final += (slice(None), )
            elif isinstance(sl, (list, np.ndarray)):
                indices.append(np.array(sl) + i)
                n = len(indices[-1])
                slicing_list_indices.append(sl)
                shape_list_keep_dims += (n, )
                slicing_keep_dims_to_final += (slice(None), )
            elif isinstance(sl, numbers.Integral):
                indices.append([sl + i])
                slicing_list_indices.append([0])
                shape_list_keep_dims += (1, )
                slicing_keep_dims_to_final += (0, )
            else:
                raise IndexError('Invalid slice specification %r!' % sl)
        indices.reverse()
        indices = itertools.product(*indices)
        indices = [i[::-1] for i in indices]
        slicing_list_indices.reverse()
        slicing_list_indices = itertools.product(*slicing_list_indices)
        slicing_list_indices = [i[::-1] for i in slicing_list_indices]
        #print(indices, slicing_list_indices, slicing_keep_dims_to_final)

        axes_to_tags = self.axes_to_tag_order()
        if len(axes_to_tags) > 1 and axes_to_tags != list(
                range(len(axes_to_tags))):
            indices = [tuple(i[j] for j in axes_to_tags) for i in indices]

        fl = [e.string_from_index(i) for i in indices]
        #print(fl);

        dtype = self.dtype

        data = np.zeros(sliced_shape_file + shape_list_keep_dims,
                        dtype=dtype,
                        order=order)

        #@ptb.parallel_traceback
        def func(filename, index, data=data, slicing=slicing_file):
            index = (Ellipsis, ) + index
            data[index] = io.read(filename,
                                  slicing=slicing,
                                  processes='serial')

        if processes is None:
            processes = mp.cpu_count()

        if processes == 'serial':
            for f, i in zip(fl, slicing_list_indices):
                func(f, i)
        else:
            with concurrent.futures.ThreadPoolExecutor(processes) as executor:
                executor.map(func, fl, slicing_list_indices)

        data = data[slicing_keep_dims_to_final]

        return data