def _test(): import numpy as np # analysis:ignore import ClearMap.IO.Slice as slc from importlib import reload reload(slc) s1 = (slice(1, 4), [1, 2, 3, 4, 5], None, Ellipsis) ss = slc.simplify_slicing(s1, ndim=5) print(ss) shape = (7, 6, 2, 3, 5) d1 = slc.sliced_ndim(s1, 5, allow_index_arrays=True) shape1 = slc.sliced_shape(s1, shape, allow_index_arrays=True) print(d1, shape1) x = np.random.rand(*shape) x1 = x[s1] x1.shape == shape1 s2 = (slice(None, None, 2), slice(3, 4), slice(None), 1, [0, 2, 1]) s12 = slc.sliced_slicing(s2, s1, shape, allow_index_arrays=True) np.all(x[s12] == x[s1][s2]) slc.is_view(s1) s1s = slc.simplify_slicing(s1) slc.is_view(s1s) s2 = (slice(None, None, 2), slice(3, 4), slice(None), 1, slice(0, 2)) y = x[s1s][s2] y.base is x s12 = slc.sliced_slicing(s2, s1s, shape) slc.is_view(s12) x[s12].base is x x = slc.src.VirtualSource(shape=(5, 10, 15), dtype=float, order='F', location='/home/test.src') s = slc.Slice(source=x, slicing=(Ellipsis, 1)) print(s) print(s.source) print(s.unpacked_slicing) reload(slc) shape = (50, 100, 200) s1 = (slice(None), slice(None), slice(0, 38)) s2 = (slice(None), slice(None), slice(None, -10)) slc.sliced_slicing(s2, s1, shape)
def __getitem__(self, slicing, processes=None, order=None): e = self.expression shape = self.shape ndim = self.ndim ndim_list = e.ntags() slicing = slc.unpack_slicing(slicing, ndim) slicing_file = slicing[:-ndim_list] slicing_list = slicing[-ndim_list:] shape_file = shape[:-ndim_list] shape_list = shape[-ndim_list:] sliced_shape_file = slc.sliced_shape(slicing=slicing_file, shape=shape_file) #sliced_shape_list = slc.sliced_shape(slicing=slicing_list, shape=shape_list); #start indices indices_start = self.expression.indices(self.file_list[0]) #print(indices_start) #TODO: steps in file list #genereate file list to read #Note: indices increase according to the axes order but their own order is in tag order indices = [] slicing_list_indices = [] shape_list_keep_dims = () slicing_keep_dims_to_final = (Ellipsis, ) for sl, s, i in zip(slicing_list, shape_list, indices_start): if isinstance(sl, slice): slice_indices = sl.indices(s) slice_indices = (slice_indices[0] + i, slice_indices[1] + i, slice_indices[2]) indices.append(range(*slice_indices)) n = len(indices[-1]) slicing_list_indices.append(range(n)) shape_list_keep_dims += (n, ) slicing_keep_dims_to_final += (slice(None), ) elif isinstance(sl, (list, np.ndarray)): indices.append(np.array(sl) + i) n = len(indices[-1]) slicing_list_indices.append(sl) shape_list_keep_dims += (n, ) slicing_keep_dims_to_final += (slice(None), ) elif isinstance(sl, numbers.Integral): indices.append([sl + i]) slicing_list_indices.append([0]) shape_list_keep_dims += (1, ) slicing_keep_dims_to_final += (0, ) else: raise IndexError('Invalid slice specification %r!' % sl) indices.reverse() indices = itertools.product(*indices) indices = [i[::-1] for i in indices] slicing_list_indices.reverse() slicing_list_indices = itertools.product(*slicing_list_indices) slicing_list_indices = [i[::-1] for i in slicing_list_indices] #print(indices, slicing_list_indices, slicing_keep_dims_to_final) axes_to_tags = self.axes_to_tag_order() if len(axes_to_tags) > 1 and axes_to_tags != list( range(len(axes_to_tags))): indices = [tuple(i[j] for j in axes_to_tags) for i in indices] fl = [e.string_from_index(i) for i in indices] #print(fl); dtype = self.dtype data = np.zeros(sliced_shape_file + shape_list_keep_dims, dtype=dtype, order=order) #@ptb.parallel_traceback def func(filename, index, data=data, slicing=slicing_file): index = (Ellipsis, ) + index data[index] = io.read(filename, slicing=slicing, processes='serial') if processes is None: processes = mp.cpu_count() if processes == 'serial': for f, i in zip(fl, slicing_list_indices): func(f, i) else: with concurrent.futures.ThreadPoolExecutor(processes) as executor: executor.map(func, fl, slicing_list_indices) data = data[slicing_keep_dims_to_final] return data