def _test():
import numpy as np
# analysis:ignore
import ClearMap.IO.Slice as slc
from importlib import reload
reload(slc)
s1 = (slice(1, 4), [1, 2, 3, 4, 5], None, Ellipsis)
ss = slc.simplify_slicing(s1, ndim=5)
print(ss)
shape = (7, 6, 2, 3, 5)
d1 = slc.sliced_ndim(s1, 5, allow_index_arrays=True)
shape1 = slc.sliced_shape(s1, shape, allow_index_arrays=True)
print(d1, shape1)
x = np.random.rand(*shape)
x1 = x[s1]
x1.shape == shape1
s2 = (slice(None, None, 2), slice(3, 4), slice(None), 1, [0, 2, 1])
s12 = slc.sliced_slicing(s2, s1, shape, allow_index_arrays=True)
np.all(x[s12] == x[s1][s2])
slc.is_view(s1)
s1s = slc.simplify_slicing(s1)
slc.is_view(s1s)
s2 = (slice(None, None, 2), slice(3, 4), slice(None), 1, slice(0, 2))
y = x[s1s][s2]
y.base is x
s12 = slc.sliced_slicing(s2, s1s, shape)
slc.is_view(s12)
x[s12].base is x
x = slc.src.VirtualSource(shape=(5, 10, 15),
dtype=float,
order='F',
location='/home/test.src')
s = slc.Slice(source=x, slicing=(Ellipsis, 1))
print(s)
print(s.source)
print(s.unpacked_slicing)
reload(slc)
shape = (50, 100, 200)
s1 = (slice(None), slice(None), slice(0, 38))
s2 = (slice(None), slice(None), slice(None, -10))
slc.sliced_slicing(s2, s1, shape)
def __getitem__(self, slicing, processes=None, order=None):
e = self.expression
shape = self.shape
ndim = self.ndim
ndim_list = e.ntags()
slicing = slc.unpack_slicing(slicing, ndim)
slicing_file = slicing[:-ndim_list]
slicing_list = slicing[-ndim_list:]
shape_file = shape[:-ndim_list]
shape_list = shape[-ndim_list:]
sliced_shape_file = slc.sliced_shape(slicing=slicing_file,
shape=shape_file)
#sliced_shape_list = slc.sliced_shape(slicing=slicing_list, shape=shape_list);
#start indices
indices_start = self.expression.indices(self.file_list[0])
#print(indices_start)
#TODO: steps in file list
#genereate file list to read
#Note: indices increase according to the axes order but their own order is in tag order
indices = []
slicing_list_indices = []
shape_list_keep_dims = ()
slicing_keep_dims_to_final = (Ellipsis, )
for sl, s, i in zip(slicing_list, shape_list, indices_start):
if isinstance(sl, slice):
slice_indices = sl.indices(s)
slice_indices = (slice_indices[0] + i, slice_indices[1] + i,
slice_indices[2])
indices.append(range(*slice_indices))
n = len(indices[-1])
slicing_list_indices.append(range(n))
shape_list_keep_dims += (n, )
slicing_keep_dims_to_final += (slice(None), )
elif isinstance(sl, (list, np.ndarray)):
indices.append(np.array(sl) + i)
n = len(indices[-1])
slicing_list_indices.append(sl)
shape_list_keep_dims += (n, )
slicing_keep_dims_to_final += (slice(None), )
elif isinstance(sl, numbers.Integral):
indices.append([sl + i])
slicing_list_indices.append([0])
shape_list_keep_dims += (1, )
slicing_keep_dims_to_final += (0, )
else:
raise IndexError('Invalid slice specification %r!' % sl)
indices.reverse()
indices = itertools.product(*indices)
indices = [i[::-1] for i in indices]
slicing_list_indices.reverse()
slicing_list_indices = itertools.product(*slicing_list_indices)
slicing_list_indices = [i[::-1] for i in slicing_list_indices]
#print(indices, slicing_list_indices, slicing_keep_dims_to_final)
axes_to_tags = self.axes_to_tag_order()
if len(axes_to_tags) > 1 and axes_to_tags != list(
range(len(axes_to_tags))):
indices = [tuple(i[j] for j in axes_to_tags) for i in indices]
fl = [e.string_from_index(i) for i in indices]
#print(fl);
dtype = self.dtype
data = np.zeros(sliced_shape_file + shape_list_keep_dims,
dtype=dtype,
order=order)
#@ptb.parallel_traceback
def func(filename, index, data=data, slicing=slicing_file):
index = (Ellipsis, ) + index
data[index] = io.read(filename,
slicing=slicing,
processes='serial')
if processes is None:
processes = mp.cpu_count()
if processes == 'serial':
for f, i in zip(fl, slicing_list_indices):
func(f, i)
else:
with concurrent.futures.ThreadPoolExecutor(processes) as executor:
executor.map(func, fl, slicing_list_indices)
data = data[slicing_keep_dims_to_final]
return data