def loop_mem (self):
# {{{
'''Loop over smaller pieces of the view that fit in memory'''
from pygeode import MAX_ARRAY_SIZE
#TODO: use itertools.product when everyone has python >= 2.6
# from itertools import product
from pygeode.tools import product
# Determine the largest chunk that can be loaded, given the size constraint
maxsize = MAX_ARRAY_SIZE
# Get the shape of a single chunk
indices = []
for i in reversed(range(len(self.axes))):
# Number of values along this axis we can get at a time
N = self.shape[i] # total length of this axis
n = min(maxsize,N) # amount we can fix in a single chunk
# break up the axis into subslices
input_indices = self.integer_indices[i]
ind = [input_indices[j:min(j+n,N)] for j in range(0,N,n)]
# Build the subslices from the last axis to the first
indices = [ind] + indices
# Take into account the count along this axis when looking at the faster-varying axes
maxsize /= n
# Loop over all combinations of slices to cover the whole view
# (take the cartesian product)
for ind in product(*indices):
yield View(self.axes, ind)
def loop_contiguous(self):
# {{{
'''Break a non-contiguous view up into contiguous pieces
(so that we don't have to handle it at a lower level)
Input: this view
Generates: outsl, start, count
outsl is the current slice into an array that contains the view in a
contiguous piece of memory
start, count are corresponding arrays giving the start of the slice & its length.
NOTE: These pieces might not fit in memory (see loop_mem for handling that).
The only guarantee is that the pieces will be contiguous. If the input
view is already contiguous, then the slice will be over the *whole* view.'''
#TODO: use itertools.product when everyone has python >= 2.6
# from itertools import product
from pygeode.tools import product
outslices = [contiguate(e) for e in self.integer_indices]
inslices = [[simplify(ind[osl]) for osl in outsl] for ind,outsl in zip(self.integer_indices, outslices)]
for outsl, insl in zip(product(*outslices), product(*inslices)):
start = [sl.start if isinstance(sl,slice) else sl for sl in insl]
count = [sl.stop - sl.start if isinstance(sl,slice) else 1 for sl in insl]
yield outsl, start, count
var = concat(var1,var2)
# The expected result
expected = np.concatenate ( (array1, array2), iaxis)
# Test this
test = varTest(testname=testname, var=var, values=expected)
# Store this test
globals()[testname] = test
# Now, do some tests
sizes = (1, 2, 3, 20)
for naxes in (1,2,3):
# Shape of the output
for shape in product(*([sizes]*naxes)):
# Concatenation axis
for iaxis in range(naxes):
n = shape[iaxis]
# Length of first array
for n1 in sorted(set([0, 1, 2, n/3, n-1, n])):
if n1 < 0 or n1 > n: continue
n2 = n - n1
shape1 = shape[:iaxis]+(n1,)+shape[iaxis+1:]
shape2 = shape[:iaxis]+(n2,)+shape[iaxis+1:]
do_concat (shape1, shape2, iaxis)
var = concat(var1,var2)
# The expected result
expected = np.concatenate ( (array1, array2), iaxis)
# Test this
test = varTest(testname=testname, var=var, values=expected)
# Store this test
globals()[testname] = test
# Now, do some tests
sizes = (1, 2, 3, 20)
for naxes in (1,2,3):
# Shape of the output
for shape in product(*([sizes]*naxes)):
# Concatenation axis
for iaxis in range(naxes):
n = shape[iaxis]
# Length of first array
for n1 in sorted(set([0, 1, 2, n//3, n-1, n])):
if n1 < 0 or n1 > n: continue
n2 = n - n1
shape1 = shape[:iaxis]+(n1,)+shape[iaxis+1:]
shape2 = shape[:iaxis]+(n2,)+shape[iaxis+1:]
do_concat (shape1, shape2, iaxis)
# Each size listed above is a dictionary entry, containing a list of slices
# to try for that axis size.
slices = {}
for size in sizes:
slices[size] = gimme_some_slices(size)
# Each axis needs to be a distinct class, or view.get() gives bizarre error messages
from pygeode.axis import XAxis, YAxis, ZAxis
axis_classes = (XAxis, YAxis, ZAxis)
# Counter for giving each test a unique name
count = 1
for naxes in (1,2):
print("Testing %s dimensions"%naxes)
for shape in product(*([sizes]*naxes)):
print(" Testing shape %s"%str(shape))
np.random.seed(shape)
values = np.random.randn(*shape)
# print "full values:", values
axes = [axis_classes[i](sorted(np.random.randn(n))) for i,n in enumerate(shape)]
for i,axis in enumerate(axes):
axis.name = 'axis%s'%i
# print "axis %s values: %s"%(i,axis.values)
var = Var(axes, values=values)
var.name = 'myvar'
slicelists = [slices[size] for size in shape]