Example #1
0
  def loop_mem (self):
  # {{{
    '''Loop over smaller pieces of the view that fit in memory'''
    from pygeode import MAX_ARRAY_SIZE
#TODO: use itertools.product when everyone has python >= 2.6
#    from itertools import product
    from pygeode.tools import product
    # Determine the largest chunk that can be loaded, given the size constraint
    maxsize = MAX_ARRAY_SIZE

    # Get the shape of a single chunk
    indices = []
    for i in reversed(range(len(self.axes))):
      # Number of values along this axis we can get at a time
      N = self.shape[i]  # total length of this axis
      n = min(maxsize,N)  # amount we can fix in a single chunk
      # break up the axis into subslices
      input_indices = self.integer_indices[i]
      ind = [input_indices[j:min(j+n,N)] for j in range(0,N,n)]
      # Build the subslices from the last axis to the first
      indices = [ind] + indices
      # Take into account the count along this axis when looking at the faster-varying axes
      maxsize /= n

    # Loop over all combinations of slices to cover the whole view
    # (take the cartesian product)
    for ind in product(*indices):
      yield View(self.axes, ind)
Example #2
0
 def loop_contiguous(self):
 # {{{
   '''Break a non-contiguous view up into contiguous pieces
     (so that we don't have to handle it at a lower level)
     Input: this view
     Generates: outsl, start, count
        outsl is the current slice into an array that contains the view in a
              contiguous piece of memory
        start, count are corresponding arrays giving the start of the slice & its length.
     NOTE: These pieces might not fit in memory (see loop_mem for handling that).
           The only guarantee is that the pieces will be contiguous.  If the input
           view is already contiguous, then the slice will be over the *whole* view.'''
   #TODO: use itertools.product when everyone has python >= 2.6
   #    from itertools import product
   from pygeode.tools import product
   outslices = [contiguate(e) for e in self.integer_indices]
   inslices = [[simplify(ind[osl]) for osl in outsl] for ind,outsl in zip(self.integer_indices, outslices)]
   for outsl, insl in zip(product(*outslices), product(*inslices)):
     start = [sl.start if isinstance(sl,slice) else sl for sl in insl]
     count = [sl.stop - sl.start if isinstance(sl,slice) else 1 for sl in insl]
     yield outsl, start, count
Example #3
0
  var = concat(var1,var2)

  # The expected result
  expected = np.concatenate ( (array1, array2), iaxis)

  # Test this
  test = varTest(testname=testname, var=var, values=expected)

  # Store this test
  globals()[testname] = test


# Now, do some tests

sizes = (1, 2, 3, 20)

for naxes in (1,2,3):
  # Shape of the output
  for shape in product(*([sizes]*naxes)):
    # Concatenation axis
    for iaxis in range(naxes):
      n = shape[iaxis]
      # Length of first array
      for n1 in sorted(set([0, 1, 2, n/3, n-1, n])):
        if n1 < 0 or n1 > n: continue
        n2 = n - n1
        shape1 = shape[:iaxis]+(n1,)+shape[iaxis+1:]
        shape2 = shape[:iaxis]+(n2,)+shape[iaxis+1:]
        do_concat (shape1, shape2, iaxis)

Example #4
0
  var = concat(var1,var2)

  # The expected result
  expected = np.concatenate ( (array1, array2), iaxis)

  # Test this
  test = varTest(testname=testname, var=var, values=expected)

  # Store this test
  globals()[testname] = test


# Now, do some tests

sizes = (1, 2, 3, 20)

for naxes in (1,2,3):
  # Shape of the output
  for shape in product(*([sizes]*naxes)):
    # Concatenation axis
    for iaxis in range(naxes):
      n = shape[iaxis]
      # Length of first array
      for n1 in sorted(set([0, 1, 2, n//3, n-1, n])):
        if n1 < 0 or n1 > n: continue
        n2 = n - n1
        shape1 = shape[:iaxis]+(n1,)+shape[iaxis+1:]
        shape2 = shape[:iaxis]+(n2,)+shape[iaxis+1:]
        do_concat (shape1, shape2, iaxis)

Example #5
0
# Each size listed above is a dictionary entry, containing a list of slices
# to try for that axis size.
slices = {}
for size in sizes:
  slices[size] = gimme_some_slices(size)

# Each axis needs to be a distinct class, or view.get() gives bizarre error messages
from pygeode.axis import XAxis, YAxis, ZAxis
axis_classes = (XAxis, YAxis, ZAxis)

# Counter for giving each test a unique name
count = 1

for naxes in (1,2):
  print("Testing %s dimensions"%naxes)
  for shape in product(*([sizes]*naxes)):
    print("  Testing shape %s"%str(shape))

    np.random.seed(shape)
    values = np.random.randn(*shape)
#    print "full values:", values

    axes = [axis_classes[i](sorted(np.random.randn(n))) for i,n in enumerate(shape)]
    for i,axis in enumerate(axes):
      axis.name = 'axis%s'%i
#      print "axis %s values: %s"%(i,axis.values)

    var = Var(axes, values=values)
    var.name = 'myvar'

    slicelists = [slices[size] for size in shape]