def getview(self, view, pbar):
import numpy as np
from pygeode.view import View, simplify
out = np.empty(view.shape, dtype=self.dtype)
out[()] = float('nan')
out_axes = view.clip().axes
# Loop over all available files.
N = 0 # Number of points covered so far
for filename, opener, axes in self._table:
subaxes = [
self._axis_manager._get_axis_intersection([a1, a2])
for a1, a2 in zip(out_axes, axes)
]
reorder = []
mask = []
if any(len(a) == 0 for a in subaxes): continue
for a1, a2 in zip(out_axes, subaxes):
# Figure out where the input chunk fits into the output
re = np.searchsorted(a2.values, a1.values)
# Mask out elements that we don't actually have in the chunk
m = [
r < len(a2.values) and a2.values[r] == v
for r, v in zip(re, a1.values)
]
m = np.array(m)
# Convert mask to integer indices
m = np.arange(len(m))[m]
# and then to a slice (where possible)
m = simplify(m)
re = re[m]
# Try to simplify the re-ordering array
if np.all(re == np.sort(re)):
re = simplify(re)
reorder.append(re)
mask.append(m)
var = [v for v in opener(filename) if v.name == self._varname][0]
v = View(subaxes)
chunk = v.get(var)
# Note: this may break if there is more than one axis with integer indices.
assert len([
r for r in reorder if isinstance(r, (tuple, np.ndarray))
]) <= 1, "Unhandled advanced indexing case."
assert len([m for m in mask if isinstance(m, (tuple, np.ndarray))
]) <= 1, "Unhandled advanced indexing case."
out[mask] = chunk[reorder]
N = N + chunk.size
pbar.update(100. * N / out.size)
return out
def getview (self, view, pbar):
import numpy as np
from pygeode.view import View, simplify
out = np.empty(view.shape, dtype=self.dtype)
out[()] = float('nan')
out_axes = view.clip().axes
# Loop over all available files.
N = 0 # Number of points covered so far
for filename, opener, axes in self._table:
subaxes = [self._axis_manager._get_axis_intersection([a1,a2]) for a1,a2 in zip(out_axes,axes)]
reorder = []
mask = []
if any(len(a)==0 for a in subaxes): continue
for a1,a2 in zip(out_axes,subaxes):
# Figure out where the input chunk fits into the output
re = np.searchsorted(a2.values, a1.values)
# Mask out elements that we don't actually have in the chunk
m = [r<len(a2.values) and a2.values[r]==v for r,v in zip(re,a1.values)]
m = np.array(m)
# Convert mask to integer indices
m = np.arange(len(m))[m]
# and then to a slice (where possible)
m = simplify(m)
re = re[m]
# Try to simplify the re-ordering array
if np.all(re == np.sort(re)):
re = simplify(re)
reorder.append(re)
mask.append(m)
var = [v for v in opener(filename) if v.name == self._varname][0]
v = View(subaxes)
chunk = v.get(var)
# Note: this may break if there is more than one axis with integer indices.
assert len([r for r in reorder if isinstance(r,(tuple,np.ndarray))]) <= 1, "Unhandled advanced indexing case."
assert len([m for m in mask if isinstance(m,(tuple,np.ndarray))]) <= 1, "Unhandled advanced indexing case."
out[mask] = chunk[reorder]
N = N + chunk.size
pbar.update(100.*N/out.size)
return out
def get_slice(self, kwargs, ignore_mismatch=False):
# {{{
import numpy as np
from pygeode.view import simplify, expand
# boolean flags indicating which axis indices will be used
n = len(self)
keep = np.ones(n, bool)
matched = []
for k, v in kwargs.items():
# Split off prefix if present
if '_' in k and not self.has_alias(k):
prefix, ax = k.split('_', 1)
else:
prefix, ax = '', k
if 'i' in prefix:
################### Select by index; key must correspond to this axis
if not self.has_alias(ax):
if ignore_mismatch: continue
raise Exception(
"'%s' is not associated with this %s axis" %
(ax, self.name))
# Build mask
kp = np.zeros(n, bool)
if not hasattr(v, '__len__'): # Treat as an index
kp[v] = True
elif len(v) > 3 or len(
v) < 2 or 'l' in prefix: # Treat as integer array
kp[np.array(v, 'int')] = True
elif len(v) == 2: # Treat as slice
kp[v[0]:v[1]] = True
elif len(v) == 3: # Treat as slice with stride
kp[v[0]:v[1]:v[2]] = True
else:
raise ValueException(
"'%s' is not associated with this %s axis" %
(ax, self.name))
else:
################### Select by value
if self.has_alias(ax): # Does key match this axis?
vals = self.values
elif ax in self.auxarrays: # What about an aux. array?
vals = self.auxarrays[ax]
else:
if ignore_mismatch: continue
raise Exception(
"'%s' is not associated with this %s axis" %
(ax, self.name))
# Build mask
kp = np.zeros(n, bool)
# Convert string representation if necessary
if isinstance(v, str): v = self.str_as_val(ax, v)
if isinstance(v, str) or not hasattr(
v, '__len__'): # Single value given
if vals.dtype.name.startswith('float'): # closest match
kp[np.argmin(np.abs(v - vals))] = True
else: # otherwise require an exact match
kp[vals == v] = True
elif 'l' in prefix:
for V in v:
# Convert string representation if necessary
if isinstance(V, str): V = self.str_as_val(ax, V)
if vals.dtype.name.startswith(
'float'): # closest match
kp[np.argmin(np.abs(V - vals))] = True
else: # otherwise require an exact match
kp[vals == V] = True
elif len(v) == 2: # Select within range
# Convert string representations if necessary
v = [
self.str_as_val(ax, V) if isinstance(V, str) else V
for V in v
]
lower, upper = min(v), max(v)
kp[(lower <= vals) & (vals <= upper)] = True
else: # Don't know what to do with more than 2 values
raise Exception('A range must be specified')
# Use complement of requested set
if 'n' in prefix: kp = ~kp
# Compute intersection of index sets
keep &= kp
matched.append(k) # Mark for removal from slice list
# Pop kw arguments that have been handled
for m in matched:
kwargs.pop(m)
# Convert boolean mask to integer indices
sl = np.flatnonzero(keep)
# Filter through view.simplify() to construct slice objects wherever possible
# (otherwise, it's a generic integer array)
return simplify(sl)
def get_slice(self, kwargs, ignore_mismatch=False):
# {{{
import numpy as np
from pygeode.view import simplify, expand
# boolean flags indicating which axis indices will be used
n = len(self)
keep = np.ones(n,bool)
matched = []
for k, v in kwargs.items():
# Split off prefix if present
if '_' in k and not self.has_alias(k):
prefix, ax = k.split('_', 1)
else:
prefix, ax = '', k
if 'i' in prefix:
################### Select by index; key must correspond to this axis
if not self.has_alias(ax):
if ignore_mismatch: continue
raise Exception("'%s' is not associated with this %s axis" % (ax, self.name))
# Build mask
kp = np.zeros(n, bool)
if not hasattr(v, '__len__'): # Treat as an index
kp[v] = True
elif len(v) > 3 or len(v) < 2 or 'l' in prefix: # Treat as integer array
kp[np.array(v, 'int')] = True
elif len(v) == 2: # Treat as slice
kp[v[0]:v[1]] = True
elif len(v) == 3: # Treat as slice with stride
kp[v[0]:v[1]:v[2]] = True
else:
raise ValueException("'%s' is not associated with this %s axis" % (ax, self.name))
else:
################### Select by value
if self.has_alias(ax): # Does key match this axis?
vals = self.values
elif ax in self.auxarrays: # What about an aux. array?
vals = self.auxarrays[ax]
else:
if ignore_mismatch: continue
raise Exception("'%s' is not associated with this %s axis" % (ax, self.name))
# Build mask
kp = np.zeros(n, bool)
# Convert string representation if necessary
if isinstance(v, str): v = self.str_as_val(ax, v)
if isinstance(v,str) or not hasattr(v,'__len__'): # Single value given
if vals.dtype.name.startswith('float'): # closest match
kp[np.argmin( np.abs(v-vals) )] = True
else: # otherwise require an exact match
kp[vals == v] = True
elif 'l' in prefix:
for V in v:
# Convert string representation if necessary
if isinstance(V, str): V = self.str_as_val(ax, V)
if vals.dtype.name.startswith('float'): # closest match
kp[np.argmin( np.abs(V-vals) )] = True
else: # otherwise require an exact match
kp[vals == V] = True
elif len(v) == 2: # Select within range
# Convert string representations if necessary
v = [self.str_as_val(ax, V) if isinstance(V, str) else V for V in v]
lower, upper = min(v), max(v)
kp[(lower <= vals) & (vals <= upper)] = True
else: # Don't know what to do with more than 2 values
raise Exception('A range must be specified')
# Use complement of requested set
if 'n' in prefix: kp = ~kp
# Compute intersection of index sets
keep &= kp
matched.append(k) # Mark for removal from slice list
# Pop kw arguments that have been handled
for m in matched:
kwargs.pop(m)
# Convert boolean mask to integer indices
sl = np.flatnonzero(keep)
# Filter through view.simplify() to construct slice objects wherever possible
# (otherwise, it's a generic integer array)
return simplify(sl)