def _validate(self):
"""
Ensure that our expression string has variables of the form x_0, x_1,
... x_(N - 1), where N is the length of our inputs.
"""
variable_names, _unused = getExprNames(self._expr, {})
expr_indices = []
for name in variable_names:
if name == "inf":
continue
match = _VARIABLE_NAME_RE.match(name)
if not match:
raise ValueError("%r is not a valid variable name" % name)
expr_indices.append(int(match.group(2)))
expr_indices.sort()
expected_indices = list(range(len(self.inputs)))
if expr_indices != expected_indices:
raise ValueError(
"Expected %s for variable indices, but got %s"
% (
expected_indices,
expr_indices,
)
)
super(NumericalExpression, self)._validate()
def compile_condition(condition, typemap, indexedcols):
"""Compile a condition and extract usable index conditions.
Looks for variable-constant comparisons in the `condition` string
involving the indexed columns whose variable names appear in
`indexedcols`. The part of `condition` having usable indexes is
returned as a compiled condition in a `CompiledCondition` container.
Expressions such as '0 < c1 <= 1' do not work as expected. The
Numexpr types of *all* variables must be given in the `typemap`
mapping. The ``function`` of the resulting `CompiledCondition`
instance is a Numexpr function object, and the ``parameters`` list
indicates the order of its parameters.
"""
# Get the expression tree and extract index conditions.
expr = stringToExpression(condition, typemap, {})
if expr.astKind != 'bool':
raise TypeError("condition ``%s`` does not have a boolean type"
% condition)
idxexprs = _get_idx_expr(expr, indexedcols)
# Post-process the answer
if isinstance(idxexprs, list):
# Simple expression
strexpr = ['e0']
else:
# Complex expression
idxexprs, strexpr = idxexprs
# Get rid of the unneccessary list wrapper for strexpr
strexpr = strexpr[0]
# Get the variable names used in the condition.
# At the same time, build its signature.
varnames = _get_variable_names(expr)
signature = [(var, typemap[var]) for var in varnames]
try:
# See the comments in `numexpr.evaluate()` for the
# reasons of inserting copy operators for unaligned,
# *unidimensional* arrays.
func = NumExpr(expr, signature)
except NotImplementedError as nie:
# Try to make this Numexpr error less cryptic.
raise _unsupported_operation_error(nie)
_, ex_uses_vml = getExprNames(condition, {})
kwargs = {'ex_uses_vml': ex_uses_vml}
params = varnames
# This is more comfortable to handle about than a tuple.
return CompiledCondition(func, params, idxexprs, strexpr, **kwargs)
def compile_condition(condition, typemap, indexedcols):
"""Compile a condition and extract usable index conditions.
Looks for variable-constant comparisons in the `condition` string
involving the indexed columns whose variable names appear in
`indexedcols`. The part of `condition` having usable indexes is
returned as a compiled condition in a `CompiledCondition` container.
Expressions such as '0 < c1 <= 1' do not work as expected. The
Numexpr types of *all* variables must be given in the `typemap`
mapping. The ``function`` of the resulting `CompiledCondition`
instance is a Numexpr function object, and the ``parameters`` list
indicates the order of its parameters.
"""
# Get the expression tree and extract index conditions.
expr = stringToExpression(condition, typemap, {})
if expr.astKind != 'bool':
raise TypeError("condition ``%s`` does not have a boolean type"
% condition)
idxexprs = _get_idx_expr(expr, indexedcols)
# Post-process the answer
if isinstance(idxexprs, list):
# Simple expression
strexpr = ['e0']
else:
# Complex expression
idxexprs, strexpr = idxexprs
# Get rid of the unneccessary list wrapper for strexpr
strexpr = strexpr[0]
# Get the variable names used in the condition.
# At the same time, build its signature.
varnames = _get_variable_names(expr)
signature = [(var, typemap[var]) for var in varnames]
try:
# See the comments in `numexpr.evaluate()` for the
# reasons of inserting copy operators for unaligned,
# *unidimensional* arrays.
func = NumExpr(expr, signature)
except NotImplementedError as nie:
# Try to make this Numexpr error less cryptic.
raise _unsupported_operation_error(nie)
_, ex_uses_vml = getExprNames(condition, {})
kwargs = {'ex_uses_vml': ex_uses_vml}
params = varnames
# This is more comfortable to handle about than a tuple.
return CompiledCondition(func, params, idxexprs, strexpr, **kwargs)
def _validate(self):
"""
Ensure that our expression string has variables of the form x_0, x_1,
... x_(N - 1), where N is the length of our inputs.
"""
variable_names, _unused = getExprNames(self._expr, {})
expr_indices = []
for name in variable_names:
match = _VARIABLE_NAME_RE.match(name)
if not match:
raise ValueError("%r is not a valid variable name" % name)
expr_indices.append(int(match.group(2)))
expr_indices.sort()
expected_indices = list(range(len(self.inputs)))
if expr_indices != expected_indices:
raise ValueError(
"Expected %s for variable indices, but got %s" % (
expected_indices, expr_indices,
)
)
return super(NumericalExpression, self)._validate()
def evaluate(ex, out=None, local_dict=None, global_dict=None, **kwargs):
"""Evaluate expression and return an array."""
# First, get the signature for the arrays in expression
context = getContext(kwargs)
names, _ = getExprNames(ex, context)
# Get the arguments based on the names.
call_frame = sys._getframe(1)
if local_dict is None:
local_dict = call_frame.f_locals
if global_dict is None:
global_dict = call_frame.f_globals
arguments = []
types = []
for name in names:
try:
a = local_dict[name]
except KeyError:
a = global_dict[name]
arguments.append(a)
if hasattr(a, 'atom'):
types.append(a.atom)
else:
types.append(a)
# Create a signature
signature = [(name, getType(type_)) for (name, type_) in zip(names, types)]
print("signature-->", signature)
# Compile the expression
compiled_ex = NumExpr(ex, signature, [], **kwargs)
print("fullsig-->", compiled_ex.fullsig)
_compute(out, compiled_ex, arguments)
return
def evaluate(ex, out=None, local_dict=None, global_dict=None, **kwargs):
"""Evaluate expression and return an array."""
# First, get the signature for the arrays in expression
context = getContext(kwargs)
names, _ = getExprNames(ex, context)
# Get the arguments based on the names.
call_frame = sys._getframe(1)
if local_dict is None:
local_dict = call_frame.f_locals
if global_dict is None:
global_dict = call_frame.f_globals
arguments = []
types = []
for name in names:
try:
a = local_dict[name]
except KeyError:
a = global_dict[name]
arguments.append(a)
if hasattr(a, 'atom'):
types.append(a.atom)
else:
types.append(a)
# Create a signature
signature = [(name, getType(type_)) for (name, type_) in zip(names, types)]
print("signature-->", signature)
# Compile the expression
compiled_ex = NumExpr(ex, signature, [], **kwargs)
print("fullsig-->", compiled_ex.fullsig)
_compute(out, compiled_ex, arguments)
return
def __init__(self, expr, uservars=None, **kwargs):
self.append_mode = False
"""The append mode for user-provided output containers."""
self.maindim = 0
"""Common main dimension for inputs in expression."""
self.names = []
"""The names of variables in expression (list)."""
self.out = None
"""The user-provided container (if any) for the expression outcome."""
self.o_start = None
"""The start range selection for the user-provided output."""
self.o_stop = None
"""The stop range selection for the user-provided output."""
self.o_step = None
"""The step range selection for the user-provided output."""
self.shape = None
"""Common shape for the arrays in expression."""
self.start, self.stop, self.step = (None,) * 3
self.start = None
"""The start range selection for the input."""
self.stop = None
"""The stop range selection for the input."""
self.step = None
"""The step range selection for the input."""
self.values = []
"""The values of variables in expression (list)."""
self._compiled_expr = None
"""The compiled expression."""
self._single_row_out = None
"""A sample of the output with just a single row."""
# First, get the signature for the arrays in expression
vars_ = self._required_expr_vars(expr, uservars)
context = getContext(kwargs)
self.names, _ = getExprNames(expr, context)
# Raise a ValueError in case we have unsupported objects
for name, var in vars_.iteritems():
if type(var) in (int, long, float, str):
continue
if not isinstance(var, (tb.Leaf, tb.Column)):
if hasattr(var, "dtype"):
# Quacks like a NumPy object
continue
raise TypeError("Unsupported variable type: %r" % var)
objname = var.__class__.__name__
if objname not in ("Array", "CArray", "EArray", "Column"):
raise TypeError("Unsupported variable type: %r" % var)
# NumPy arrays to be copied? (we don't need to worry about
# PyTables objects, as the reads always return contiguous and
# aligned objects, or at least I think so).
for name, var in vars_.iteritems():
if isinstance(var, np.ndarray):
# See numexpr.necompiler.evaluate for a rational
# of the code below
if not var.flags.aligned:
if var.ndim != 1:
# Do a copy of this variable
var = var.copy()
# Update the vars_ dictionary
vars_[name] = var
# Get the variables and types
values = self.values
types_ = []
for name in self.names:
value = vars_[name]
if hasattr(value, 'atom'):
types_.append(value.atom)
elif hasattr(value, 'dtype'):
types_.append(value)
else:
# try to convert into a NumPy array
value = np.array(value)
types_.append(value)
values.append(value)
# Create a signature for the expression
signature = [(name, getType(type_))
for (name, type_) in zip(self.names, types_)]
# Compile the expression
self._compiled_expr = NumExpr(expr, signature, **kwargs)
# Guess the shape for the outcome and the maindim of inputs
self.shape, self.maindim = self._guess_shape()
def __init__(self, expr, uservars=None, **kwargs):
self.append_mode = False
"""The append mode for user-provided output containers."""
self.maindim = 0
"""Common main dimension for inputs in expression."""
self.names = []
"""The names of variables in expression (list)."""
self.out = None
"""The user-provided container (if any) for the expression outcome."""
self.o_start = None
"""The start range selection for the user-provided output."""
self.o_stop = None
"""The stop range selection for the user-provided output."""
self.o_step = None
"""The step range selection for the user-provided output."""
self.shape = None
"""Common shape for the arrays in expression."""
self.start, self.stop, self.step = (None, ) * 3
self.start = None
"""The start range selection for the input."""
self.stop = None
"""The stop range selection for the input."""
self.step = None
"""The step range selection for the input."""
self.values = []
"""The values of variables in expression (list)."""
self._compiled_expr = None
"""The compiled expression."""
self._single_row_out = None
"""A sample of the output with just a single row."""
# First, get the signature for the arrays in expression
vars_ = self._requiredExprVars(expr, uservars)
context = getContext(kwargs)
self.names, _ = getExprNames(expr, context)
# Raise a ValueError in case we have unsupported objects
for name, var in vars_.iteritems():
if type(var) in (int, long, float, str):
continue
if not isinstance(var, (tb.Leaf, tb.Column)):
if hasattr(var, "dtype"):
# Quacks like a NumPy object
continue
raise TypeError("Unsupported variable type: %r" % var)
objname = var.__class__.__name__
if objname not in ("Array", "CArray", "EArray", "Column"):
raise TypeError("Unsupported variable type: %r" % var)
# NumPy arrays to be copied? (we don't need to worry about
# PyTables objects, as the reads always return contiguous and
# aligned objects, or at least I think so).
copy_args = []
for name, var in vars_.iteritems():
if isinstance(var, np.ndarray):
# See numexpr.necompiler.evaluate for a rational
# of the code below
if not var.flags.aligned:
if var.ndim == 1:
copy_args.append(name)
else:
# Do a copy of this variable
var = var.copy()
# Update the vars_ dictionary
vars_[name] = var
# Get the variables and types
values = self.values
types = []
for name in self.names:
value = vars_[name]
if hasattr(value, 'atom'):
types.append(value.atom)
elif hasattr(value, 'dtype'):
types.append(value)
else:
# try to convert into a NumPy array
value = np.array(value)
types.append(value)
values.append(value)
# Create a signature for the expression
signature = [(name, getType(type_))
for (name, type_) in zip(self.names, types)]
# Compile the expression
self._compiled_expr = NumExpr(expr, signature, copy_args, **kwargs)
# Guess the shape for the outcome and the maindim of inputs
self.shape, self.maindim = self._guess_shape()
def __init__(self, expr, uservars=None, **kwargs):
"""Compile the expression and initialize internal structures.
`expr` must be specified as a string like "2*a+3*b".
The `uservars` mapping may be used to define the variable names
appearing in `expr`. This mapping should consist of
identifier-like strings pointing to any `Array`, `CArray`,
`EArray`, `Column` or NumPy ndarray instances (or even others
which will tried to be converted to ndarrays).
When `uservars` is not provided or `None`, the current local and
global namespace is sought instead of `uservars`. It is also
possible to pass just some of the variables in expression via
the `uservars` mapping, and the rest will be retrieved from the
current local and global namespaces.
`**kwargs` is meant to pass additional parameters to the Numexpr
kernel. This is basically the same as the `**kwargs` argument
in `Numexpr.evaluate()`, and is mainly meant for advanced
use.
After initialized, an `Expr` instance can be evaluated via its
`eval()` method. This class also provides an `__iter__()`
method that iterates over all the resulting rows in expression.
Example of use:
>>> a = f.createArray('/', 'a', np.array([1,2,3]))
>>> b = f.createArray('/', 'b', np.array([3,4,5]))
>>> c = np.array([4,5,6])
>>> expr = tb.Expr("2*a+b*c") # initialize the expression
>>> expr.eval() # evaluate it
array([14, 24, 36])
>>> sum(expr) # use as an iterator
74
where you can see that you can mix different containers in the
expression (whenever shapes are consistent).
You can also work with multidimensional arrays:
>>> a2 = f.createArray('/', 'a2', np.array([[1,2],[3,4]]))
>>> b2 = f.createArray('/', 'b2', np.array([[3,4],[5,6]]))
>>> c2 = np.array([4,5]) # This will be broadcasted
>>> expr = tb.Expr("2*a2+b2-c2")
>>> expr.eval()
array([[1, 3],
[7, 9]])
>>> sum(expr)
array([ 8, 12])
"""
self.append_mode = False
"""The append mode for user-provided output containers."""
self.maindim = 0
"""Common main dimension for inputs in expression."""
self.names = []
"""The names of variables in expression (list)."""
self.out = None
"""The user-provided container (if any) for the expression outcome."""
self.o_start, self.o_stop, self.o_step = (None, ) * 3
"""The range selection for the user-provided output."""
self.shape = None
"""Common shape for the arrays in expression."""
self.start, self.stop, self.step = (None, ) * 3
"""The range selection for all the inputs."""
self.values = []
"""The values of variables in expression (list)."""
self._compiled_expr = None
"""The compiled expression."""
self._single_row_out = None
"""A sample of the output with just a single row."""
# First, get the signature for the arrays in expression
vars_ = self._requiredExprVars(expr, uservars)
context = getContext(kwargs)
self.names, _ = getExprNames(expr, context)
# Raise a ValueError in case we have unsupported objects
for name, var in vars_.items():
if type(var) in (int, long, float, str):
continue
if not isinstance(var, (tb.Leaf, tb.Column)):
if hasattr(var, "dtype"):
# Quacks like a NumPy object
continue
raise TypeError("Unsupported variable type: %r" % var)
objname = var.__class__.__name__
if objname not in ("Array", "CArray", "EArray", "Column"):
raise TypeError("Unsupported variable type: %r" % var)
# NumPy arrays to be copied? (we don't need to worry about
# PyTables objects, as the reads always return contiguous and
# aligned objects, or at least I think so).
copy_args = []
for name, var in vars_.items():
if type(var) == np.ndarray:
# See numexpr.necompiler.evaluate for a rational
# of the code below
if not var.flags.aligned:
if var.ndim == 1:
copy_args.append(name)
else:
# Do a copy of this variable
var = var.copy()
# Update the vars_ dictionary
vars_[name] = var
# Get the variables and types
values = self.values
types = []
for name in self.names:
value = vars_[name]
if hasattr(value, 'atom'):
types.append(value.atom)
elif hasattr(value, 'dtype'):
types.append(value)
else:
# try to convert into a NumPy array
value = np.array(value)
types.append(value)
values.append(value)
# Create a signature for the expression
signature = [(name, getType(type_))
for (name, type_) in zip(self.names, types)]
# Compile the expression
self._compiled_expr = NumExpr(expr, signature, copy_args, **kwargs)
# Guess the shape for the outcome and the maindim of inputs
self.shape, self.maindim = self._guess_shape()
def __init__(self, expr, uservars=None, **kwargs):
"""Compile the expression and initialize internal structures.
`expr` must be specified as a string like "2*a+3*b".
The `uservars` mapping may be used to define the variable names
appearing in `expr`. This mapping should consist of
identifier-like strings pointing to any `Array`, `CArray`,
`EArray`, `Column` or NumPy ndarray instances (or even others
which will tried to be converted to ndarrays).
When `uservars` is not provided or `None`, the current local and
global namespace is sought instead of `uservars`. It is also
possible to pass just some of the variables in expression via
the `uservars` mapping, and the rest will be retrieved from the
current local and global namespaces.
`**kwargs` is meant to pass additional parameters to the Numexpr
kernel. This is basically the same as the `**kwargs` argument
in `Numexpr.evaluate()`, and is mainly meant for advanced
use.
After initialized, an `Expr` instance can be evaluated via its
`eval()` method. This class also provides an `__iter__()`
method that iterates over all the resulting rows in expression.
Example of use:
>>> a = f.createArray('/', 'a', np.array([1,2,3]))
>>> b = f.createArray('/', 'b', np.array([3,4,5]))
>>> c = np.array([4,5,6])
>>> expr = tb.Expr("2*a+b*c") # initialize the expression
>>> expr.eval() # evaluate it
array([14, 24, 36])
>>> sum(expr) # use as an iterator
74
where you can see that you can mix different containers in the
expression (whenever shapes are consistent).
You can also work with multidimensional arrays:
>>> a2 = f.createArray('/', 'a2', np.array([[1,2],[3,4]]))
>>> b2 = f.createArray('/', 'b2', np.array([[3,4],[5,6]]))
>>> c2 = np.array([4,5]) # This will be broadcasted
>>> expr = tb.Expr("2*a2+b2-c2")
>>> expr.eval()
array([[1, 3],
[7, 9]])
>>> sum(expr)
array([ 8, 12])
"""
self.append_mode = False
"""The append mode for user-provided output containers."""
self.maindim = 0
"""Common main dimension for inputs in expression."""
self.names = []
"""The names of variables in expression (list)."""
self.out = None
"""The user-provided container (if any) for the expression outcome."""
self.o_start, self.o_stop, self.o_step = (None,) * 3
"""The range selection for the user-provided output."""
self.shape = None
"""Common shape for the arrays in expression."""
self.start, self.stop, self.step = (None,) * 3
"""The range selection for all the inputs."""
self.values = []
"""The values of variables in expression (list)."""
self._compiled_expr = None
"""The compiled expression."""
self._single_row_out = None
"""A sample of the output with just a single row."""
# First, get the signature for the arrays in expression
vars_ = self._requiredExprVars(expr, uservars)
context = getContext(kwargs)
self.names, _ = getExprNames(expr, context)
# Raise a ValueError in case we have unsupported objects
for name, var in vars_.items():
if type(var) in (int, long, float, str):
continue
if not isinstance(var, (tb.Leaf, tb.Column)):
if hasattr(var, "dtype"):
# Quacks like a NumPy object
continue
raise TypeError("Unsupported variable type: %r" % var)
objname = var.__class__.__name__
if objname not in ("Array", "CArray", "EArray", "Column"):
raise TypeError("Unsupported variable type: %r" % var)
# NumPy arrays to be copied? (we don't need to worry about
# PyTables objects, as the reads always return contiguous and
# aligned objects, or at least I think so).
copy_args = []
for name, var in vars_.items():
if type(var) == np.ndarray:
# See numexpr.necompiler.evaluate for a rational
# of the code below
if not var.flags.aligned:
if var.ndim == 1:
copy_args.append(name)
else:
# Do a copy of this variable
var = var.copy()
# Update the vars_ dictionary
vars_[name] = var
# Get the variables and types
values = self.values
types = []
for name in self.names:
value = vars_[name]
if hasattr(value, "atom"):
types.append(value.atom)
elif hasattr(value, "dtype"):
types.append(value)
else:
# try to convert into a NumPy array
value = np.array(value)
types.append(value)
values.append(value)
# Create a signature for the expression
signature = [(name, getType(type_)) for (name, type_) in zip(self.names, types)]
# Compile the expression
self._compiled_expr = NumExpr(expr, signature, copy_args, **kwargs)
# Guess the shape for the outcome and the maindim of inputs
self.shape, self.maindim = self._guess_shape()