def execute_binary_op_series_group_by(op, left, right, **kwargs):
left_groupings = left.grouper.groupings
right_groupings = right.grouper.groupings
if left_groupings != right_groupings:
raise ValueError('Cannot perform {} operation on two series with '
'different groupings'.format(type(op).__name__))
result = execute_node(op, left.obj, right.obj, **kwargs)
return result.groupby(left_groupings)
def execute_with_scope(expr, scope, context=None, **kwargs):
"""Execute an expression `expr`, with data provided in `scope`.
Parameters
----------
expr : ir.Expr
The expression to execute.
scope : dict
A dictionary mapping :class:`~ibis.expr.types.Node` subclass instances
to concrete data such as a pandas DataFrame.
Returns
-------
result : scalar, pd.Series, pd.DataFrame
"""
op = expr.op()
# base case: our op has been computed (or is a leaf data node), so
# return the corresponding value
if op in scope:
return scope[op]
if context is None:
context = ctx.Summarize()
try:
computed_args = [scope[t] for t in op.root_tables()]
except KeyError:
pass
else:
try:
# special case: we have a definition of execute_first that matches
# our current operation and data leaves
return execute_first(op,
*computed_args,
scope=scope,
context=context,
**kwargs)
except NotImplementedError:
pass
args = op.args
# recursively compute the op's arguments
computed_args = [
execute(arg, scope, context=context, **kwargs)
if hasattr(arg, 'op') else arg for arg in args
if isinstance(arg, _VALID_INPUT_TYPES)
]
# Compute our op, with its computed arguments
return execute_node(op,
*computed_args,
scope=scope,
context=context,
**kwargs)
def execute_log_series_gb_series_gb(op, left, right, **kwargs):
result = execute_node(op, left.obj, right.obj, **kwargs)
return result.groupby(left.grouper.groupings)
def execute_unary_op_series_gb(op, operand, **kwargs):
result = execute_node(op, operand.obj, **kwargs)
return result.groupby(operand.grouper.groupings)
def execute_binary_op_simple_series_gb(op, left, right, **kwargs):
result = execute_node(op, left, right.obj, **kwargs)
return result.groupby(right.grouper.groupings)
def execute_cast_series_group_by(op, data, type, **kwargs):
result = execute_node(op, data.obj, type, **kwargs)
return result.groupby(data.grouper.groupings)
def execute_bottom_up(expr,
scope,
aggcontext=None,
post_execute_=None,
clients=None,
**kwargs):
"""Execute `expr` bottom-up.
Parameters
----------
expr : ibis.expr.types.Expr
scope : Mapping[ibis.expr.operations.Node, object]
aggcontext : Optional[ibis.pandas.aggcontext.AggregationContext]
kwargs : Dict[str, object]
Returns
-------
result : Mapping[
ibis.expr.operations.Node,
Union[pandas.Series, pandas.DataFrame, scalar_types]
]
A mapping from node to the computed result of that Node
"""
assert post_execute_ is not None, 'post_execute_ is None'
op = expr.op()
# if we're in scope then return the scope, this will then be passed back
# into execute_bottom_up, which will then terminate
if op in scope:
return scope
elif isinstance(op, ops.Literal):
# special case literals to avoid the overhead of dispatching
# execute_node
return {
op:
execute_literal(op,
op.value,
expr.type(),
aggcontext=aggcontext,
**kwargs)
}
# figure out what arguments we're able to compute on based on the
# expressions inputs. things like expressions, None, and scalar types are
# computable whereas ``list``s are not
computable_args = [arg for arg in op.inputs if is_computable_input(arg)]
# recursively compute each node's arguments until we've changed type
scopes = [
execute_bottom_up(
arg,
scope,
aggcontext=aggcontext,
post_execute_=post_execute_,
clients=clients,
**kwargs,
) if hasattr(arg, 'op') else {
arg: arg
} for arg in computable_args
]
# if we're unable to find data then raise an exception
if not scopes:
raise com.UnboundExpressionError(
'Unable to find data for expression:\n{}'.format(repr(expr)))
# there should be exactly one dictionary per computable argument
assert len(computable_args) == len(scopes)
new_scope = toolz.merge(scopes)
# pass our computed arguments to this node's execute_node implementation
data = [
new_scope[arg.op()] if hasattr(arg, 'op') else arg
for arg in computable_args
]
result = execute_node(
op,
*data,
scope=scope,
aggcontext=aggcontext,
clients=clients,
**kwargs,
)
computed = post_execute_(op, result)
return {op: computed}
def execute_decimal_log_with_np_integer_base(op, data, base, **kwargs):
return execute_node(op, data, int(base), **kwargs)
def execute_until_in_scope(expr,
scope,
aggcontext=None,
clients=None,
post_execute_=None,
**kwargs):
"""Execute until our op is in `scope`.
Parameters
----------
expr : ibis.expr.types.Expr
scope : Mapping
aggcontext : Optional[AggregationContext]
clients : List[ibis.client.Client]
kwargs : Mapping
"""
# these should never be None
assert aggcontext is not None, 'aggcontext is None'
assert clients is not None, 'clients is None'
assert post_execute_ is not None, 'post_execute_ is None'
# base case: our op has been computed (or is a leaf data node), so
# return the corresponding value
op = expr.op()
if op in scope:
return scope
elif isinstance(op, ops.Literal):
# special case literals to avoid the overhead of dispatching
# execute_node
return {
op:
execute_literal(op,
op.value,
expr.type(),
aggcontext=aggcontext,
**kwargs)
}
pre_executed_scope = pre_execute(op,
*clients,
scope=scope,
aggcontext=aggcontext,
**kwargs)
new_scope = toolz.merge(scope, pre_executed_scope)
# Short circuit: if pre_execute puts op in scope, then we don't need to
# execute its computable_args
if op in new_scope:
return new_scope
# figure out what arguments we're able to compute on based on the
# expressions inputs. things like expressions, None, and scalar types are
# computable whereas ``list``s are not
computable_args = [arg for arg in op.inputs if is_computable_input(arg)]
# recursively compute each node's arguments until we've changed type
scopes = [
execute_until_in_scope(
arg,
new_scope,
aggcontext=aggcontext,
post_execute_=post_execute_,
clients=clients,
**kwargs,
) if hasattr(arg, 'op') else {
arg: arg
} for arg in computable_args
]
# if we're unable to find data then raise an exception
if not scopes and computable_args:
raise com.UnboundExpressionError(
'Unable to find data for expression:\n{}'.format(repr(expr)))
# there should be exactly one dictionary per computable argument
assert len(computable_args) == len(scopes)
new_scope = toolz.merge(new_scope, *scopes)
# pass our computed arguments to this node's execute_node implementation
data = [
new_scope[arg.op()] if hasattr(arg, 'op') else arg
for arg in computable_args
]
result = execute_node(
op,
*data,
scope=scope,
aggcontext=aggcontext,
clients=clients,
**kwargs,
)
computed = post_execute_(op, result)
return {op: computed}
def execute_bottom_up(
expr, scope, aggcontext=None, post_execute_=None, clients=None, **kwargs
):
"""Execute `expr` bottom-up.
Parameters
----------
expr : ibis.expr.types.Expr
scope : Mapping[ibis.expr.operations.Node, object]
aggcontext : Optional[ibis.pandas.aggcontext.AggregationContext]
kwargs : Dict[str, object]
Returns
-------
result : Mapping[
ibis.expr.operations.Node,
Union[pandas.Series, pandas.DataFrame, scalar_types]
]
A mapping from node to the computed result of that Node
"""
assert post_execute_ is not None, 'post_execute_ is None'
op = expr.op()
# if we're in scope then return the scope, this will then be passed back
# into execute_bottom_up, which will then terminate
if op in scope:
return scope
elif isinstance(op, ops.Literal):
# special case literals to avoid the overhead of dispatching
# execute_node
return {
op: execute_literal(
op, op.value, expr.type(), aggcontext=aggcontext, **kwargs
)
}
# figure out what arguments we're able to compute on based on the
# expressions inputs. things like expressions, None, and scalar types are
# computable whereas ``list``s are not
computable_args = [arg for arg in op.inputs if is_computable_input(arg)]
# recursively compute each node's arguments until we've changed type
scopes = [
execute_bottom_up(
arg,
scope,
aggcontext=aggcontext,
post_execute_=post_execute_,
clients=clients,
**kwargs,
)
if hasattr(arg, 'op')
else {arg: arg}
for arg in computable_args
]
# if we're unable to find data then raise an exception
if not scopes:
raise com.UnboundExpressionError(
'Unable to find data for expression:\n{}'.format(repr(expr))
)
# there should be exactly one dictionary per computable argument
assert len(computable_args) == len(scopes)
new_scope = toolz.merge(scopes)
# pass our computed arguments to this node's execute_node implementation
data = [
new_scope[arg.op()] if hasattr(arg, 'op') else arg
for arg in computable_args
]
result = execute_node(
op,
*data,
scope=scope,
aggcontext=aggcontext,
clients=clients,
**kwargs,
)
computed = post_execute_(op, result)
return {op: computed}
def execute_decimal_log_with_real_base(op, data, base, **kwargs):
return execute_node(op, data, decimal.Decimal(base), **kwargs)
def execute_log_series_gb_series_gb(op, left, right, **kwargs):
result = execute_node(op, left.obj, right.obj, **kwargs)
return result.groupby(left.grouper.groupings)
def execute_unary_op_series_gb(op, operand, **kwargs):
result = execute_node(op, operand.obj, **kwargs)
return result.groupby(operand.grouper.groupings)
def execute_until_in_scope(
expr,
scope: Scope,
timecontext: Optional[TimeContext] = None,
aggcontext=None,
clients=None,
post_execute_=None,
**kwargs,
) -> Scope:
"""Execute until our op is in `scope`.
Parameters
----------
expr : ibis.expr.types.Expr
scope : Scope
timecontext : Optional[TimeContext]
aggcontext : Optional[AggregationContext]
clients : List[ibis.client.Client]
kwargs : Mapping
"""
# these should never be None
assert aggcontext is not None, 'aggcontext is None'
assert clients is not None, 'clients is None'
assert post_execute_ is not None, 'post_execute_ is None'
# base case: our op has been computed (or is a leaf data node), so
# return the corresponding value
op = expr.op()
if scope.get_value(op, timecontext) is not None:
return scope
if isinstance(op, ops.Literal):
# special case literals to avoid the overhead of dispatching
# execute_node
return Scope(
{
op:
execute_literal(
op, op.value, expr.type(), aggcontext=aggcontext, **kwargs)
},
timecontext,
)
# figure out what arguments we're able to compute on based on the
# expressions inputs. things like expressions, None, and scalar types are
# computable whereas ``list``s are not
computable_args = [arg for arg in op.inputs if is_computable_input(arg)]
# pre_executed_states is a list of states with same the length of
# computable_args, these states are passed to each arg
if timecontext:
arg_timecontexts = compute_time_context(
op,
num_args=len(computable_args),
timecontext=timecontext,
clients=clients,
)
else:
arg_timecontexts = [None] * len(computable_args)
pre_executed_scope = pre_execute(
op,
*clients,
scope=scope,
timecontext=timecontext,
aggcontext=aggcontext,
**kwargs,
)
new_scope = scope.merge_scope(pre_executed_scope)
# Short circuit: if pre_execute puts op in scope, then we don't need to
# execute its computable_args
if new_scope.get_value(op, timecontext) is not None:
return new_scope
# recursively compute each node's arguments until we've changed type.
# compute_time_context should return with a list with the same length
# as computable_args, the two lists will be zipping together for
# further execution
if len(arg_timecontexts) != len(computable_args):
raise com.IbisError(
'arg_timecontexts differ with computable_arg in length '
f'for type:\n{type(op).__name__}.')
scopes = [
execute_until_in_scope(
arg,
new_scope,
timecontext=timecontext,
aggcontext=aggcontext,
post_execute_=post_execute_,
clients=clients,
**kwargs,
) if hasattr(arg, 'op') else Scope({arg: arg}, timecontext)
for (arg, timecontext) in zip(computable_args, arg_timecontexts)
]
# if we're unable to find data then raise an exception
if not scopes and computable_args:
raise com.UnboundExpressionError(
'Unable to find data for expression:\n{}'.format(repr(expr)))
# there should be exactly one dictionary per computable argument
assert len(computable_args) == len(scopes)
new_scope = new_scope.merge_scopes(scopes)
# pass our computed arguments to this node's execute_node implementation
data = [
new_scope.get_value(arg.op(), timecontext)
if hasattr(arg, 'op') else arg for arg in computable_args
]
result = execute_node(
op,
*data,
scope=scope,
timecontext=timecontext,
aggcontext=aggcontext,
clients=clients,
**kwargs,
)
computed = post_execute_(op, result, timecontext=timecontext)
return Scope({op: computed}, timecontext)
def execute_decimal_log_with_np_integer_base(op, data, base, **kwargs):
return execute_node(op, data, int(base), **kwargs)
def execute_decimal_log_with_real_base(op, data, base, **kwargs):
return execute_node(op, data, decimal.Decimal(base), **kwargs)
