def _rewrite_reduction_filter(self, expr):
# Find the table that this reduction references.
# TODO: what about reductions that reference a join that isn't visible
# at this level? Means we probably have the wrong design, but will have
# to revisit when it becomes a problem.
aggregation, _ = L.reduction_to_aggregation(expr, default_name='tmp')
return aggregation.to_array()
def _adapt_expr(expr):
# Non-table expressions need to be adapted to some well-formed table
# expression, along with a way to adapt the results to the desired
# arity (whether array-like or scalar, for example)
#
# Canonical case is scalar values or arrays produced by some reductions
# (simple reductions, or distinct, say)
if isinstance(expr, ir.Table):
return expr, toolz.identity
if isinstance(expr, ir.Scalar):
if not expr.has_name():
expr = expr.name('tmp')
if L.is_scalar_reduction(expr):
table_expr = L.reduction_to_aggregation(expr)
return table_expr, _get_scalar(expr.get_name())
else:
return expr, _get_scalar(expr.get_name())
elif isinstance(expr, ir.Analytic):
return expr.to_aggregation(), toolz.identity
elif isinstance(expr, ir.Column):
op = expr.op()
if isinstance(op, ops.TableColumn):
table_expr = op.table[[op.name]]
result_handler = _get_column(op.name)
else:
if not expr.has_name():
expr = expr.name('tmp')
table_expr = expr.to_projection()
result_handler = _get_column(expr.get_name())
return table_expr, result_handler
else:
raise com.TranslationError(
f'Do not know how to execute: {type(expr)}')
def _adapt_expr(expr):
# Non-table expressions need to be adapted to some well-formed table
# expression, along with a way to adapt the results to the desired
# arity (whether array-like or scalar, for example)
#
# Canonical case is scalar values or arrays produced by some reductions
# (simple reductions, or distinct, say)
def as_is(x):
return x
if isinstance(expr, ir.TableExpr):
return expr, as_is
def _get_scalar(field):
def scalar_handler(results):
return results[field][0]
return scalar_handler
if isinstance(expr, ir.ScalarExpr):
if L.is_scalar_reduce(expr):
table_expr, name = L.reduction_to_aggregation(
expr, default_name='tmp')
return table_expr, _get_scalar(name)
else:
base_table = ir.find_base_table(expr)
if base_table is None:
# expr with no table refs
return expr.name('tmp'), _get_scalar('tmp')
else:
raise NotImplementedError(expr._repr())
elif isinstance(expr, ir.AnalyticExpr):
return expr.to_aggregation(), as_is
elif isinstance(expr, ir.ExprList):
exprs = expr.exprs()
is_aggregation = True
any_aggregation = False
for x in exprs:
if not L.is_scalar_reduce(x):
is_aggregation = False
else:
any_aggregation = True
if is_aggregation:
table = ir.find_base_table(exprs[0])
return table.aggregate(exprs), as_is
elif not any_aggregation:
return expr, as_is
else:
raise NotImplementedError(expr._repr())
elif isinstance(expr, ir.ArrayExpr):
op = expr.op()
def _get_column(name):
def column_handler(results):
return results[name]
return column_handler
if isinstance(op, ops.TableColumn):
table_expr = op.table[[op.name]]
result_handler = _get_column(op.name)
else:
# Something more complicated.
base_table = L.find_source_table(expr)
if isinstance(op, ops.DistinctArray):
expr = op.arg
try:
name = op.arg.get_name()
except Exception:
name = 'tmp'
table_expr = (base_table.projection([expr.name(name)])
.distinct())
result_handler = _get_column(name)
else:
table_expr = base_table.projection([expr.name('tmp')])
result_handler = _get_column('tmp')
return table_expr, result_handler
else:
raise com.TranslationError('Do not know how to execute: {0}'
.format(type(expr)))
def _adapt_expr(expr):
# Non-table expressions need to be adapted to some well-formed table
# expression, along with a way to adapt the results to the desired
# arity (whether array-like or scalar, for example)
#
# Canonical case is scalar values or arrays produced by some reductions
# (simple reductions, or distinct, say)
if isinstance(expr, ir.TableExpr):
return expr, toolz.identity
def _get_scalar(field):
def scalar_handler(results):
return results[field][0]
return scalar_handler
if isinstance(expr, ir.ScalarExpr):
if L.is_scalar_reduction(expr):
table_expr, name = L.reduction_to_aggregation(
expr, default_name='tmp'
)
return table_expr, _get_scalar(name)
else:
base_table = ir.find_base_table(expr)
if base_table is None:
# exprs with no table refs
# TODO(phillipc): remove ScalarParameter hack
if isinstance(expr.op(), ops.ScalarParameter):
name = expr.get_name()
assert (
name is not None
), f'scalar parameter {expr} has no name'
return expr, _get_scalar(name)
return expr.name('tmp'), _get_scalar('tmp')
raise NotImplementedError(repr(expr))
elif isinstance(expr, ir.AnalyticExpr):
return expr.to_aggregation(), toolz.identity
elif isinstance(expr, ir.ColumnExpr):
op = expr.op()
def _get_column(name):
def column_handler(results):
return results[name]
return column_handler
if isinstance(op, ops.TableColumn):
table_expr = op.table[[op.name]]
result_handler = _get_column(op.name)
else:
# Something more complicated.
base_table = L.find_source_table(expr)
if isinstance(op, ops.DistinctColumn):
expr = op.arg
try:
name = op.arg.get_name()
except Exception:
name = 'tmp'
table_expr = base_table.projection(
[expr.name(name)]
).distinct()
result_handler = _get_column(name)
else:
table_expr = base_table.projection([expr.name('tmp')])
result_handler = _get_column('tmp')
return table_expr, result_handler
else:
raise com.TranslationError(
f'Do not know how to execute: {type(expr)}'
)