def consume(self, t, src, state):
code = ""
assignment_template = """
%(dst_name)s.set(%(dst_fieldnum)s, %(src_expr_compiled)s);
"""
dst_name = self.newtuple.name
dst_type_name = self.newtuple.getTupleTypename()
# declaration of tuple instance
code += """%(dst_type_name)s %(dst_name)s;""" % locals()
for dst_fieldnum, src_label_expr in enumerate(self.emitters):
src_label, src_expr = src_label_expr
# make sure to resolve attribute positions using input schema
src_expr_unnamed = expression.ensure_unnamed(src_expr, self.input)
# tag the attributes with references
# TODO: use an immutable approach instead
# (ie an expression Visitor for compiling)
[_ for _ in src_expr_unnamed.postorder(getTaggingFunc(t))]
src_expr_compiled, expr_decls, expr_inits = \
self.language().compile_expression(src_expr_unnamed)
state.addInitializers(expr_inits)
state.addDeclarations(expr_decls)
code += assignment_template % locals()
innercode = self.parent().consume(self.newtuple, self, state)
code += innercode
return code
def consume(self, t, src, state):
code = ""
assignment_template = _cgenv.get_template('assignment.cpp')
dst_name = self.newtuple.name
dst_type_name = self.newtuple.getTupleTypename()
# declaration of tuple instance
code += _cgenv.get_template('tuple_declaration.cpp').render(locals())
for dst_fieldnum, src_label_expr in enumerate(self.emitters):
dst_set_func = self.newtuple.set_func_code(dst_fieldnum)
src_label, src_expr = src_label_expr
# make sure to resolve attribute positions using input schema
src_expr_unnamed = expression.ensure_unnamed(src_expr, self.input)
src_expr_compiled, expr_decls, expr_inits = \
self.language().compile_expression(
src_expr_unnamed, tupleref=t)
state.addInitializers(expr_inits)
state.addDeclarations(expr_decls)
code += assignment_template.render(locals())
innercode = self.parent().consume(self.newtuple, self, state)
code += innercode
return code
def _compile_condition(self, t, state):
condition_as_unnamed = expression.ensure_unnamed(self.condition, self)
# compile the predicate into code
conditioncode, cond_decls, cond_inits = \
self.language().compile_expression(
condition_as_unnamed, tupleref=t)
state.addInitializers(cond_inits)
state.addDeclarations(cond_decls)
return conditioncode
def consume(self, t, src, state):
basic_select_template = _cgenv.get_template('select.cpp')
condition_as_unnamed = expression.ensure_unnamed(self.condition, self)
# compile the predicate into code
conditioncode, cond_decls, cond_inits = \
self.language().compile_expression(
condition_as_unnamed, tupleref=t)
state.addInitializers(cond_inits)
state.addDeclarations(cond_decls)
inner_code_compiled = self.parent().consume(t, self, state)
code = basic_select_template.render(locals())
return code
def _apply_statements(self, t, state):
assignment_template = _cgenv.get_template('assignment.cpp')
dst_name = self.newtuple.name
dst_type_name = self.newtuple.getTupleTypename()
code = ""
for dst_fieldnum, src_label_expr in enumerate(self.emitters):
dst_set_func = self.newtuple.set_func_code(dst_fieldnum)
src_label, src_expr = src_label_expr
# make sure to resolve attribute positions using input schema
src_expr_unnamed = expression.ensure_unnamed(src_expr, self.input)
src_expr_compiled, expr_decls, expr_inits = \
self.language().compile_expression(
src_expr_unnamed, tupleref=t)
state.addInitializers(expr_inits)
state.addDeclarations(expr_decls)
code += assignment_template.render(locals())
return code
def consume(self, t, src, state):
basic_select_template = """if (%(conditioncode)s) {
%(inner_code_compiled)s
}
"""
condition_as_unnamed = expression.ensure_unnamed(self.condition, self)
# tag the attributes with references
# TODO: use an immutable approach instead
# (ie an expression Visitor for compiling)
[_ for _ in condition_as_unnamed.postorder(getTaggingFunc(t))]
# compile the predicate into code
conditioncode, cond_decls, cond_inits = \
self.language().compile_expression(condition_as_unnamed)
state.addInitializers(cond_inits)
state.addDeclarations(cond_decls)
inner_code_compiled = self.parent().consume(t, self, state)
code = basic_select_template % locals()
return code
def descend_tree(op, cond):
"""Recursively push a selection condition down a tree of operators.
:param op: The root of an operator tree
:type op: raco.algebra.Operator
:param cond: The selection condition
:type cond: raco.expression.expression
:return: A (possibly modified) operator.
"""
if isinstance(op, algebra.Select):
# Keep pushing; selects are commutative
op.input = PushSelects.descend_tree(op.input, cond)
return op
elif isinstance(op, algebra.CompositeBinaryOperator):
# Joins and cross-products; consider conversion to an equijoin
# Expressions containing random do not commute across joins
has_random = any(isinstance(e, RANDOM) for e in cond.walk())
if not has_random:
left_len = len(op.left.scheme())
accessed = accessed_columns(cond)
in_left = [col < left_len for col in accessed]
if all(in_left):
# Push the select into the left sub-tree.
op.left = PushSelects.descend_tree(op.left, cond)
return op
elif not any(in_left):
# Push into right subtree; rebase column indexes
expression.rebase_expr(cond, left_len)
op.right = PushSelects.descend_tree(op.right, cond)
return op
else:
# Selection includes both children; attempt to create an
# equijoin condition
cols = PushSelects.is_column_equality_comparison(cond)
if cols:
return op.add_equijoin_condition(cols[0], cols[1])
elif isinstance(op, algebra.Apply):
# Convert accessed to a list from a set to ensure consistent order
accessed = list(accessed_columns(cond))
accessed_emits = [op.emitters[i][1] for i in accessed]
if all(isinstance(e, expression.AttributeRef)
for e in accessed_emits):
unnamed_emits = expression.ensure_unnamed(
accessed_emits, op.input)
# This condition only touches columns that are copied verbatim
# from the child, so we can push it.
index_map = {a: e.position
for (a, e) in zip(accessed, unnamed_emits)}
expression.reindex_expr(cond, index_map)
op.input = PushSelects.descend_tree(op.input, cond)
return op
elif isinstance(op, algebra.GroupBy):
# Convert accessed to a list from a set to ensure consistent order
accessed = list(accessed_columns(cond))
if all((a < len(op.grouping_list)) for a in accessed):
accessed_grps = [op.grouping_list[a] for a in accessed]
# This condition only touches columns that are copied verbatim
# from the child (grouping keys), so we can push it.
assert all(isinstance(e, expression.AttributeRef)
for e in op.grouping_list)
unnamed_grps = expression.ensure_unnamed(accessed_grps,
op.input)
index_map = {a: e.position
for (a, e) in zip(accessed, unnamed_grps)}
expression.reindex_expr(cond, index_map)
op.input = PushSelects.descend_tree(op.input, cond)
return op
# Can't push any more: instantiate the selection
new_op = algebra.Select(cond, op)
new_op.has_been_pushed = True
return new_op
def get_unnamed_update_exprs(self):
"""Get the update list for this GroupBy after ensuring that all
attribute references are UnnamedAttributeRefs."""
ups = [expr for _, expr in self.updaters]
return expression.ensure_unnamed(ups, self.input)
def get_unnamed_aggregate_list(self):
"""Get the aggregate list for this GroupBy after ensuring that all
attribute references are UnnamedAttributeRefs."""
return expression.ensure_unnamed(self.aggregate_list, self.input)
def get_unnamed_column_list(self):
"""Get the column list for this Project after ensuring that all
attribute references are UnnamedAttributeRefs."""
return expression.ensure_unnamed(self.columnlist, self.input)
def get_unnamed_condition(self):
"""Get the filter condition for this Select after ensuring that all
attribute references are UnnamedAttributeRefs."""
return expression.ensure_unnamed(self.condition, self.input)
def get_unnamed_emit_exprs(self):
"""Get the emit expressions for this Apply after ensuring that all
attribute references are UnnamedAttributeRefs."""
emits = [e[1] for e in self.emitters]
return expression.ensure_unnamed(emits, self.input)
def descend_tree(op, cond):
"""Recursively push a selection condition down a tree of operators.
:param op: The root of an operator tree
:type op: raco.algebra.Operator
:param cond: The selection condition
:type cond: raco.expression.expression
:return: A (possibly modified) operator.
"""
if isinstance(op, algebra.Select):
# Keep pushing; selects are commutative
op.input = PushSelects.descend_tree(op.input, cond)
return op
elif isinstance(op, algebra.CompositeBinaryOperator):
# Joins and cross-products; consider conversion to an equijoin
# Expressions containing random do not commute across joins
has_random = any(isinstance(e, RANDOM) for e in cond.walk())
if not has_random:
left_len = len(op.left.scheme())
accessed = accessed_columns(cond)
in_left = [col < left_len for col in accessed]
if all(in_left):
# Push the select into the left sub-tree.
op.left = PushSelects.descend_tree(op.left, cond)
return op
elif not any(in_left):
# Push into right subtree; rebase column indexes
expression.rebase_expr(cond, left_len)
op.right = PushSelects.descend_tree(op.right, cond)
return op
else:
# Selection includes both children; attempt to create an
# equijoin condition
cols = PushSelects.is_column_equality_comparison(cond)
if cols:
return op.add_equijoin_condition(cols[0], cols[1])
elif isinstance(op, algebra.Apply):
# Convert accessed to a list from a set to ensure consistent order
accessed = list(accessed_columns(cond))
accessed_emits = [op.emitters[i][1] for i in accessed]
if all(isinstance(e, expression.AttributeRef)
for e in accessed_emits):
unnamed_emits = expression.ensure_unnamed(
accessed_emits, op.input)
# This condition only touches columns that are copied verbatim
# from the child, so we can push it.
index_map = {a: e.position
for (a, e) in zip(accessed, unnamed_emits)}
expression.reindex_expr(cond, index_map)
op.input = PushSelects.descend_tree(op.input, cond)
return op
elif isinstance(op, algebra.GroupBy):
# Convert accessed to a list from a set to ensure consistent order
accessed = list(accessed_columns(cond))
if all((a < len(op.grouping_list)) for a in accessed):
accessed_grps = [op.grouping_list[a] for a in accessed]
# This condition only touches columns that are copied verbatim
# from the child (grouping keys), so we can push it.
assert all(isinstance(e, expression.AttributeRef)
for e in op.grouping_list)
unnamed_grps = expression.ensure_unnamed(accessed_grps,
op.input)
index_map = {a: e.position
for (a, e) in zip(accessed, unnamed_grps)}
expression.reindex_expr(cond, index_map)
op.input = PushSelects.descend_tree(op.input, cond)
return op
# Can't push any more: instantiate the selection
new_op = algebra.Select(cond, op)
new_op.has_been_pushed = True
return new_op
