def xform_opr_to_group_expr(self,
opr: Operator,
root_group_id: int = INVALID_GROUP_ID,
is_root: bool = False,
copy_opr: bool = True) -> GroupExpression:
"""
Generate a group expression from a logical operator.
If the root_group_id is given, then the group_id of the
root logical operator is guaranteed.
"""
# Go through the children first.
child_ids = []
for child_opr in opr.children:
child_id = self.xform_opr_to_group_expr(
opr=child_opr,
root_group_id=root_group_id,
is_root=False,
copy_opr=copy_opr).group_id
child_ids.append(child_id)
# Check if we need copy
if copy_opr:
opr_copy = copy.deepcopy(opr)
opr = opr_copy
if is_root:
expr = GroupExpression(opr=opr,
group_id=root_group_id,
children=child_ids)
expr = self.memo.add_group_expr(expr)
assert root_group_id == INVALID_GROUP_ID or \
expr.group_id == root_group_id
else:
# If not root, we do not care the group_id
expr = GroupExpression(opr=opr,
group_id=INVALID_GROUP_ID,
children=child_ids)
expr = self.memo.add_group_expr(expr)
if root_group_id != INVALID_GROUP_ID and \
expr.group_id == root_group_id:
# This expr has the same group_id with the root one,
# so we need give it a new group.
self.memo._remove_expr(expr)
self.memo._append_expr(expr)
assert root_group_id == INVALID_GROUP_ID or \
expr.group_id != root_group_id
return expr
def execute(self):
implementation_rules = RulesManager().implementation_rules
valid_rules = []
for rule in implementation_rules:
if rule.top_match(self.root_expr.opr):
valid_rules.append(rule)
sorted(valid_rules, key=lambda x: x.promise(), reverse=True)
for rule in valid_rules:
binder = Binder(self.root_expr, rule.pattern,
self.optimizer_context.memo)
for match in iter(binder):
if not rule.check(match, self.optimizer_context):
continue
LoggingManager().log(
'In Optimize physical expression,'
'Rule {} matched for {}'.format(rule, self.root_expr),
LoggingLevel.INFO)
after = rule.apply(match, self.optimizer_context)
new_expr = GroupExpression(after, self.root_expr.group_id,
self.root_expr.children)
# LoggingManager().log('After rewiting {}'.format(new_expr),
# LoggingLevel.INFO)
self.optimizer_context.memo.add_group_expr(new_expr)
# Optimize inputs for this physical expr
self.optimizer_context.task_stack.push(
OptimizeInputs(new_expr, self.optimizer_context))
# Optimize the child groups
for child_id in self.root_expr.children:
self.optimizer_context.task_stack.push(
OptimizeGroup(child_id, self.optimizer_context))
def add_group_expr(self, expr: GroupExpression) -> GroupExpression:
"""
Add an expression into the memo.
If expr.group_id is not set, we will try reuse the exsiting one
(i.e., for rule_explored).
Otherwise, the expr will be considered as a new expression and
inserted into targeted group.
"""
# If not forcing a group id
if expr.group_id == INVALID_GROUP_ID:
group_id = self.get_group_id(expr)
if group_id != INVALID_GROUP_ID:
# we found exsiting one
return self.groups[group_id].logical_exprs[0]
else:
# we append the expr as new one
self._append_expr(expr)
return expr
# If forcing a group id
else:
group_id = expr.group_id
assert group_id < len(self.groups), 'Group Id out of the bound'
assert len(self.groups[group_id].logical_exprs) < 2, \
'Unexpected number of expressions: %s' \
% self.groups[group_id].logical_exprs
if len(self.groups[group_id].logical_exprs) == 1:
old_expr = self.groups[group_id].logical_exprs[0]
self._remove_expr(old_expr)
expr.group_id = INVALID_GROUP_ID
self._insert_expr(expr, group_id)
return expr
def _append_expr(self, expr: GroupExpression):
"""
Append the expr into a new group, and update the group_id of expr
"""
expr.group_id = len(self._groups)
self._groups.append(Group(expr.group_id))
self.groups[expr.group_id].add_expr(expr)
self._group_exprs[expr] = expr.group_id
def test_add_group_expr_cost(self):
grp = Group(0)
prpty = Property(PropertyType(1))
grp_expr1 = GroupExpression(MagicMock(), 1)
grp_expr1.opr.is_logical = lambda: True
grp_expr2 = GroupExpression(MagicMock())
grp_expr2.opr.is_logical = lambda: False
grp.add_expr(grp_expr1)
grp.add_expr_cost(grp_expr1, prpty, 1)
grp.add_expr(grp_expr2)
grp.add_expr_cost(grp_expr2, prpty, 0)
self.assertEqual(grp.get_best_expr(prpty), grp_expr2)
self.assertEqual(grp.get_best_expr_cost(prpty), 0)
def test_add_group_expr_with_unmatched_group_id(self):
grp = Group(0)
grp_expr1 = GroupExpression(MagicMock(), 1)
grp_expr1.opr.is_logical = lambda: True
grp.add_expr(grp_expr1)
self.assertEquals(len(grp.logical_exprs), 0)
self.assertEquals(len(grp.physical_exprs), 0)
def add_expr(self, expr: GroupExpression):
if expr.group_id == INVALID_GROUP_ID:
expr.group_id = self.group_id
if expr.group_id != self.group_id:
LoggingManager().log('Expected group id {}, found {}'.format(
self.group_id, expr.group_id))
return
if expr.opr.is_logical():
self._add_logical_expr(expr)
else:
self._add_physical_expr(expr)
def test_simple_add_group_expr(self):
grp = Group(0)
grp_expr1 = GroupExpression(MagicMock())
grp_expr1.opr.is_logical = lambda: True
grp_expr2 = GroupExpression(MagicMock())
grp_expr2.opr.is_logical = lambda: False
grp_expr3 = GroupExpression(MagicMock(), 0)
grp_expr3.opr.is_logical = lambda: True
grp.add_expr(grp_expr1)
self.assertEquals(len(grp.logical_exprs), 1)
grp.add_expr(grp_expr2)
self.assertEquals(len(grp.logical_exprs), 1)
self.assertEquals(len(grp.physical_exprs), 1)
grp.add_expr(grp_expr3)
self.assertEquals(len(grp.logical_exprs), 2)
self.assertEquals(len(grp.physical_exprs), 1)
def test_opr_to_group_expr(self, child1_opr, child2_opr, child3_opr,
root_opr):
child1_opr.children = []
child1_expr = GroupExpression(child1_opr, INVALID_GROUP_ID, [])
child2_opr.children = []
child2_expr = GroupExpression(child2_opr, INVALID_GROUP_ID, [])
child3_opr.children = []
child3_expr = GroupExpression(child3_opr, INVALID_GROUP_ID, [])
root_expr = GroupExpression(root_opr, INVALID_GROUP_ID, [0, 1, 2])
root_opr.children = [child1_opr, child2_opr, child3_opr]
opt_ctxt = OptimizerContext()
opt_ctxt.xform_opr_to_group_expr(root_opr,
is_root=True,
copy_opr=False)
self.assertEqual(len(opt_ctxt.memo.group_exprs), 4)
self.assertEqual(opt_ctxt.memo.get_group_id(child3_expr), 2)
self.assertEqual(opt_ctxt.memo.get_group_id(child2_expr), 1)
self.assertEqual(opt_ctxt.memo.get_group_id(child1_expr), 0)
self.assertEqual(opt_ctxt.memo.get_group_id(root_expr), 3)
def _remove_expr(self, expr: GroupExpression):
"""
Remove the expr from the memo, and update the group_id of expr
to be INVALID_GROUP_ID.
"""
group_id = self.get_group_id(expr)
assert group_id == expr.group_id, \
'Inconsistent memo found when removing expression: %s' % expr
if group_id == INVALID_GROUP_ID:
return
del self._group_exprs[expr]
self.groups[group_id].clear_grp_exprs()
expr.group_id = INVALID_GROUP_ID
def _insert_expr(self, expr: GroupExpression, group_id: int):
"""
Insert a group expressoin into a particular group, update the
group_id of expr
"""
assert expr.group_id == INVALID_GROUP_ID, \
'Expression: %s is already in the memo' % expr
assert group_id < len(self.groups), 'Group Id out of the bound'
group = self.groups[group_id]
assert len(group.logical_exprs) == 0, \
'Expression exists in the targeted inserted group. Details: %s' \
% group.logical_exprs
self.groups[group_id].add_expr(expr)
self._group_exprs[expr] = group_id
expr.group_id = group_id