def _evolve_meta_plan(self,
plan: MetaQueryPlan,
depth: int,
nlabel_to_unit_plan: Dict[int, Set[UnitMetaPlan]]):
# Replace an input node of plan with the output of a unit meta-plan (contained in nlabel_to_meta_plan).
# Thus we extend an existing plan with the output of exactly one component, thus increasing the program
# depth by exactly one.
plan_transformation: Transformation = plan.canonical_transformations[depth - 1]
all_input_nodes: Set[Node] = set(plan_transformation.get_input_nodes())
for inp_node in all_input_nodes:
# Even if it is a placeholder node, it can only be extended via the empty transform. This makes sense
# as no matter what the extension is, it will never "influence" the final output, as there is no path
# between a placeholder node and the output node. This helps reduce the size of the collection of
# meta query-plans by a large margin.
remaining_inputs = all_input_nodes - {inp_node}
for extender in nlabel_to_unit_plan[int(inp_node.label)]:
# We can map the other inputs to the inputs of the extender plan. They can also be distinct
# inputs of their own. The total number of inputs should, however, be less than max_inputs.
# For the remaining inputs, if they are a placeholder node, they can be mapped to *any* of the
# input nodes of the extender plan, regardless of the label.
nlabel_to_inp_node: Dict[int, Set[Node]] = collections.defaultdict(set)
extender_inputs: Set[Node] = set(extender.transformation.get_input_nodes())
# Guaranteed to be a single output node by construction.
extender_output: Node = next(extender.transformation.get_output_nodes())
for inp in extender_inputs:
nlabel_to_inp_node[inp.label].add(inp)
nlabel_to_inp_node[extender_output.label].add(extender_output)
mapping_possibilities: Dict[Node, Set[Node]] = {inp_node: {extender_output}}
for inp in remaining_inputs:
if inp.label == PLACEHOLDER_LABEL:
mapping_possibilities[inp] = extender_inputs | {extender_output, inp}
else:
mapping_possibilities[inp] = nlabel_to_inp_node[inp.label] | {inp}
node_list = list(all_input_nodes)
for border_node_mapping in itertools.product(*[mapping_possibilities[n] for n in node_list]):
border_node_mapping: Dict[Node, Node] = dict(zip(node_list, border_node_mapping))
border_mapping = GraphMapping(m_ent={k.entity: v.entity for k, v in border_node_mapping.items()},
m_node=border_node_mapping.copy())
# Create a deepcopy of the extender for safety
copied_extender, copy_mapping = extender.deepcopy()
copied_extender_inputs: Set[Node] = set(copied_extender.transformation.get_input_nodes())
copied_extender_output: Node = next(copied_extender.transformation.get_output_nodes())
border_mapping = border_mapping.apply_mapping(copy_mapping, only_values=True)
assert border_mapping.m_node != border_node_mapping
# The new inputs are the inputs of extender, plus the nodes of the current plan
# which were not bound to any of the inputs of extender.
# We also decide the order of the nodes/entities right now.
new_input_nodes: List[Node] = []
for inp in plan_transformation.get_input_nodes():
mapped = border_mapping.m_node[inp]
if mapped is inp:
new_input_nodes.append(inp)
elif mapped is copied_extender_output:
new_input_nodes.extend(i for i in copied_extender.transformation.get_input_nodes()
if i not in new_input_nodes)
elif mapped not in new_input_nodes:
assert mapped in copied_extender_inputs
new_input_nodes.append(mapped)
new_input_entities = [n.entity for n in new_input_nodes]
# Every entity is associated with one node so the following should hold true.
assert len(new_input_entities) == len(set(new_input_entities))
if len(new_input_entities) > self._config.max_inputs:
continue
new_output_entity = plan_transformation.get_output_entity()
# Obtain the transformation by establishing common edges between the node pairs in
# border_node_mapping, taking the transitive closure w.r.t equality, and finally the
# induced subgraph by removing the input nodes of the current plan.
joint_graph = Graph.from_graph(copied_extender.transformation)
joint_graph.merge(plan_transformation)
final_border_mapping = GraphMapping()
for k, v in border_mapping.m_node.items():
if k is not v:
final_border_mapping.m_node[k] = v
final_border_mapping.m_ent[k.entity] = v.entity
for edge in plan_transformation.iter_edges(src=k):
joint_graph.add_edge(Edge(v, edge.dst, edge.label))
for edge in plan_transformation.iter_edges(dst=k):
joint_graph.add_edge(Edge(edge.src, v, edge.label))
join_nodes: Set[Node] = set(all_input_nodes)
join_nodes.difference_update(new_input_nodes)
join_nodes.add(copied_extender_output)
self._domain.perform_transitive_closure(joint_graph, join_nodes=join_nodes)
keep_nodes = set(joint_graph.iter_nodes())
keep_nodes.difference_update(join_nodes)
new_transformation_subgraph = joint_graph.induced_subgraph(keep_nodes=keep_nodes)
new_transformation = Transformation.build_from_graph(new_transformation_subgraph,
input_entities=new_input_entities,
output_entity=new_output_entity)
# Record the transformation and how it was obtained.
if new_transformation not in self._meta_plans:
# The transformation was never seen before.
blueprint = collections.defaultdict(lambda: collections.defaultdict(list))
meta_plan = MetaQueryPlan(transformation=new_transformation.deepcopy()[0],
blueprint=blueprint)
self._meta_plans[new_transformation] = meta_plan
else:
meta_plan = self._meta_plans[new_transformation]
if depth not in meta_plan.canonical_transformations:
copy, mapping = new_transformation.deepcopy()
meta_plan.canonical_transformations[depth] = copy
# mapping = mapping.slice(nodes=set(copied_extender.transformation.iter_nodes()))
mapping = mapping.reverse()
else:
canonical = meta_plan.canonical_transformations[depth]
mapping = next(new_transformation.get_subgraph_mappings(canonical))
# mapping = mapping.slice(nodes=set(copied_extender.transformation.iter_nodes()))
mapping = mapping.reverse()
bp_item = MetaQueryPlan.BlueprintItem(depth=depth,
unit=copied_extender,
canonical_mapping=mapping,
sub_plan=plan,
border_mapping=final_border_mapping)
for c in copied_extender.component_entries:
meta_plan.blueprint[depth][c].append(bp_item)
def build(cls, domain: SynthesisDomain, config: EngineConfig, witness_set: WitnessSet) -> 'QueryPlanner':
q_planner = QueryPlanner(domain=domain, config=config)
logger_color = logger.opt(colors=True)
# ---------------------------------------------------------------------------------------------------------- #
# Stage 1 : Build unit plans
# ---------------------------------------------------------------------------------------------------------- #
logger.debug("Extracting meta query-plans of length 1 from individual components...")
for component_name in domain.get_available_components():
for witness_entry in witness_set.entries[component_name]:
q_planner._extract_unit_plans(component_name, witness_entry)
# Log some information for post-mortem analysis if necessary
# Total plans found.
logger_color.debug(f"Found <green>{len(q_planner._unit_plans)}</green> unit plans in total.")
# Plans found per component.
components_to_units: Dict[str, List[UnitMetaPlan]] = collections.defaultdict(list)
for unit in q_planner._unit_plans.values():
for c in unit.component_entries:
components_to_units[c].append(unit)
with logutils.temporary_add(f"{config.path}/logs/query_planner/unit_plans.log",
level="TRACE",
only_sink=True) as logger_:
logger_ = logger_.opt(raw=True)
for component_name, units in components_to_units.items():
logger_color.debug(f"Found <green>{len(units)}</green> unit plans from "
f"<blue>{component_name}</blue>.")
logger_.opt(colors=True).debug(f"Found <green>{len(units)}</green> unit plans from "
f"<blue>{component_name}</blue>.")
for unit in units:
logger_.trace(f"Component: {component_name}\n")
logger_.trace("-----------------\n")
logger_.trace("Transformation\n")
logger_.trace("-----------------\n")
logger_.trace(unit.transformation.to_str(domain))
logger_.trace("\n-----------------\n")
logger_.trace("Argument Mappings\n")
logger_.trace("-----------------\n")
logger_.trace(unit.component_entries[component_name].argument_mappings)
logger_.trace("\n========xxx========\n\n")
# ---------------------------------------------------------------------------------------------------------- #
# Stage 2 : Strengthen Unit Plans
# ---------------------------------------------------------------------------------------------------------- #
logger_color.debug("Strengthening Unit Plans...")
for unit_plan in debug_iter(list(q_planner._unit_plans.values()), desc='Strengthening Unit Plans'):
query: Transformation = unit_plan.transformation
for component_name in unit_plan.component_entries.keys():
# The witness set examples are guaranteed to have a placeholder node for each entity,
# so the use of placeholder nodes in transformations should not cause issues.
if unit_plan.empty:
strengthened, mapping = query.deepcopy()
strengthened = Graph.from_graph(strengthened)
else:
examples: List[Transformation] = witness_set.get_transformations(component_name)
strengthened, mapping = query.get_greatest_common_universal_supergraph(examples)
inp_entities = set(query.get_input_entities())
m_reverse = mapping.reverse()
keep_nodes = [n for n in strengthened.iter_nodes()
if m_reverse.m_ent.get(n.entity, None) in inp_entities]
strengthened = strengthened.induced_subgraph(keep_nodes=keep_nodes)
unit_plan.strengthenings[component_name] = (strengthened, mapping)
logger_color.debug(f"Strengthened <green>{len(q_planner._unit_plans)}</green> unit plans.")
# ---------------------------------------------------------------------------------------------------------- #
# Stage 3 : Combining unit plans to obtain query plans upto max-depth
# Note that these are not explicitly constructed. Rather, a recursive formulation is established
# to construct the query plans quickly during test-time.
# ---------------------------------------------------------------------------------------------------------- #
# Initialize the meta-plans from these unit plans
logger.debug("Initializing meta-plans from unit-plans...")
for transformation, unit_plan in q_planner._unit_plans.items():
q_planner._meta_plans[transformation] = MetaQueryPlan.initialize_from_unit_plan(unit_plan.deepcopy()[0])
logger_color.debug(f"Evolving meta query-plans upto a maximum length of "
f"<blue>{config.max_length}</blue>")
# Setup a mapping from the label of the output node of transformations
# to the meta plan for that transformation. This helps in quickly finding
# appropriate unit plans to extend a plan with.
nlabel_to_unit_plan: Dict[int, Set[UnitMetaPlan]] = collections.defaultdict(set)
for transformation, unit_plan in q_planner._unit_plans.items():
# Guaranteed to be a single node with that entity.
output_node = next(transformation.get_output_nodes())
nlabel_to_unit_plan[output_node.label].add(unit_plan)
# At every depth, the worklist will be the set of query plans with an entry for depth-1 in the blueprint.
worklist: Set[MetaQueryPlan] = set(q_planner._meta_plans.values())
for depth in range(2, config.max_length + 1):
for unit in worklist:
q_planner._evolve_meta_plan(unit, depth, nlabel_to_unit_plan)
worklist = {plan for plan in q_planner._meta_plans.values() if depth in plan.blueprint}
logger_color.debug(f"Found <green>{len(worklist)}</green> transformations in total "
f"at depth <blue>{depth}</blue>.")
if len(worklist) == 0:
break
logger_color.debug(f"Found <green>{len(q_planner._meta_plans)}</green> transformations and meta "
f"query plans in total with <blue>max_depth={config.max_length}</blue>.")
# TODO : Log the transformations to a file.
# ---------------------------------------------------------------------------------------------------------- #
# Stage 4 : Setup auxiliary data-structures
# ---------------------------------------------------------------------------------------------------------- #
# Flattened blue-print items enable access to all the items agnostic of the component name.
for plan in q_planner._meta_plans.values():
plan.blueprint_items = collections.defaultdict(set)
for depth, bp_dict in plan.blueprint.items():
for items_list in bp_dict.values():
plan.blueprint_items[depth].update(items_list)
# Sequence tries help in quickly computing candidate sequences given a synthesis problem.
logger_color.debug("Constructing Sequence Tries...")
q_planner._compute_sequence_tries()
logger_color.debug(f"Sequence Tries constructed for every depth.")
return q_planner
