def _evolve_meta_plan(self, plan: MetaQueryPlan, depth: int, nlabel_to_unit_plan: Dict[int, Set[UnitMetaPlan]]): # Replace an input node of plan with the output of a unit meta-plan (contained in nlabel_to_meta_plan). # Thus we extend an existing plan with the output of exactly one component, thus increasing the program # depth by exactly one. plan_transformation: Transformation = plan.canonical_transformations[depth - 1] all_input_nodes: Set[Node] = set(plan_transformation.get_input_nodes()) for inp_node in all_input_nodes: # Even if it is a placeholder node, it can only be extended via the empty transform. This makes sense # as no matter what the extension is, it will never "influence" the final output, as there is no path # between a placeholder node and the output node. This helps reduce the size of the collection of # meta query-plans by a large margin. remaining_inputs = all_input_nodes - {inp_node} for extender in nlabel_to_unit_plan[int(inp_node.label)]: # We can map the other inputs to the inputs of the extender plan. They can also be distinct # inputs of their own. The total number of inputs should, however, be less than max_inputs. # For the remaining inputs, if they are a placeholder node, they can be mapped to *any* of the # input nodes of the extender plan, regardless of the label. nlabel_to_inp_node: Dict[int, Set[Node]] = collections.defaultdict(set) extender_inputs: Set[Node] = set(extender.transformation.get_input_nodes()) # Guaranteed to be a single output node by construction. extender_output: Node = next(extender.transformation.get_output_nodes()) for inp in extender_inputs: nlabel_to_inp_node[inp.label].add(inp) nlabel_to_inp_node[extender_output.label].add(extender_output) mapping_possibilities: Dict[Node, Set[Node]] = {inp_node: {extender_output}} for inp in remaining_inputs: if inp.label == PLACEHOLDER_LABEL: mapping_possibilities[inp] = extender_inputs | {extender_output, inp} else: mapping_possibilities[inp] = nlabel_to_inp_node[inp.label] | {inp} node_list = list(all_input_nodes) for border_node_mapping in itertools.product(*[mapping_possibilities[n] for n in node_list]): border_node_mapping: Dict[Node, Node] = dict(zip(node_list, border_node_mapping)) border_mapping = GraphMapping(m_ent={k.entity: v.entity for k, v in border_node_mapping.items()}, m_node=border_node_mapping.copy()) # Create a deepcopy of the extender for safety copied_extender, copy_mapping = extender.deepcopy() copied_extender_inputs: Set[Node] = set(copied_extender.transformation.get_input_nodes()) copied_extender_output: Node = next(copied_extender.transformation.get_output_nodes()) border_mapping = border_mapping.apply_mapping(copy_mapping, only_values=True) assert border_mapping.m_node != border_node_mapping # The new inputs are the inputs of extender, plus the nodes of the current plan # which were not bound to any of the inputs of extender. # We also decide the order of the nodes/entities right now. new_input_nodes: List[Node] = [] for inp in plan_transformation.get_input_nodes(): mapped = border_mapping.m_node[inp] if mapped is inp: new_input_nodes.append(inp) elif mapped is copied_extender_output: new_input_nodes.extend(i for i in copied_extender.transformation.get_input_nodes() if i not in new_input_nodes) elif mapped not in new_input_nodes: assert mapped in copied_extender_inputs new_input_nodes.append(mapped) new_input_entities = [n.entity for n in new_input_nodes] # Every entity is associated with one node so the following should hold true. assert len(new_input_entities) == len(set(new_input_entities)) if len(new_input_entities) > self._config.max_inputs: continue new_output_entity = plan_transformation.get_output_entity() # Obtain the transformation by establishing common edges between the node pairs in # border_node_mapping, taking the transitive closure w.r.t equality, and finally the # induced subgraph by removing the input nodes of the current plan. joint_graph = Graph.from_graph(copied_extender.transformation) joint_graph.merge(plan_transformation) final_border_mapping = GraphMapping() for k, v in border_mapping.m_node.items(): if k is not v: final_border_mapping.m_node[k] = v final_border_mapping.m_ent[k.entity] = v.entity for edge in plan_transformation.iter_edges(src=k): joint_graph.add_edge(Edge(v, edge.dst, edge.label)) for edge in plan_transformation.iter_edges(dst=k): joint_graph.add_edge(Edge(edge.src, v, edge.label)) join_nodes: Set[Node] = set(all_input_nodes) join_nodes.difference_update(new_input_nodes) join_nodes.add(copied_extender_output) self._domain.perform_transitive_closure(joint_graph, join_nodes=join_nodes) keep_nodes = set(joint_graph.iter_nodes()) keep_nodes.difference_update(join_nodes) new_transformation_subgraph = joint_graph.induced_subgraph(keep_nodes=keep_nodes) new_transformation = Transformation.build_from_graph(new_transformation_subgraph, input_entities=new_input_entities, output_entity=new_output_entity) # Record the transformation and how it was obtained. if new_transformation not in self._meta_plans: # The transformation was never seen before. blueprint = collections.defaultdict(lambda: collections.defaultdict(list)) meta_plan = MetaQueryPlan(transformation=new_transformation.deepcopy()[0], blueprint=blueprint) self._meta_plans[new_transformation] = meta_plan else: meta_plan = self._meta_plans[new_transformation] if depth not in meta_plan.canonical_transformations: copy, mapping = new_transformation.deepcopy() meta_plan.canonical_transformations[depth] = copy # mapping = mapping.slice(nodes=set(copied_extender.transformation.iter_nodes())) mapping = mapping.reverse() else: canonical = meta_plan.canonical_transformations[depth] mapping = next(new_transformation.get_subgraph_mappings(canonical)) # mapping = mapping.slice(nodes=set(copied_extender.transformation.iter_nodes())) mapping = mapping.reverse() bp_item = MetaQueryPlan.BlueprintItem(depth=depth, unit=copied_extender, canonical_mapping=mapping, sub_plan=plan, border_mapping=final_border_mapping) for c in copied_extender.component_entries: meta_plan.blueprint[depth][c].append(bp_item)
def build(cls, domain: SynthesisDomain, config: EngineConfig, witness_set: WitnessSet) -> 'QueryPlanner': q_planner = QueryPlanner(domain=domain, config=config) logger_color = logger.opt(colors=True) # ---------------------------------------------------------------------------------------------------------- # # Stage 1 : Build unit plans # ---------------------------------------------------------------------------------------------------------- # logger.debug("Extracting meta query-plans of length 1 from individual components...") for component_name in domain.get_available_components(): for witness_entry in witness_set.entries[component_name]: q_planner._extract_unit_plans(component_name, witness_entry) # Log some information for post-mortem analysis if necessary # Total plans found. logger_color.debug(f"Found <green>{len(q_planner._unit_plans)}</green> unit plans in total.") # Plans found per component. components_to_units: Dict[str, List[UnitMetaPlan]] = collections.defaultdict(list) for unit in q_planner._unit_plans.values(): for c in unit.component_entries: components_to_units[c].append(unit) with logutils.temporary_add(f"{config.path}/logs/query_planner/unit_plans.log", level="TRACE", only_sink=True) as logger_: logger_ = logger_.opt(raw=True) for component_name, units in components_to_units.items(): logger_color.debug(f"Found <green>{len(units)}</green> unit plans from " f"<blue>{component_name}</blue>.") logger_.opt(colors=True).debug(f"Found <green>{len(units)}</green> unit plans from " f"<blue>{component_name}</blue>.") for unit in units: logger_.trace(f"Component: {component_name}\n") logger_.trace("-----------------\n") logger_.trace("Transformation\n") logger_.trace("-----------------\n") logger_.trace(unit.transformation.to_str(domain)) logger_.trace("\n-----------------\n") logger_.trace("Argument Mappings\n") logger_.trace("-----------------\n") logger_.trace(unit.component_entries[component_name].argument_mappings) logger_.trace("\n========xxx========\n\n") # ---------------------------------------------------------------------------------------------------------- # # Stage 2 : Strengthen Unit Plans # ---------------------------------------------------------------------------------------------------------- # logger_color.debug("Strengthening Unit Plans...") for unit_plan in debug_iter(list(q_planner._unit_plans.values()), desc='Strengthening Unit Plans'): query: Transformation = unit_plan.transformation for component_name in unit_plan.component_entries.keys(): # The witness set examples are guaranteed to have a placeholder node for each entity, # so the use of placeholder nodes in transformations should not cause issues. if unit_plan.empty: strengthened, mapping = query.deepcopy() strengthened = Graph.from_graph(strengthened) else: examples: List[Transformation] = witness_set.get_transformations(component_name) strengthened, mapping = query.get_greatest_common_universal_supergraph(examples) inp_entities = set(query.get_input_entities()) m_reverse = mapping.reverse() keep_nodes = [n for n in strengthened.iter_nodes() if m_reverse.m_ent.get(n.entity, None) in inp_entities] strengthened = strengthened.induced_subgraph(keep_nodes=keep_nodes) unit_plan.strengthenings[component_name] = (strengthened, mapping) logger_color.debug(f"Strengthened <green>{len(q_planner._unit_plans)}</green> unit plans.") # ---------------------------------------------------------------------------------------------------------- # # Stage 3 : Combining unit plans to obtain query plans upto max-depth # Note that these are not explicitly constructed. Rather, a recursive formulation is established # to construct the query plans quickly during test-time. # ---------------------------------------------------------------------------------------------------------- # # Initialize the meta-plans from these unit plans logger.debug("Initializing meta-plans from unit-plans...") for transformation, unit_plan in q_planner._unit_plans.items(): q_planner._meta_plans[transformation] = MetaQueryPlan.initialize_from_unit_plan(unit_plan.deepcopy()[0]) logger_color.debug(f"Evolving meta query-plans upto a maximum length of " f"<blue>{config.max_length}</blue>") # Setup a mapping from the label of the output node of transformations # to the meta plan for that transformation. This helps in quickly finding # appropriate unit plans to extend a plan with. nlabel_to_unit_plan: Dict[int, Set[UnitMetaPlan]] = collections.defaultdict(set) for transformation, unit_plan in q_planner._unit_plans.items(): # Guaranteed to be a single node with that entity. output_node = next(transformation.get_output_nodes()) nlabel_to_unit_plan[output_node.label].add(unit_plan) # At every depth, the worklist will be the set of query plans with an entry for depth-1 in the blueprint. worklist: Set[MetaQueryPlan] = set(q_planner._meta_plans.values()) for depth in range(2, config.max_length + 1): for unit in worklist: q_planner._evolve_meta_plan(unit, depth, nlabel_to_unit_plan) worklist = {plan for plan in q_planner._meta_plans.values() if depth in plan.blueprint} logger_color.debug(f"Found <green>{len(worklist)}</green> transformations in total " f"at depth <blue>{depth}</blue>.") if len(worklist) == 0: break logger_color.debug(f"Found <green>{len(q_planner._meta_plans)}</green> transformations and meta " f"query plans in total with <blue>max_depth={config.max_length}</blue>.") # TODO : Log the transformations to a file. # ---------------------------------------------------------------------------------------------------------- # # Stage 4 : Setup auxiliary data-structures # ---------------------------------------------------------------------------------------------------------- # # Flattened blue-print items enable access to all the items agnostic of the component name. for plan in q_planner._meta_plans.values(): plan.blueprint_items = collections.defaultdict(set) for depth, bp_dict in plan.blueprint.items(): for items_list in bp_dict.values(): plan.blueprint_items[depth].update(items_list) # Sequence tries help in quickly computing candidate sequences given a synthesis problem. logger_color.debug("Constructing Sequence Tries...") q_planner._compute_sequence_tries() logger_color.debug(f"Sequence Tries constructed for every depth.") return q_planner