def test_multiple_depend_on_same_rule(self): rules = _suba_root_rules + [ TaskRule(B, [Select(A)], noop), TaskRule(C, [Select(A)], noop), TaskRule(A, [Select(SubA)], noop) ] subgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} "Select(C) for SubA" [color=blue] "Select(C) for SubA" -> {"(C, (Select(A),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "(C, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} }""").strip(), subgraph)
def test_noop_removal_transitive(self): # If a noop-able rule has rules that depend on it, # they should be removed from the graph. rules = [ (Exactly(B), (Select(C),), noop), (Exactly(A), (Select(B),), noop), (Exactly(A), tuple(), noop), ] intrinsics = [ (D, C, BoringRule(C)) ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns, ) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), subgraph)
def test_ruleset_with_selector_only_provided_as_root_subject(self): rules = [(A, (Select(B),), noop)] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (B,)}) validator.validate()
def test_full_graph_for_planner_example(self): symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json') tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks() intrinsics = create_fs_intrinsics('Let us pretend that this is a ProjectTree!') rule_index = RuleIndex.create(tasks, intrinsics) graphmaker = GraphMaker(rule_index, root_subject_fns={k: lambda p: Select(p) for k in (Address, # TODO, use the actual fns. PathGlobs, SingleAddress, SiblingAddresses, DescendantAddresses, AscendantAddresses )}) fullgraph = graphmaker.full_graph() print('---diagnostic------') print(fullgraph.error_message()) print('/---diagnostic------') print(fullgraph) # Assert that all of the rules specified the various task fns are present declared_rules = rule_index.all_rules() rules_remaining_in_graph_strs = set(str(r.rule) for r in fullgraph.rule_dependencies.keys()) declared_rule_strings = set(str(r) for r in declared_rules) self.assertEquals(declared_rule_strings, rules_remaining_in_graph_strs ) # statically assert that the number of dependency keys is fixed self.assertEquals(41, len(fullgraph.rule_dependencies))
def test_full_graph_for_planner_example(self): symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json') rules = create_graph_rules(address_mapper, symbol_table_cls) + create_fs_rules() rule_index = RuleIndex.create(rules) fullgraph_str = self.create_full_graph(rule_index) print('---diagnostic------') print(fullgraph_str) print('/---diagnostic------') in_root_rules = False in_all_rules = False all_rules = [] root_rule_lines = [] for line in fullgraph_str.splitlines(): if line.startswith(' // root subject types:'): pass elif line.startswith(' // root entries'): in_root_rules = True elif line.startswith(' // internal entries'): in_all_rules = True elif in_all_rules: all_rules.append(line) elif in_root_rules: root_rule_lines.append(line) else: pass self.assertEquals(36, len(all_rules)) self.assertEquals(66, len(root_rule_lines)) # 2 lines per entry
def test_smallest_full_test_multiple_root_subject_types(self): rules = [ RootRule(SubA), RootRule(A), TaskRule(A, [Select(SubA)], noop), TaskRule(B, [Select(A)], noop) ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: A, SubA // root entries "Select(A) for A" [color=blue] "Select(A) for A" -> {"SubjectIsProduct(A)"} "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "Select(B) for A" [color=blue] "Select(B) for A" -> {"(B, (Select(A),), noop) of A"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(B, (Select(A),), noop) of A" -> {"SubjectIsProduct(A)"} "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} }""").strip(), fullgraph)
def test_select_dependencies_recurse_with_different_type(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop), (B, (Select(A),), noop), (C, (Select(SubA),), noop), (SubA, tuple(), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) all_rules: (B, (Select(A),), noop) of C => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C,) (B, (Select(A),), noop) of D => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C => ((SubA, (), noop) of C, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D => ((SubA, (), noop) of D, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (SubA, (), noop) of C => (,) (SubA, (), noop) of D => (,) }""").strip(), subgraph)
def test_fails_if_root_subject_types_empty(self): rules = [ (A, (Select(B),), noop), ] with self.assertRaises(ValueError) as cm: GraphMaker(RuleIndex.create(rules), tuple()) self.assertEquals(dedent(""" root_subject_fns must not be empty """).strip(), str(cm.exception))
def test_ruleset_with_explicit_type_constraint(self): rules = [ (Exactly(A), (Select(B),), noop), (B, (Select(A),), noop) ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) validator.validate()
def __init__(self, work_dir, goals, rules, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). root_subject_types = { Address, BuildFileAddress, AscendantAddresses, DescendantAddresses, PathGlobs, SiblingAddresses, SingleAddress, } rules = list(rules) + create_snapshot_rules() rule_index = RuleIndex.create(rules) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, work_dir, project_tree.ignore_patterns, rule_index, root_subject_types) # If configured, visualize the rule graph before asserting that it is valid. if self._scheduler.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file(os.path.join(self._scheduler.visualize_to_dir(), rule_graph_name)) self._scheduler.assert_ruleset_valid()
def test_ruleset_with_missing_product_type(self): rules = [(A, (Select(B),), noop)] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (A, (Select(B),), noop): no matches for Select(B) with subject types: SubA """).strip(), str(cm.exception))
def test_ruleset_with_goal_not_produced(self): # The graph is complete, but the goal 'goal-name' requests A, # which is not produced by any rule. rules = [ (B, (Select(SubA),), noop) ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={'goal-name': AGoal}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing("no task for product used by goal \"goal-name\": AGoal", str(cm.exception))
def __init__(self, goals, tasks, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param tasks: A set of (output, input selection clause, task function) triples which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). root_subject_types = { Address, BuildFileAddress, AscendantAddresses, DescendantAddresses, PathGlobs, SiblingAddresses, SingleAddress, } singletons = create_snapshot_singletons() rule_index = RuleIndex.create(tasks, intrinsic_entries=[], singleton_entries=singletons) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, project_tree.ignore_patterns, rule_index, root_subject_types) self._scheduler.assert_ruleset_valid()
def test_smallest_full_test(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] fullgraph = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}).full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), fullgraph)
def test_smallest_full_test(self): rules = _suba_root_rules + [ RootRule(SubA), TaskRule(Exactly(A), [Select(SubA)], noop) ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} }""").strip(), fullgraph)
def test_initial_select_projection_failure(self): rules = [ (Exactly(A), (SelectProjection(B, D, ('some',), C),), noop), ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (Exactly(A), (SelectProjection(B, D, (u'some',), C),), noop): no matches for Select(C) when resolving SelectProjection(B, D, (u'some',), C) with subject types: SubA """).strip(), str(cm.exception))
def test_select_dependencies_non_matching_subselector_because_of_intrinsic(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop), ] intrinsics = [ (C, B, noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing('{empty graph}', subgraph) self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop): no matches for Select(B) when resolving SelectDependencies(B, SubA, field_types=(D,)) with subject types: D""").strip(), subgraph.error_message())
def __init__(self, goals, tasks, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param tasks: A set of (output, input selection clause, task function) triples which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._native = native self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) # Create the ExternContext, and the native Scheduler. self._scheduler = native.new_scheduler( has_products_constraint, constraint_for(Address), constraint_for(Variants) ) self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). select_product = lambda product: Select(product) root_selector_fns = { Address: select_product, AscendantAddresses: select_product, DescendantAddresses: select_product, PathGlobs: select_product, SiblingAddresses: select_product, SingleAddress: select_product, } intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree) singletons = create_snapshot_singletons(project_tree) rule_index = RuleIndex.create(tasks, intrinsics, singletons) RulesetValidator(rule_index, goals, root_selector_fns).validate() self._register_tasks(rule_index.tasks) self._register_intrinsics(rule_index.intrinsics) self._register_singletons(rule_index.singletons)
def test_single_rule_depending_on_subject_selection(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), subgraph)
def test_noop_removal_full_single_subject_type(self): rules = _suba_root_rules + [ TaskRule(Exactly(A), [Select(C)], noop), TaskRule(Exactly(A), [], noop), ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (,), noop) of SubA"} // internal entries "(A, (,), noop) of SubA" -> {} }""").strip(), fullgraph)
def test_select_literal(self): literally_a = A() rules = [ (B, (SelectLiteral(literally_a, A),), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(B) for SubA => ((B, (SelectLiteral(A(), A),), noop) of SubA,) all_rules: (B, (SelectLiteral(A(), A),), noop) of SubA => (Literal(A(), A),) }""").strip(), subgraph)
def __init__(self, work_dir, goals, rules, project_tree, native, include_trace_on_error=True, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param native: An instance of engine.native.Native. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._include_trace_on_error = include_trace_on_error self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. rules = list(rules) + create_snapshot_rules() rule_index = RuleIndex.create(rules) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, work_dir, project_tree.ignore_patterns, rule_index) # If configured, visualize the rule graph before asserting that it is valid. if self._scheduler.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file(os.path.join(self._scheduler.visualize_to_dir(), rule_graph_name)) self._scheduler.assert_ruleset_valid()
def test_ruleset_with_superclass_of_selected_type_produced_fails(self): rules = [ (A, (Select(B),), noop), (B, (Select(SubA),), noop) ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (C,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 2 (A, (Select(B),), noop): depends on unfulfillable (B, (Select(SubA),), noop) of C with subject types: C (B, (Select(SubA),), noop): no matches for Select(SubA) with subject types: C """).strip(), str(cm.exception))
def test_depends_on_multiple_one_noop(self): rules = [ (B, (Select(A),), noop), (A, (Select(C),), noop), (A, (Select(SubA),), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) all_rules: (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph)
def test_root_tuple_removed_when_no_matches(self): rules = [ RootRule(C), RootRule(D), TaskRule(Exactly(A), [Select(C)], noop), TaskRule(Exactly(B), [Select(D), Select(A)], noop), ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: C, D // root entries "Select(A) for C" [color=blue] "Select(A) for C" -> {"(A, (Select(C),), noop) of C"} // internal entries "(A, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"} }""").strip(), fullgraph)
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self): rules = [ (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop), ] intrinsics = [ (B, C, noop), ] validator = RulesetValidator(RuleIndex.create(rules, intrinsics), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (A,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop): Unreachable with subject types: Any """).strip(), str(cm.exception))
def test_successful_when_one_field_type_is_unfulfillable(self): # NB We may want this to be a warning, since it may not be intentional rules = [ (B, (Select(SubA),), noop), (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C))), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=D) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(D) for SubA => ((D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA,) all_rules: (B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA => ((B, (Select(SubA),), noop) of SubA, SubjectIsProduct(SubA), (B, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph)
def test_ruleset_with_failure_due_to_incompatible_subject_for_intrinsic(self): rules = [ (D, (Select(C),), noop) ] intrinsics = [ (B, C, noop), ] validator = RulesetValidator(RuleIndex.create(rules, intrinsics), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (A,)}) with self.assertRaises(ValueError) as cm: validator.validate() # This error message could note near matches like the intrinsic. self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (D, (Select(C),), noop): no matches for Select(C) with subject types: A """).strip(), str(cm.exception))
def test_select_dependencies_with_matching_intrinsic(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop), ] intrinsics = [ (B, C, noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA,) all_rules: (Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA => (SubjectIsProduct(SubA), IntrinsicRule((C, B), noop) of C,) IntrinsicRule((C, B), noop) of C => (,) }""").strip(), subgraph)
def test_noop_removal_full_single_subject_type(self): rules = [ # C is provided by an intrinsic, but only if the subject is B. (Exactly(A), (Select(C),), noop), (Exactly(A), tuple(), noop), ] intrinsics = [ (B, C, noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) fullgraph = graphmaker.full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), fullgraph)
def test_noop_removal_transitive(self): # If a noop-able rule has rules that depend on it, # they should be removed from the graph. rules = [ (Exactly(B), (Select(C), ), noop), (Exactly(A), (Select(B), ), noop), (Exactly(A), tuple(), noop), ] intrinsics = [(D, C, BoringRule(C))] graphmaker = GraphMaker( RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns, ) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing( dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), subgraph)
def create_subgraph(self, requested_product, rules, subject): rules = rules + _suba_root_rules rule_index = RuleIndex.create(rules) return self.create_real_subgraph(rule_index, type(subject), requested_product)
def __init__( self, *, native, ignore_patterns: List[str], use_gitignore: bool, build_root: str, local_store_dir: str, local_execution_root_dir: str, rules: Tuple[Rule, ...], union_rules: Dict[Type, "OrderedSet[Type]"], execution_options: ExecutionOptions, include_trace_on_error: bool = True, visualize_to_dir: Optional[str] = None, validate: bool = True, ) -> None: """ :param native: An instance of engine.native.Native. :param ignore_patterns: A list of gitignore-style file patterns for pants to ignore. :param use_gitignore: If set, pay attention to .gitignore files. :param build_root: The build root as a string. :param work_dir: The pants work dir. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param local_execution_root_dir: The directory to use for local execution sandboxes. :param rules: A set of Rules which is used to compute values in the graph. :param union_rules: A dict mapping union base types to member types so that rules can be written against abstract union types without knowledge of downstream rulesets. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param validate: True to assert that the ruleset is valid. """ self._native = native self.include_trace_on_error = include_trace_on_error self._visualize_to_dir = visualize_to_dir # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(list(rules), union_rules) self._root_subject_types = [r.output_type for r in rule_index.roots] # Create the native Scheduler and Session. tasks = self._register_rules(rule_index) self._scheduler = native.new_scheduler( tasks=tasks, root_subject_types=self._root_subject_types, build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, ignore_patterns=ignore_patterns, use_gitignore=use_gitignore, execution_options=execution_options, ) # If configured, visualize the rule graph before asserting that it is valid. if self._visualize_to_dir is not None: rule_graph_name = "rule_graph.dot" self.visualize_rule_graph_to_file( os.path.join(self._visualize_to_dir, rule_graph_name)) if validate: self._assert_ruleset_valid()
def test_creation_fails_with_intrinsic_that_overwrites_another_intrinsic( self): a_intrinsic = (A, A, noop) with self.assertRaises(ValueError): RuleIndex.create([BoringRule(A)], (a_intrinsic, a_intrinsic))
def __init__( self, native, project_tree, work_dir, local_store_dir, rules, union_rules, execution_options, include_trace_on_error=True, validate=True, visualize_to_dir=None, ): """ :param native: An instance of engine.native.Native. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param rules: A set of Rules which is used to compute values in the graph. :param union_rules: A dict mapping union base types to member types so that rules can be written against abstract union types without knowledge of downstream rulesets. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param validate: True to assert that the ruleset is valid. """ if execution_options.remote_execution_server and not execution_options.remote_store_server: raise ValueError( "Cannot set remote execution server without setting remote store server" ) self._native = native self.include_trace_on_error = include_trace_on_error self._visualize_to_dir = visualize_to_dir # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(list(rules), union_rules) self._root_subject_types = [r.output_type for r in rule_index.roots] # Create the native Scheduler and Session. # TODO: This `_tasks` reference could be a local variable, since it is not used # after construction. self._tasks = native.new_tasks() self._register_rules(rule_index) self._scheduler = native.new_scheduler( tasks=self._tasks, root_subject_types=self._root_subject_types, build_root=project_tree.build_root, work_dir=work_dir, local_store_dir=local_store_dir, ignore_patterns=project_tree.ignore_patterns, execution_options=execution_options, construct_directory_digest=Digest, construct_snapshot=Snapshot, construct_file_content=FileContent, construct_files_content=FilesContent, construct_process_result=FallibleExecuteProcessResult, type_address=Address, type_path_globs=PathGlobs, type_directory_digest=Digest, type_snapshot=Snapshot, type_merge_snapshots_request=MergedDirectories, type_files_content=FilesContent, type_dir=Dir, type_file=File, type_link=Link, type_process_request=ExecuteProcessRequest, type_process_result=FallibleExecuteProcessResult, type_generator=GeneratorType, type_url_to_fetch=UrlToFetch, ) # If configured, visualize the rule graph before asserting that it is valid. if self._visualize_to_dir is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file( os.path.join(self._visualize_to_dir, rule_graph_name)) if validate: self._assert_ruleset_valid()
def __init__( self, *, native, ignore_patterns: List[str], use_gitignore: bool, build_root: str, local_store_dir: str, local_execution_root_dir: str, named_caches_dir: str, ca_certs_path: Optional[str], rules: Iterable[Rule], union_membership: UnionMembership, execution_options: ExecutionOptions, executor: PyExecutor, include_trace_on_error: bool = True, visualize_to_dir: Optional[str] = None, validate_reachability: bool = True, ) -> None: """ :param native: An instance of engine.native.Native. :param ignore_patterns: A list of gitignore-style file patterns for pants to ignore. :param use_gitignore: If set, pay attention to .gitignore files. :param build_root: The build root as a string. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param local_execution_root_dir: The directory to use for local execution sandboxes. :param named_caches_dir: The directory to use as the root for named mutable caches. :param ca_certs_path: Path to pem file for custom CA, if needed. :param rules: A set of Rules which is used to compute values in the graph. :param union_membership: All the registered and normalized union rules. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :param validate_reachability: True to assert that all rules in an otherwise successfully constructed rule graph are reachable: if a graph cannot be successfully constructed, it is always a fatal error. """ self._native = native self.include_trace_on_error = include_trace_on_error self._visualize_to_dir = visualize_to_dir # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(rules) # Create the native Scheduler and Session. tasks = self._register_rules(rule_index, union_membership) types = PyTypes( file_digest=FileDigest, snapshot=Snapshot, paths=Paths, file_content=FileContent, digest_contents=DigestContents, path_globs=PathGlobs, merge_digests=MergeDigests, add_prefix=AddPrefix, remove_prefix=RemovePrefix, create_digest=CreateDigest, digest_subset=DigestSubset, download_file=DownloadFile, platform=Platform, multi_platform_process=MultiPlatformProcess, process_result=FallibleProcessResultWithPlatform, coroutine=CoroutineType, session_values=SessionValues, interactive_process_result=InteractiveProcessResult, engine_aware_parameter=EngineAwareParameter, ) self._scheduler = native.new_scheduler( tasks=tasks, build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, named_caches_dir=named_caches_dir, ca_certs_path=ca_certs_path, ignore_patterns=ignore_patterns, use_gitignore=use_gitignore, executor=executor, execution_options=execution_options, types=types, ) # If configured, visualize the rule graph before asserting that it is valid. if self._visualize_to_dir is not None: rule_graph_name = "rule_graph.dot" self.visualize_rule_graph_to_file(os.path.join(self._visualize_to_dir, rule_graph_name)) if validate_reachability: self._native.lib.validate_reachability(self._scheduler)
def test_creation_fails_with_bad_declaration_type(self): with self.assertRaises(TypeError) as cm: RuleIndex.create([A()]) self.assertEquals("Unexpected rule type: <class 'pants_test.engine.test_rules.A'>." " Rules either extend Rule, or are static functions decorated with @rule.", str(cm.exception))
def test_creation_fails_with_bad_declaration_type(self): with self.assertRaisesWithMessage( TypeError, """\ Rule entry A() had an unexpected type: <class 'pants_test.engine.test_rules.A'>. Rules either extend Rule or UnionRule, or are static functions decorated with @rule.""" ): RuleIndex.create([A()])
def __init__( self, native, project_tree, work_dir, rules, execution_options, include_trace_on_error=True, validate=True, ): """ :param native: An instance of engine.native.Native. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param rules: A set of Rules which is used to compute values in the graph. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param validate: True to assert that the ruleset is valid. """ if execution_options.remote_execution_server and not execution_options.remote_store_server: raise ValueError( "Cannot set remote execution server without setting remote store server" ) self._native = native self.include_trace_on_error = include_trace_on_error # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(list(rules)) self._root_subject_types = sorted(rule_index.roots) # Create the native Scheduler and Session. # TODO: This `_tasks` reference could be a local variable, since it is not used # after construction. self._tasks = native.new_tasks() self._register_rules(rule_index) self._scheduler = native.new_scheduler( self._tasks, self._root_subject_types, project_tree.build_root, work_dir, project_tree.ignore_patterns, execution_options, DirectoryDigest, Snapshot, FileContent, FilesContent, Path, Dir, File, Link, FallibleExecuteProcessResult, has_products_constraint, constraint_for(Address), constraint_for(Variants), constraint_for(PathGlobs), constraint_for(DirectoryDigest), constraint_for(Snapshot), constraint_for(FilesContent), constraint_for(Dir), constraint_for(File), constraint_for(Link), constraint_for(ExecuteProcessRequest), constraint_for(FallibleExecuteProcessResult), constraint_for(GeneratorType), ) # If configured, visualize the rule graph before asserting that it is valid. if self.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file( os.path.join(self.visualize_to_dir(), rule_graph_name)) if validate: self._assert_ruleset_valid()
def create_native_scheduler(self, intrinsic_entries, root_subject_types, rules): init_subsystem(Native.Factory) rule_index = RuleIndex.create(rules, intrinsic_entries) native = Native.Factory.global_instance().create() scheduler = WrappedNativeScheduler(native, '.', [], rule_index, root_subject_types) return scheduler
def create_subgraph_with_intrinsics(self, intrinsics, requested_product, rules, subject, subject_types): rule_index = RuleIndex.create(rules, intrinsics) return self.create_real_subgraph(subject_types, rule_index, type(subject), requested_product)
def __init__( self, *, ignore_patterns: list[str], use_gitignore: bool, build_root: str, local_execution_root_dir: str, named_caches_dir: str, ca_certs_path: str | None, rules: Iterable[Rule], union_membership: UnionMembership, execution_options: ExecutionOptions, local_store_options: LocalStoreOptions, executor: PyExecutor, include_trace_on_error: bool = True, visualize_to_dir: str | None = None, validate_reachability: bool = True, watch_filesystem: bool = True, ) -> None: """ :param ignore_patterns: A list of gitignore-style file patterns for pants to ignore. :param use_gitignore: If set, pay attention to .gitignore files. :param build_root: The build root as a string. :param local_execution_root_dir: The directory to use for local execution sandboxes. :param named_caches_dir: The directory to use as the root for named mutable caches. :param ca_certs_path: Path to pem file for custom CA, if needed. :param rules: A set of Rules which is used to compute values in the graph. :param union_membership: All the registered and normalized union rules. :param execution_options: Execution options for (remote) processes. :param local_store_options: Options for the engine's LMDB store(s). :param include_trace_on_error: Include the trace through the graph upon encountering errors. :param validate_reachability: True to assert that all rules in an otherwise successfully constructed rule graph are reachable: if a graph cannot be successfully constructed, it is always a fatal error. :param watch_filesystem: False if filesystem watching should be disabled. """ self.include_trace_on_error = include_trace_on_error self._visualize_to_dir = visualize_to_dir self._visualize_run_count = 0 # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(rules) tasks = register_rules(rule_index, union_membership) # Create the native Scheduler and Session. types = PyTypes( file_digest=FileDigest, snapshot=Snapshot, paths=Paths, file_content=FileContent, file_entry=FileEntry, directory=Directory, digest_contents=DigestContents, digest_entries=DigestEntries, path_globs=PathGlobs, merge_digests=MergeDigests, add_prefix=AddPrefix, remove_prefix=RemovePrefix, create_digest=CreateDigest, digest_subset=DigestSubset, download_file=DownloadFile, platform=Platform, multi_platform_process=MultiPlatformProcess, process_result=FallibleProcessResult, process_result_metadata=ProcessResultMetadata, coroutine=CoroutineType, session_values=SessionValues, run_id=RunId, interactive_process=InteractiveProcess, interactive_process_result=InteractiveProcessResult, engine_aware_parameter=EngineAwareParameter, ) remoting_options = PyRemotingOptions( execution_enable=execution_options.remote_execution, store_address=execution_options.remote_store_address, execution_address=execution_options.remote_execution_address, execution_process_cache_namespace=execution_options. process_execution_cache_namespace, instance_name=execution_options.remote_instance_name, root_ca_certs_path=execution_options.remote_ca_certs_path, store_headers=tuple( execution_options.remote_store_headers.items()), store_chunk_bytes=execution_options.remote_store_chunk_bytes, store_chunk_upload_timeout=execution_options. remote_store_chunk_upload_timeout_seconds, store_rpc_retries=execution_options.remote_store_rpc_retries, store_rpc_concurrency=execution_options. remote_store_rpc_concurrency, store_batch_api_size_limit=execution_options. remote_store_batch_api_size_limit, cache_warnings_behavior=execution_options.remote_cache_warnings. value, cache_eager_fetch=execution_options.remote_cache_eager_fetch, cache_rpc_concurrency=execution_options. remote_cache_rpc_concurrency, execution_extra_platform_properties=tuple( tuple(pair.split("=", 1)) for pair in execution_options.remote_execution_extra_platform_properties), execution_headers=tuple( execution_options.remote_execution_headers.items()), execution_overall_deadline_secs=execution_options. remote_execution_overall_deadline_secs, execution_rpc_concurrency=execution_options. remote_execution_rpc_concurrency, ) py_local_store_options = PyLocalStoreOptions( store_dir=local_store_options.store_dir, process_cache_max_size_bytes=local_store_options. processes_max_size_bytes, files_max_size_bytes=local_store_options.files_max_size_bytes, directories_max_size_bytes=local_store_options. directories_max_size_bytes, lease_time_millis=LOCAL_STORE_LEASE_TIME_SECS * 1000, shard_count=local_store_options.shard_count, ) exec_stategy_opts = PyExecutionStrategyOptions( local_cache=execution_options.process_execution_local_cache, remote_cache_read=execution_options.remote_cache_read, remote_cache_write=execution_options.remote_cache_write, local_cleanup=execution_options.process_execution_local_cleanup, local_parallelism=execution_options. process_execution_local_parallelism, local_enable_nailgun=execution_options. process_execution_local_enable_nailgun, remote_parallelism=execution_options. process_execution_remote_parallelism, ) self._py_scheduler = native_engine.scheduler_create( executor, tasks, types, build_root, local_execution_root_dir, named_caches_dir, ca_certs_path, ignore_patterns, use_gitignore, watch_filesystem, remoting_options, py_local_store_options, exec_stategy_opts, ) # If configured, visualize the rule graph before asserting that it is valid. if self._visualize_to_dir is not None: rule_graph_name = "rule_graph.dot" self.visualize_rule_graph_to_file( os.path.join(self._visualize_to_dir, rule_graph_name)) if validate_reachability: native_engine.validate_reachability(self.py_scheduler)
def create_native_scheduler(self, root_subject_types, rules): rule_index = RuleIndex.create(rules) native = init_native() scheduler = WrappedNativeScheduler(native, '.', './.pants.d', [], rule_index, root_subject_types) return scheduler