def test_rule_index_creation_fails_with_bad_declaration_type(): with pytest.raises(TypeError) as exc: RuleIndex.create([A()]) assert str(exc.value) == ( "Rule entry A() had an unexpected type: <class 'pants.engine.rules_test.A'>. Rules " "either extend Rule or UnionRule, or are static functions decorated with @rule." )
def test_creation_fails_with_bad_declaration_type(self): with self.assertRaises(TypeError) as cm: RuleIndex.create([A()]) self.assertEqual( "Unexpected rule type: <class 'pants_test.engine.test_rules.A'>." " Rules either extend Rule, or are static functions decorated with @rule.", str(cm.exception))
def test_creation_fails_with_bad_declaration_type(self): with self.assertRaisesWithMessage( TypeError, "Rule entry A() had an unexpected type: <class " "'tests.python.pants_test.engine.test_rules.A'>. Rules either extend Rule or UnionRule, or " "are static functions decorated with @rule."""): RuleIndex.create([A()])
def create_native_scheduler(self, root_subject_types, rules): init_subsystem(Native.Factory) rule_index = RuleIndex.create(rules) native = Native.Factory.global_instance().create() scheduler = WrappedNativeScheduler(native, '.', './.pants.d', [], rule_index, root_subject_types) return scheduler
def register_rules(self, rules): """Registers the given rules. param rules: The rules to register. :type rules: :class:`collections.Iterable` containing :class:`pants.engine.rules.Rule` instances. """ if not isinstance(rules, Iterable): raise TypeError( "The rules must be an iterable, given {!r}".format(rules)) # "Index" the rules to normalize them and expand their dependencies. normalized_rules = RuleIndex.create(rules).normalized_rules() indexed_rules = normalized_rules.rules union_rules = normalized_rules.union_rules # Store the rules and record their dependency Optionables. self._rules.update(indexed_rules) for union_base, new_members in union_rules.items(): existing_members = self._union_rules.get(union_base, None) if existing_members is None: self._union_rules[union_base] = new_members else: existing_members.update(new_members) dependency_optionables = { do for rule in indexed_rules for do in rule.dependency_optionables if rule.dependency_optionables } self.register_optionables(dependency_optionables)
def register_rules(self, rules): """Registers the given rules. param rules: The rules to register. :type rules: :class:`collections.Iterable` containing :class:`pants.engine.rules.Rule` instances. """ if not isinstance(rules, Iterable): raise TypeError( 'The rules must be an iterable, given {!r}'.format(rules)) # "Index" the rules to normalize them and expand their dependencies. normalized_rules = RuleIndex.create(rules).normalized_rules() indexed_rules = normalized_rules.rules union_rules = normalized_rules.union_rules # Store the rules and record their dependency Optionables. self._rules.update(indexed_rules) self._union_rules.update(union_rules) dependency_optionables = { do for rule in indexed_rules for do in rule.dependency_optionables if rule.dependency_optionables } self.register_optionables(dependency_optionables)
def create_native_scheduler(rules): """Create a WrappedNativeScheduler, with an initialized native instance.""" rule_index = RuleIndex.create(rules) native = init_native() scheduler = WrappedNativeScheduler(native, '.', './.pants.d', [], rule_index) return scheduler
def test_multiple_depend_on_same_rule(self): rules = _suba_root_rules + [ TaskRule(B, [Select(A)], noop), TaskRule(C, [Select(A)], noop), TaskRule(A, [Select(SubA)], noop) ] subgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} "Select(C) for SubA" [color=blue] "Select(C) for SubA" -> {"(C, (Select(A),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "(C, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} }""").strip(), subgraph)
def __init__( self, *, native, ignore_patterns: List[str], use_gitignore: bool, build_root: str, local_store_dir: str, local_execution_root_dir: str, rules: Tuple[Rule, ...], union_rules: Dict[Type, "OrderedSet[Type]"], execution_options: ExecutionOptions, include_trace_on_error: bool = True, visualize_to_dir: Optional[str] = None, validate: bool = True, ) -> None: """ :param native: An instance of engine.native.Native. :param ignore_patterns: A list of gitignore-style file patterns for pants to ignore. :param use_gitignore: If set, pay attention to .gitignore files. :param build_root: The build root as a string. :param work_dir: The pants work dir. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param local_execution_root_dir: The directory to use for local execution sandboxes. :param rules: A set of Rules which is used to compute values in the graph. :param union_rules: A dict mapping union base types to member types so that rules can be written against abstract union types without knowledge of downstream rulesets. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param validate: True to assert that the ruleset is valid. """ self._native = native self.include_trace_on_error = include_trace_on_error self._visualize_to_dir = visualize_to_dir # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(list(rules), union_rules) self._root_subject_types = [r.output_type for r in rule_index.roots] # Create the native Scheduler and Session. tasks = self._register_rules(rule_index) self._scheduler = native.new_scheduler( tasks=tasks, root_subject_types=self._root_subject_types, build_root=build_root, local_store_dir=local_store_dir, local_execution_root_dir=local_execution_root_dir, ignore_patterns=ignore_patterns, use_gitignore=use_gitignore, execution_options=execution_options, ) # If configured, visualize the rule graph before asserting that it is valid. if self._visualize_to_dir is not None: rule_graph_name = "rule_graph.dot" self.visualize_rule_graph_to_file(os.path.join(self._visualize_to_dir, rule_graph_name)) if validate: self._assert_ruleset_valid()
def test_ruleset_with_selector_only_provided_as_root_subject(self): rules = [(A, (Select(B),), noop)] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (B,)}) validator.validate()
def test_full_graph_for_planner_example(self): symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json') tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks() intrinsics = create_fs_intrinsics('Let us pretend that this is a ProjectTree!') rule_index = RuleIndex.create(tasks, intrinsics) graphmaker = GraphMaker(rule_index, root_subject_fns={k: lambda p: Select(p) for k in (Address, # TODO, use the actual fns. PathGlobs, SingleAddress, SiblingAddresses, DescendantAddresses, AscendantAddresses )}) fullgraph = graphmaker.full_graph() print('---diagnostic------') print(fullgraph.error_message()) print('/---diagnostic------') print(fullgraph) # Assert that all of the rules specified the various task fns are present declared_rules = rule_index.all_rules() rules_remaining_in_graph_strs = set(str(r.rule) for r in fullgraph.rule_dependencies.keys()) declared_rule_strings = set(str(r) for r in declared_rules) self.assertEquals(declared_rule_strings, rules_remaining_in_graph_strs ) # statically assert that the number of dependency keys is fixed self.assertEquals(41, len(fullgraph.rule_dependencies))
def test_smallest_full_test_multiple_root_subject_types(self): rules = [ RootRule(SubA), RootRule(A), TaskRule(A, [Select(SubA)], noop), TaskRule(B, [Select(A)], noop) ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: A, SubA // root entries "Select(A) for A" [color=blue] "Select(A) for A" -> {"SubjectIsProduct(A)"} "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "Select(B) for A" [color=blue] "Select(B) for A" -> {"(B, (Select(A),), noop) of A"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(B, (Select(A),), noop) of A" -> {"SubjectIsProduct(A)"} "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} }""").strip(), fullgraph)
def test_select_dependencies_recurse_with_different_type(self): rules = [(Exactly(A), (SelectDependencies(B, SubA, field_types=( C, D, )), ), noop), (B, (Select(A), ), noop), (C, (Select(SubA), ), noop), (SubA, tuple(), noop)] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing( dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) all_rules: (B, (Select(A),), noop) of C => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C,) (B, (Select(A),), noop) of D => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C => ((SubA, (), noop) of C, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D => ((SubA, (), noop) of D, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (SubA, (), noop) of C => (,) (SubA, (), noop) of D => (,) }""").strip(), subgraph)
def test_noop_removal_transitive(self): # If a noop-able rule has rules that depend on it, # they should be removed from the graph. rules = [ (Exactly(B), (Select(C),), noop), (Exactly(A), (Select(B),), noop), (Exactly(A), tuple(), noop), ] intrinsics = [ (D, C, BoringRule(C)) ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns, ) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), subgraph)
def test_select_dependencies_recurse_with_different_type(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop), (B, (Select(A),), noop), (C, (Select(SubA),), noop), (SubA, tuple(), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) all_rules: (B, (Select(A),), noop) of C => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C,) (B, (Select(A),), noop) of D => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C => ((SubA, (), noop) of C, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D => ((SubA, (), noop) of D, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) (SubA, (), noop) of C => (,) (SubA, (), noop) of D => (,) }""").strip(), subgraph)
def test_smallest_full_test_multiple_root_subject_types(self): rules = [ RootRule(SubA), RootRule(A), TaskRule(A, [Select(SubA)], noop), TaskRule(B, [Select(A)], noop) ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: A, SubA // root entries "Select(A) for A" [color=blue] "Select(A) for A" -> {"SubjectIsProduct(A)"} "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "Select(B) for A" [color=blue] "Select(B) for A" -> {"(B, (Select(A),), noop) of A"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(B, (Select(A),), noop) of A" -> {"SubjectIsProduct(A)"} "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} }""").strip(), fullgraph)
def test_full_graph_for_planner_example(self): symbol_table_cls = TargetTable address_mapper = AddressMapper(symbol_table_cls, JsonParser, '*.BUILD.json') rules = create_graph_rules(address_mapper, symbol_table_cls) + create_fs_rules() rule_index = RuleIndex.create(rules) fullgraph_str = self.create_full_graph(rule_index) print('---diagnostic------') print(fullgraph_str) print('/---diagnostic------') in_root_rules = False in_all_rules = False all_rules = [] root_rule_lines = [] for line in fullgraph_str.splitlines(): if line.startswith(' // root subject types:'): pass elif line.startswith(' // root entries'): in_root_rules = True elif line.startswith(' // internal entries'): in_all_rules = True elif in_all_rules: all_rules.append(line) elif in_root_rules: root_rule_lines.append(line) else: pass self.assertEquals(36, len(all_rules)) self.assertEquals(66, len(root_rule_lines)) # 2 lines per entry
def test_full_graph_for_planner_example(self): symbol_table = TargetTable() address_mapper = AddressMapper(JsonParser(symbol_table), '*.BUILD.json') rules = create_graph_rules(address_mapper, symbol_table) + create_fs_rules() rule_index = RuleIndex.create(rules) fullgraph_str = self.create_full_graph(rule_index) print('---diagnostic------') print(fullgraph_str) print('/---diagnostic------') in_root_rules = False in_all_rules = False all_rules = [] root_rule_lines = [] for line in fullgraph_str.splitlines(): if line.startswith(' // root subject types:'): pass elif line.startswith(' // root entries'): in_root_rules = True elif line.startswith(' // internal entries'): in_all_rules = True elif in_all_rules: all_rules.append(line) elif in_root_rules: root_rule_lines.append(line) else: pass self.assertEquals(41, len(all_rules)) self.assertEquals(78, len(root_rule_lines)) # 2 lines per entry
def test_multiple_depend_on_same_rule(self): rules = _suba_root_rules + [ TaskRule(B, [Select(A)], noop), TaskRule(C, [Select(A)], noop), TaskRule(A, [Select(SubA)], noop) ] subgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "Select(B) for SubA" [color=blue] "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} "Select(C) for SubA" [color=blue] "Select(C) for SubA" -> {"(C, (Select(A),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} "(C, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} }""").strip(), subgraph)
def test_select_dependencies_multiple_field_types_all_resolvable_with_deps( self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=( C, D, )), ), noop), # for the C type, it'll just be a literal, but for D, it'll traverse one more edge (B, (Select(C), ), noop), (C, (Select(D), ), noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing( dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) all_rules: (B, (Select(C),), noop) of C => (SubjectIsProduct(C),) (B, (Select(C),), noop) of D => ((C, (Select(D),), noop) of D,) (C, (Select(D),), noop) of D => (SubjectIsProduct(D),) (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(C),), noop) of C, (B, (Select(C),), noop) of D,) }""").strip(), subgraph)
def test_fails_if_root_subject_types_empty(self): rules = [ (A, (Select(B),), noop), ] with self.assertRaises(ValueError) as cm: GraphMaker(RuleIndex.create(rules), tuple()) self.assertEquals(dedent(""" root_subject_fns must not be empty """).strip(), str(cm.exception))
def test_ruleset_with_explicit_type_constraint(self): rules = [(Exactly(A), (Select(B), ), noop), (B, (Select(A), ), noop)] validator = RulesetValidator( RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA, )}) validator.validate()
def test_ruleset_with_selector_only_provided_as_root_subject(self): rules = [(A, (Select(B), ), noop)] validator = RulesetValidator( RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (B, )}) validator.validate()
def test_ruleset_with_explicit_type_constraint(self): rules = [ (Exactly(A), (Select(B),), noop), (B, (Select(A),), noop) ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) validator.validate()
def test_fails_if_root_subject_types_empty(self): rules = [ (A, (Select(B), ), noop), ] with self.assertRaises(ValueError) as cm: GraphMaker(RuleIndex.create(rules), tuple()) self.assertEquals( dedent(""" root_subject_fns must not be empty """).strip(), str(cm.exception))
def __init__(self, work_dir, goals, rules, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). root_subject_types = { Address, BuildFileAddress, AscendantAddresses, DescendantAddresses, PathGlobs, SiblingAddresses, SingleAddress, } rules = list(rules) + create_snapshot_rules() rule_index = RuleIndex.create(rules) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, work_dir, project_tree.ignore_patterns, rule_index, root_subject_types) # If configured, visualize the rule graph before asserting that it is valid. if self._scheduler.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file(os.path.join(self._scheduler.visualize_to_dir(), rule_graph_name)) self._scheduler.assert_ruleset_valid()
def __init__( self, native, project_tree, local_store_dir, rules, union_rules, execution_options, include_trace_on_error=True, validate=True, visualize_to_dir=None, ): """ :param native: An instance of engine.native.Native. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param local_store_dir: The directory to use for storing the engine's LMDB store in. :param rules: A set of Rules which is used to compute values in the graph. :param union_rules: A dict mapping union base types to member types so that rules can be written against abstract union types without knowledge of downstream rulesets. :param execution_options: Execution options for (remote) processes. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param validate: True to assert that the ruleset is valid. """ self._native = native self.include_trace_on_error = include_trace_on_error self._visualize_to_dir = visualize_to_dir # Validate and register all provided and intrinsic tasks. rule_index = RuleIndex.create(list(rules), union_rules) self._root_subject_types = [r.output_type for r in rule_index.roots] # Create the native Scheduler and Session. # TODO: This `_tasks` reference could be a local variable, since it is not used # after construction. self._tasks = native.new_tasks() self._register_rules(rule_index) self._scheduler = native.new_scheduler( tasks=self._tasks, root_subject_types=self._root_subject_types, build_root=project_tree.build_root, local_store_dir=local_store_dir, ignore_patterns=project_tree.ignore_patterns, execution_options=execution_options, ) # If configured, visualize the rule graph before asserting that it is valid. if self._visualize_to_dir is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file( os.path.join(self._visualize_to_dir, rule_graph_name)) if validate: self._assert_ruleset_valid()
def __init__(self, work_dir, goals, rules, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). root_subject_types = { Address, BuildFileAddress, AscendantAddresses, DescendantAddresses, PathGlobs, SiblingAddresses, SingleAddress, } rules = list(rules) + create_snapshot_rules() rule_index = RuleIndex.create(rules) self._scheduler = WrappedNativeScheduler( native, project_tree.build_root, work_dir, project_tree.ignore_patterns, rule_index, root_subject_types) # If configured, visualize the rule graph before asserting that it is valid. if self._scheduler.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file( os.path.join(self._scheduler.visualize_to_dir(), rule_graph_name)) self._scheduler.assert_ruleset_valid()
def __init__(self, goals, tasks, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param tasks: A set of (output, input selection clause, task function) triples which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._native = native self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) # Create the ExternContext, and the native Scheduler. self._scheduler = native.new_scheduler(has_products_constraint, constraint_for(Address), constraint_for(Variants)) self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). select_product = lambda product: Select(product) root_selector_fns = { Address: select_product, AscendantAddresses: select_product, DescendantAddresses: select_product, PathGlobs: select_product, SiblingAddresses: select_product, SingleAddress: select_product, } intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree) singletons = create_snapshot_singletons(project_tree) rule_index = RuleIndex.create(tasks, intrinsics, singletons) RulesetValidator(rule_index, goals, root_selector_fns).validate() self._register_tasks(rule_index.tasks) self._register_intrinsics(rule_index.intrinsics) self._register_singletons(rule_index.singletons)
def test_smallest_full_test(self): rules = [TaskRule(Exactly(A), [Select(SubA)], noop)] fullgraph = self.create_full_graph({SubA}, RuleIndex.create(rules)) self.assert_equal_with_printing( dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} }""").strip(), fullgraph)
def test_ruleset_with_missing_product_type(self): rules = [(A, (Select(B),), noop)] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (A, (Select(B),), noop): no matches for Select(B) with subject types: SubA """).strip(), str(cm.exception))
def test_ruleset_with_goal_not_produced(self): # The graph is complete, but the goal 'goal-name' requests A, # which is not produced by any rule. rules = [ (B, (Select(SubA),), noop) ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={'goal-name': AGoal}, root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing("no task for product used by goal \"goal-name\": AGoal", str(cm.exception))
def __init__(self, goals, tasks, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param tasks: A set of (output, input selection clause, task function) triples which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). root_subject_types = { Address, BuildFileAddress, AscendantAddresses, DescendantAddresses, PathGlobs, SiblingAddresses, SingleAddress, } singletons = create_snapshot_singletons() rule_index = RuleIndex.create(tasks, intrinsic_entries=[], singleton_entries=singletons) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, project_tree.ignore_patterns, rule_index, root_subject_types) self._scheduler.assert_ruleset_valid()
def test_smallest_full_test(self): rules = _suba_root_rules + [ RootRule(SubA), TaskRule(Exactly(A), [Select(SubA)], noop) ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} }""").strip(), fullgraph)
def test_ruleset_with_missing_product_type(self): rules = [(A, (Select(B), ), noop)] validator = RulesetValidator( RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (SubA, )}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (A, (Select(B),), noop): no matches for Select(B) with subject types: SubA """).strip(), str(cm.exception))
def test_ruleset_with_goal_not_produced(self): # The graph is complete, but the goal 'goal-name' requests A, # which is not produced by any rule. rules = [(B, (Select(SubA), ), noop)] validator = RulesetValidator( RuleIndex.create(rules, tuple()), goal_to_product={'goal-name': AGoal}, root_subject_fns={k: lambda p: Select(p) for k in (SubA, )}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing( "no task for product used by goal \"goal-name\": AGoal", str(cm.exception))
def test_single_rule_depending_on_subject_selection(self): rules = [(Exactly(A), (Select(SubA), ), noop)] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing( dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), subgraph)
def test_smallest_full_test(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] fullgraph = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns={k: lambda p: Select(p) for k in (SubA,)}).full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), fullgraph)
def test_select_literal(self): literally_a = A() rules = [(B, (SelectLiteral(literally_a, A), ), noop)] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing( dedent(""" { root_subject_types: (SubA,) root_rules: Select(B) for SubA => ((B, (SelectLiteral(A(), A),), noop) of SubA,) all_rules: (B, (SelectLiteral(A(), A),), noop) of SubA => (Literal(A(), A),) }""").strip(), subgraph)
def __init__(self, goals, tasks, project_tree, native, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param tasks: A set of (output, input selection clause, task function) triples which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param native: An instance of engine.subsystem.native.Native. :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._native = native self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # TODO: The only (?) case where we use inheritance rather than exact type unions. has_products_constraint = SubclassesOf(HasProducts) # Create the ExternContext, and the native Scheduler. self._scheduler = native.new_scheduler( has_products_constraint, constraint_for(Address), constraint_for(Variants) ) self._execution_request = None # Validate and register all provided and intrinsic tasks. # TODO: This bounding of input Subject types allows for closed-world validation, but is not # strictly necessary for execution. We might eventually be able to remove it by only executing # validation below the execution roots (and thus not considering paths that aren't in use). select_product = lambda product: Select(product) root_selector_fns = { Address: select_product, AscendantAddresses: select_product, DescendantAddresses: select_product, PathGlobs: select_product, SiblingAddresses: select_product, SingleAddress: select_product, } intrinsics = create_fs_intrinsics(project_tree) + create_snapshot_intrinsics(project_tree) singletons = create_snapshot_singletons(project_tree) rule_index = RuleIndex.create(tasks, intrinsics, singletons) RulesetValidator(rule_index, goals, root_selector_fns).validate() self._register_tasks(rule_index.tasks) self._register_intrinsics(rule_index.intrinsics) self._register_singletons(rule_index.singletons)
def test_select_dependencies_non_matching_subselector_because_of_intrinsic(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop), ] intrinsics = [ (C, B, noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing('{empty graph}', subgraph) self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop): no matches for Select(B) when resolving SelectDependencies(B, SubA, field_types=(D,)) with subject types: D""").strip(), subgraph.error_message())
def test_secondary_select_projection_failure(self): rules = [(Exactly(A), (SelectProjection(B, D, ('some', ), C), ), noop), (C, tuple(), noop)] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (Exactly(A), (SelectProjection(B, D, (u'some',), C),), noop): no matches for Select(B) when resolving SelectProjection(B, D, (u'some',), C) with subject types: D """).strip(), str(cm.exception))
def test_smallest_full_test(self): rules = [(Exactly(A), (Select(SubA), ), noop)] fullgraph = GraphMaker( RuleIndex.create(rules, tuple()), root_subject_fns={k: lambda p: Select(p) for k in (SubA, )}).full_graph() self.assert_equal_with_printing( dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), fullgraph)
def test_single_rule_depending_on_subject_selection(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) all_rules: (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) }""").strip(), subgraph)
def test_initial_select_projection_failure(self): rules = [ (Exactly(A), (SelectProjection(B, D, ('some',), C),), noop), ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns=_suba_root_subject_fns) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (Exactly(A), (SelectProjection(B, D, (u'some',), C),), noop): no matches for Select(C) when resolving SelectProjection(B, D, (u'some',), C) with subject types: SubA """).strip(), str(cm.exception))
def register_rules(self, plugin_or_backend: str, rules: Iterable[Rule | UnionRule]): """Registers the given rules.""" if not isinstance(rules, Iterable): raise TypeError( f"The rules must be an iterable, given {rules!r}") # "Index" the rules to normalize them and expand their dependencies. rule_index = RuleIndex.create(rules) self._rules.update(rule_index.rules) self._rules.update(rule_index.queries) self._union_rules.update(rule_index.union_rules) self.register_subsystems( plugin_or_backend, (rule.output_type for rule in self._rules if issubclass(rule.output_type, Subsystem)), )
def __init__(self, work_dir, goals, rules, project_tree, native, include_trace_on_error=True, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param native: An instance of engine.subsystem.native.Native. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._include_trace_on_error = include_trace_on_error self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. rules = list(rules) + create_snapshot_rules() rule_index = RuleIndex.create(rules) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, work_dir, project_tree.ignore_patterns, rule_index) # If configured, visualize the rule graph before asserting that it is valid. if self._scheduler.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file( os.path.join(self._scheduler.visualize_to_dir(), rule_graph_name)) self._scheduler.assert_ruleset_valid()
def register_rules(self, rules): """Registers the given rules. param rules: The rules to register. :type rules: :class:`collections.Iterable` containing :class:`pants.engine.rules.Rule` instances. """ if not isinstance(rules, Iterable): raise TypeError( "The rules must be an iterable, given {!r}".format(rules)) # "Index" the rules to normalize them and expand their dependencies. rules, union_rules = RuleIndex.create(rules).normalized_rules() self._rules.update(rules) self._union_rules.update(union_rules) self.register_optionables( rule.output_type for rule in self._rules if issubclass(rule.output_type, Optionable))
def test_noop_removal_full_single_subject_type(self): rules = _suba_root_rules + [ TaskRule(Exactly(A), [Select(C)], noop), TaskRule(Exactly(A), [], noop), ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (,), noop) of SubA"} // internal entries "(A, (,), noop) of SubA" -> {} }""").strip(), fullgraph)
def test_select_literal(self): literally_a = A() rules = [ (B, (SelectLiteral(literally_a, A),), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(B) for SubA => ((B, (SelectLiteral(A(), A),), noop) of SubA,) all_rules: (B, (SelectLiteral(A(), A),), noop) of SubA => (Literal(A(), A),) }""").strip(), subgraph)
def __init__(self, work_dir, goals, rules, project_tree, native, include_trace_on_error=True, graph_lock=None): """ :param goals: A dict from a goal name to a product type. A goal is just an alias for a particular (possibly synthetic) product. :param rules: A set of Rules which is used to compute values in the product graph. :param project_tree: An instance of ProjectTree for the current build root. :param work_dir: The pants work dir. :param native: An instance of engine.native.Native. :param include_trace_on_error: Include the trace through the graph upon encountering errors. :type include_trace_on_error: bool :param graph_lock: A re-entrant lock to use for guarding access to the internal product Graph instance. Defaults to creating a new threading.RLock(). """ self._products_by_goal = goals self._project_tree = project_tree self._include_trace_on_error = include_trace_on_error self._product_graph_lock = graph_lock or threading.RLock() self._run_count = 0 # Create the ExternContext, and the native Scheduler. self._execution_request = None # Validate and register all provided and intrinsic tasks. rules = list(rules) + create_snapshot_rules() rule_index = RuleIndex.create(rules) self._scheduler = WrappedNativeScheduler(native, project_tree.build_root, work_dir, project_tree.ignore_patterns, rule_index) # If configured, visualize the rule graph before asserting that it is valid. if self._scheduler.visualize_to_dir() is not None: rule_graph_name = 'rule_graph.dot' self.visualize_rule_graph_to_file(os.path.join(self._scheduler.visualize_to_dir(), rule_graph_name)) self._scheduler.assert_ruleset_valid()
def test_ruleset_with_superclass_of_selected_type_produced_fails(self): rules = [ (A, (Select(B),), noop), (B, (Select(SubA),), noop) ] validator = RulesetValidator(RuleIndex.create(rules, tuple()), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (C,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 2 (A, (Select(B),), noop): depends on unfulfillable (B, (Select(SubA),), noop) of C with subject types: C (B, (Select(SubA),), noop): no matches for Select(SubA) with subject types: C """).strip(), str(cm.exception))
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self): rules = [ (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop), ] intrinsics = [ (B, C, noop), ] validator = RulesetValidator(RuleIndex.create(rules, intrinsics), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (A,)}) with self.assertRaises(ValueError) as cm: validator.validate() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop): Unreachable with subject types: Any """).strip(), str(cm.exception))
def test_root_tuple_removed_when_no_matches(self): rules = [ RootRule(C), RootRule(D), TaskRule(Exactly(A), [Select(C)], noop), TaskRule(Exactly(B), [Select(D), Select(A)], noop), ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: C, D // root entries "Select(A) for C" [color=blue] "Select(A) for C" -> {"(A, (Select(C),), noop) of C"} // internal entries "(A, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"} }""").strip(), fullgraph)
def test_depends_on_multiple_one_noop(self): rules = [ (B, (Select(A),), noop), (A, (Select(C),), noop), (A, (Select(SubA),), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) all_rules: (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph)
def test_ruleset_with_failure_due_to_incompatible_subject_for_intrinsic(self): rules = [ (D, (Select(C),), noop) ] intrinsics = [ (B, C, noop), ] validator = RulesetValidator(RuleIndex.create(rules, intrinsics), goal_to_product={}, root_subject_fns={k: lambda p: Select(p) for k in (A,)}) with self.assertRaises(ValueError) as cm: validator.validate() # This error message could note near matches like the intrinsic. self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (D, (Select(C),), noop): no matches for Select(C) with subject types: A """).strip(), str(cm.exception))
def test_successful_when_one_field_type_is_unfulfillable(self): # NB We may want this to be a warning, since it may not be intentional rules = [ (B, (Select(SubA),), noop), (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C))), noop) ] graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=D) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(D) for SubA => ((D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA,) all_rules: (B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA => ((B, (Select(SubA),), noop) of SubA, SubjectIsProduct(SubA), (B, (Select(SubA),), noop) of SubA,) }""").strip(), subgraph)
def test_noop_removal_full_single_subject_type(self): rules = [ # C is provided by an intrinsic, but only if the subject is B. (Exactly(A), (Select(C),), noop), (Exactly(A), tuple(), noop), ] intrinsics = [ (B, C, noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) fullgraph = graphmaker.full_graph() self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) all_rules: (Exactly(A), (), noop) of SubA => (,) }""").strip(), fullgraph)
def test_select_dependencies_with_matching_intrinsic(self): rules = [ (Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop), ] intrinsics = [ (B, C, noop), ] graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), root_subject_fns=_suba_root_subject_fns) subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) self.assert_equal_with_printing(dedent(""" { root_subject_types: (SubA,) root_rules: Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA,) all_rules: (Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA => (SubjectIsProduct(SubA), IntrinsicRule((C, B), noop) of C,) IntrinsicRule((C, B), noop) of C => (,) }""").strip(), subgraph)