def test_smallest_full_test(self): rules = _suba_root_rules + [ RootRule(SubA), TaskRule(Exactly(A), [Select(SubA)], noop) ] fullgraph = self.create_full_graph(RuleIndex.create(rules)) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} // internal entries "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} }""").strip(), fullgraph)
def test_no_include_trace_error_multiple_paths_raises_executionerror(self): rules = [ RootRule(B), nested_raise, ] scheduler = self.scheduler(rules, include_trace_on_error=False) with self.assertRaises(ExecutionError) as cm: list(scheduler.product_request(A, subjects=[B(), B()])) self.assert_equal_with_printing( dedent(''' 2 Exceptions encountered: Exception: An exception for B Exception: An exception for B''').lstrip(), str(cm.exception))
def rules(cls): return ( *super().rules(), *repl_rules(), *python_repl.rules(), *pex.rules(), *download_pex_bin.rules(), *archive.rules(), *external_tool.rules(), *importable_python_sources.rules(), *pex_from_targets.rules(), *python_native_code.rules(), *strip_source_roots.rules(), *subprocess_environment.rules(), RootRule(PythonRepl), )
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self): rules = [ RootRule(A), TaskRule(A, [SelectDependencies(B, SubA, field_types=(D,))], noop), ] validator = self.create_validator({}, rules) with self.assertRaises(ValueError) as cm: validator.assert_ruleset_valid() self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop): Unreachable with subject types: Any """).strip(), str(cm.exception))
def rules(cls): return ( *super().rules(), *pytest_coverage.rules(), *pytest_runner.rules(), *download_pex_bin.rules(), *determine_source_files.rules(), *importable_python_sources.rules(), *pex.rules(), *pex_from_targets.rules(), *python_native_code.rules(), *strip_source_roots.rules(), *subprocess_environment.rules(), subsystem_rule(TestOptions), RootRule(PytestRunner), )
def test_streaming_workunits_parent_id_and_rule_metadata(self): rules = [RootRule(Input), rule_one_function, rule_two, rule_three, rule_four] scheduler = self.mk_scheduler( rules, include_trace_on_error=False, should_report_workunits=True ) tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, callbacks=[tracker.add], report_interval_seconds=0.01 ) with handler.session(): i = Input() scheduler.product_request(Beta, subjects=[i]) assert tracker.finished # rule_one should complete well-after the other rules because of the artificial delay in it caused by the sleep(). assert {item["name"] for item in tracker.finished_workunit_chunks[0]} == { "rule_two", "rule_three", "rule_four", } # Because of the artificial delay in rule_one, it should have time to be reported as # started but not yet finished. started = list(itertools.chain.from_iterable(tracker.started_workunit_chunks)) assert len(list(item for item in started if item["name"] == "rule_one")) > 0 assert {item["name"] for item in tracker.finished_workunit_chunks[1]} == {"rule_one"} finished = list(itertools.chain.from_iterable(tracker.finished_workunit_chunks)) r1 = next(item for item in finished if item["name"] == "rule_one") r2 = next(item for item in finished if item["name"] == "rule_two") r3 = next(item for item in finished if item["name"] == "rule_three") r4 = next(item for item in finished if item["name"] == "rule_four") # rule_one should have no parent_id because its actual parent workunit was filted based on level assert r1.get("parent_id", None) is None assert r2["parent_id"] == r1["span_id"] assert r3["parent_id"] == r1["span_id"] assert r4["parent_id"] == r2["span_id"] assert r3["description"] == "Rule number 3" assert r4["description"] == "Rule number 4" assert r4["level"] == "INFO"
def test_ruleset_with_superclass_of_selected_type_produced_fails(self): rules = [ RootRule(C), TaskRule(A, [Select(B)], noop), TaskRule(B, [Select(SubA)], noop) ] with self.assertRaises(ValueError) as cm: create_scheduler(rules) self.assert_equal_with_printing(dedent(""" Rules with errors: 2 (A, (Select(B),), noop): depends on unfulfillable (B, (Select(SubA),), noop) of C with subject types: C (B, (Select(SubA),), noop): no matches for Select(SubA) with subject types: C """).strip(), str(cm.exception))
def test_ruleset_with_failure_due_to_incompatible_subject_for_singleton(self): rules = [ RootRule(A), TaskRule(D, [Select(C)], noop), SingletonRule(B, B()), ] with self.assertRaises(ValueError) as cm: create_scheduler(rules) # This error message could note near matches like the singleton. self.assert_equal_with_printing(dedent(""" Rules with errors: 1 (D, (Select(C),), noop): no matches for Select(C) with subject types: A """).strip(), str(cm.exception))
def test_ruleset_with_superclass_of_selected_type_produced_fails(self): rules = [ RootRule(C), TaskRule(A, [Select(B)], noop), TaskRule(B, [Select(SubA)], noop) ] with self.assertRaises(Exception) as cm: create_scheduler(rules) self.assert_equal_with_printing( dedent(""" Rules with errors: 2 (A, [Select(B)], noop): No rule was available to compute B with parameter type C (B, [Select(SubA)], noop): No rule was available to compute SubA with parameter type C """).strip(), str(cm.exception))
def rules(cls): return ( *super().rules(), *black_rules(), create_pex, create_subprocess_encoding_environment, create_pex_native_build_environment, download_pex_bin, RootRule(CreatePex), RootRule(FormattablePythonTarget), RootRule(Black), RootRule(BlackSetup), RootRule(PythonSetup), RootRule(PythonNativeCode), RootRule(SubprocessEnvironment), )
def rules(cls): return ( *super().rules(), *isort_rules(), create_pex, create_subprocess_encoding_environment, create_pex_native_build_environment, download_pex_bin, RootRule(CreatePex), RootRule(Isort), RootRule(IsortSetup), RootRule(IsortTarget), RootRule(PythonSetup), RootRule(PythonNativeCode), RootRule(SubprocessEnvironment), )
def test_ruleset_with_failure_due_to_incompatible_subject_for_singleton( self): rules = [ RootRule(A), TaskRule(D, [Select(C)], noop), SingletonRule(B, B()), ] with self.assertRaises(Exception) as cm: create_scheduler(rules) # This error message could note near matches like the singleton. self.assert_equal_with_printing( dedent(""" Rules with errors: 1 (D, [Select(C)], noop): No rule was available to compute C with parameter type A """).strip(), str(cm.exception))
def create_graph_rules(address_mapper, symbol_table): """Creates tasks used to parse Structs from BUILD files. :param address_mapper_key: The subject key for an AddressMapper instance. :param symbol_table: A SymbolTable instance to provide symbols for Address lookups. """ symbol_table_constraint = symbol_table.constraint() return [ TaskRule( BuildFilesCollection, [SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir, ))], BuildFilesCollection), # A singleton to provide the AddressMapper. SingletonRule(AddressMapper, address_mapper), # Support for resolving Structs from Addresses. TaskRule(symbol_table_constraint, [ Select(AddressMapper), Select(UnhydratedStruct), SelectDependencies(symbol_table_constraint, UnhydratedStruct, field_types=(Address, )) ], hydrate_struct), resolve_unhydrated_struct, TaskRule(HydratedStructs, [ SelectDependencies(symbol_table_constraint, BuildFileAddresses, field_types=(Address, ), field='addresses') ], HydratedStructs), # BUILD file parsing. parse_address_family, build_files, buildfile_path_globs_for_dir, # Spec handling: locate directories that contain build files, and request # AddressFamilies for each of them. addresses_from_address_families, filter_build_dirs, spec_to_globs, # Root rules representing parameters that might be provided via root subjects. RootRule(Address), RootRule(BuildFileAddress), RootRule(BuildFileAddresses), RootRule(AscendantAddresses), RootRule(DescendantAddresses), RootRule(SiblingAddresses), RootRule(SingleAddress), ]
def test_javac_version_example(self): scheduler = self.mk_scheduler_in_example_fs([ RootRule(JavacVersionExecutionRequest), process_request_from_javac_version, get_javac_version_output, ]) request = JavacVersionExecutionRequest( BinaryLocation('/usr/bin/javac')) self.assertEqual( repr(request), "JavacVersionExecutionRequest(binary_location=BinaryLocation(bin_path='/usr/bin/javac'))" ) results = self.execute(scheduler, JavacVersionOutput, request) self.assertEqual(1, len(results)) javac_version_output = results[0] self.assertIn('javac', javac_version_output.value)
def test_ruleset_with_missing_product_type(self): @rule def a_from_b(b: B) -> A: pass rules = [RootRule(SubA), a_from_b] with self.assertRaises(Exception) as cm: create_scheduler(rules) self.assert_equal_with_printing( dedent(f"""\ Rules with errors: 1 {fmt_rule(a_from_b)}: No rule was available to compute B with parameter type SubA """).strip(), str(cm.exception), )
def _init_engine(cls): if cls._scheduler is not None: return # NB: This uses the long form of initialization because it needs to directly specify # `cls.alias_groups` rather than having them be provided by bootstrap options. graph_session = EngineInitializer.setup_legacy_graph_extended( pants_ignore_patterns=None, workdir=cls._pants_workdir(), build_file_imports_behavior='allow', native=init_native(), build_configuration=cls.build_config(), build_ignore_patterns=None, # Required for sources_for: rules=[RootRule(SourcesField)], ).new_session() cls._scheduler = graph_session.scheduler_session cls._build_graph, cls._address_mapper = graph_session.create_build_graph( TargetRoots([]), cls._build_root())
def test_trace_multi(self): # Tests that when multiple distinct failures occur, they are each rendered. rules = [ RootRule(B), TaskRule(D, [Select(B)], nested_raise), TaskRule(C, [Select(B)], nested_raise), TaskRule(A, [Select(C), Select(D)], A), ] scheduler = self.scheduler(rules, include_trace_on_error=True) with self.assertRaises(Exception) as cm: list(scheduler.product_request(A, subjects=[(B())])) self.assert_equal_with_printing(dedent(''' Received unexpected Throw state(s): Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A) Computing Task(A, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A) Computing Task(nested_raise, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =D) Throw(An exception for B) Traceback (most recent call last): File LOCATION-INFO, in call val = func(*args) File LOCATION-INFO, in nested_raise fn_raises(x) File LOCATION-INFO, in fn_raises raise Exception('An exception for {}'.format(type(x).__name__)) Exception: An exception for B Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A) Computing Task(A, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A) Computing Task(nested_raise, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =C) Throw(An exception for B) Traceback (most recent call last): File LOCATION-INFO, in call val = func(*args) File LOCATION-INFO, in nested_raise fn_raises(x) File LOCATION-INFO, in fn_raises raise Exception('An exception for {}'.format(type(x).__name__)) Exception: An exception for B ''').lstrip()+'\n', remove_locations_from_traceback(str(cm.exception)))
def rules(cls): # TODO: A convenient way to bring in all the rules needed to build a pex without # having to enumerate them here. return super().rules() + [ create_python_awslambda, setup_lambdex, create_pex, create_pex_native_build_environment, create_subprocess_encoding_environment, strip_source_root, download_pex_bin, inject_init, create_pex_from_target_closure, RootRule(Digest), RootRule(SourceRootConfig), RootRule(PythonSetup), RootRule(PythonNativeCode), RootRule(SubprocessEnvironment), RootRule(Lambdex), RootRule(LambdexSetup), RootRule(PythonAWSLambdaAdaptor), ]
def test_smallest_full_test(self): @rule(A, [SubA]) def a_from_suba(suba): pass rules = _suba_root_rules + [ RootRule(SubA), a_from_suba, ] fullgraph = self.create_full_graph(rules) self.assert_equal_with_printing(dedent(""" digraph { // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {"(A, [SubA], a_from_suba()) for SubA"} // internal entries "(A, [SubA], a_from_suba()) for SubA" -> {"Param(SubA)"} }""").strip(), fullgraph)
def rules(cls): return ( *super().rules(), *flake8_rules(), *download_pex_bin.rules(), *pex.rules(), *python_native_code.rules(), *subprocess_environment.rules(), RootRule(CreatePex), RootRule(Flake8), RootRule(Flake8Target), RootRule(PythonSetup), RootRule(PythonNativeCode), RootRule(SubprocessEnvironment), )
def test_ruleset_with_missing_product_type(self): @rule def a_from_b(b: B) -> A: pass rules = [RootRule(SubA), a_from_b] with self.assertRaises(Exception) as cm: create_scheduler(rules) self.assert_equal_with_printing( dedent( f"""\ Rules with errors: 1 {fmt_rule(a_from_b)}: No rule was able to compute B. No installed rules return the type B: Is the rule that you're expecting to run registered? If that type should be provided from outside the rule graph, consider declaring RootRule(B). """ ).strip(), str(cm.exception), )
def rules(cls): return super(SchedulerTest, cls).rules() + [ RootRule(A), # B is both a RootRule and an intermediate product here. RootRule(B), RootRule(C), consumes_a_and_b, transitive_b_c, transitive_coroutine_rule, RootRule(UnionWrapper), UnionRule(UnionBase, UnionA), RootRule(UnionA), select_union_a, UnionRule(union_base=UnionBase, union_member=UnionB), RootRule(UnionB), select_union_b, a_union_test, a_typecheck_fail_test, RootRule(TypeCheckFailWrapper), ]
def test_include_trace_error_raises_error_with_trace(self): rules = [RootRule(B), TaskRule(A, [Select(B)], nested_raise)] scheduler = self.scheduler(rules, include_trace_on_error=True) with self.assertRaises(Exception) as cm: list(scheduler.product_request(A, subjects=[(B())])) self.assert_equal_with_printing( dedent(''' Received unexpected Throw state(s): Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A) Computing Task(<function nested_raise at 0xEEEEEEEEE>, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A) Throw(An exception for B) Traceback (most recent call last): File LOCATION-INFO, in extern_invoke_runnable val = runnable(*args) File LOCATION-INFO, in nested_raise fn_raises(x) File LOCATION-INFO, in fn_raises raise Exception('An exception for {}'.format(type(x).__name__)) Exception: An exception for B ''').lstrip() + '\n', remove_locations_from_traceback(str(cm.exception)))
def rules(cls): return super().rules() + [ RootRule(A), # B is both a RootRule and an intermediate product here. RootRule(B), RootRule(C), RootRule(UnionX), no_docstring_test_rule, consumes_a_and_b, transitive_b_c, transitive_coroutine_rule, RootRule(UnionWrapper), UnionRule(UnionBase, UnionA), UnionRule(NoDocstringUnion, UnionX), RootRule(UnionA), select_union_a, UnionRule(union_base=UnionBase, union_member=UnionB), RootRule(UnionB), select_union_b, a_union_test, ]
def test_include_trace_error_raises_error_with_trace(self): rules = [ RootRule(B), nested_raise, ] scheduler = self.scheduler(rules, include_trace_on_error=True) with self.assertRaises(ExecutionError) as cm: list(scheduler.product_request(A, subjects=[(B())])) self.assert_equal_with_printing( dedent(""" 1 Exception encountered: Traceback (most recent call last): File LOCATION-INFO, in nested_raise fn_raises(x) File LOCATION-INFO, in fn_raises raise Exception(f"An exception for {type(x).__name__}") Exception: An exception for B """).lstrip(), remove_locations_from_traceback(str(cm.exception)), )
def test_smallest_full_test(self): @rule def a_from_suba(suba: SubA) -> A: pass rules = _suba_root_rules + [ RootRule(SubA), a_from_suba, ] fullgraph = self.create_full_graph(rules) self.assert_equal_with_printing( dedent(f"""\ digraph {{ // root subject types: SubA // root entries "Select(A) for SubA" [color=blue] "Select(A) for SubA" -> {{"{fmt_rule(a_from_suba)} for SubA"}} // internal entries "{fmt_rule(a_from_suba)} for SubA" -> {{"Param(SubA)"}} }}""").strip(), fullgraph, )
def test_streaming_workunit_log_levels(self) -> None: rules = [ RootRule(Input), rule_one_function, rule_two, rule_three, rule_four ] scheduler = self.mk_scheduler(rules, include_trace_on_error=False, should_report_workunits=True) tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, callbacks=[tracker.add], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.TRACE, ) with handler.session(): i = Input() scheduler.product_request(Beta, subjects=[i]) assert tracker.finished finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) # With the max_workunit_verbosity set to TRACE, we should see the workunit corresponding to the Select node. select = next( item for item in finished if item["name"] not in { "canonical_rule_one", "pants.engine.internals.engine_test.rule_two", "pants.engine.internals.engine_test.rule_three", "pants.engine.internals.engine_test.rule_four", }) assert select["name"] == "select" assert select["level"] == "DEBUG" r1 = next(item for item in finished if item["name"] == "canonical_rule_one") assert r1["parent_id"] == select["span_id"]
def test_engine_aware_none_case(self): @dataclass(frozen=True) # If level() returns None, the engine shouldn't try to set # a new workunit level. class ModifiedOutput(EngineAware): _level: Optional[LogLevel] val: int def level(self): return self._level @rule(desc="a_rule") def a_rule(n: int) -> ModifiedOutput: return ModifiedOutput(val=n, _level=None) rules = [a_rule, RootRule(int)] scheduler = self.mk_scheduler(rules, include_trace_on_error=False, should_report_workunits=True) tracker = WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, callbacks=[tracker.add], report_interval_seconds=0.01, max_workunit_verbosity=LogLevel.DEBUG, ) with handler.session(): scheduler.product_request(ModifiedOutput, subjects=[0]) finished = list( itertools.chain.from_iterable(tracker.finished_workunit_chunks)) workunit = next( item for item in finished if item["name"] == "pants.engine.internals.engine_test.a_rule") assert workunit["level"] == "DEBUG"
def test_smallest_full_test(self): @rule def a_from_suba(suba: SubA) -> A: pass rules = _suba_root_rules + [ RootRule(SubA), a_from_suba, ] fullgraph = self.create_full_graph(rules) assert_equal_graph_output( self, dedent(f"""\ digraph {{ // root subject types: SubA // root entries {fmt_non_param_edge(A, SubA)} {fmt_non_param_edge(A, SubA, RuleFormatRequest(a_from_suba))} // internal entries {fmt_param_edge(SubA, SubA, RuleFormatRequest(a_from_suba))} }}""").strip(), fullgraph, )
def test_streaming_workunits_reporting(self): rules = [fib, RootRule(int)] scheduler = self.mk_scheduler( rules, include_trace_on_error=False, should_report_workunits=True ) tracker = self.WorkunitTracker() handler = StreamingWorkunitHandler( scheduler, callbacks=[tracker.add], report_interval_seconds=0.01 ) with handler.session(): scheduler.product_request(Fib, subjects=[0]) # The execution of the single named @rule "fib" should be providing this one workunit. self.assertEquals(len(tracker.workunits), 1) tracker.workunits = [] with handler.session(): scheduler.product_request(Fib, subjects=[10]) # Requesting a bigger fibonacci number will result in more rule executions and thus more reported workunits. # In this case, we expect 10 invocations of the `fib` rule. assert len(tracker.workunits) == 10 assert tracker.finished