def test_smallest_full_test(self):
rules = _suba_root_rules + [
RootRule(SubA),
TaskRule(Exactly(A), [Select(SubA)], noop)
]
fullgraph = self.create_full_graph(RuleIndex.create(rules))
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"}
// internal entries
"(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"}
}""").strip(), fullgraph)
def test_no_include_trace_error_multiple_paths_raises_executionerror(self):
rules = [
RootRule(B),
nested_raise,
]
scheduler = self.scheduler(rules, include_trace_on_error=False)
with self.assertRaises(ExecutionError) as cm:
list(scheduler.product_request(A, subjects=[B(), B()]))
self.assert_equal_with_printing(
dedent('''
2 Exceptions encountered:
Exception: An exception for B
Exception: An exception for B''').lstrip(), str(cm.exception))
def rules(cls):
return (
*super().rules(),
*repl_rules(),
*python_repl.rules(),
*pex.rules(),
*download_pex_bin.rules(),
*archive.rules(),
*external_tool.rules(),
*importable_python_sources.rules(),
*pex_from_targets.rules(),
*python_native_code.rules(),
*strip_source_roots.rules(),
*subprocess_environment.rules(),
RootRule(PythonRepl),
)
def test_ruleset_unreachable_due_to_product_of_select_dependencies(self):
rules = [
RootRule(A),
TaskRule(A, [SelectDependencies(B, SubA, field_types=(D,))], noop),
]
validator = self.create_validator({}, rules)
with self.assertRaises(ValueError) as cm:
validator.assert_ruleset_valid()
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop):
Unreachable with subject types: Any
""").strip(),
str(cm.exception))
def rules(cls):
return (
*super().rules(),
*pytest_coverage.rules(),
*pytest_runner.rules(),
*download_pex_bin.rules(),
*determine_source_files.rules(),
*importable_python_sources.rules(),
*pex.rules(),
*pex_from_targets.rules(),
*python_native_code.rules(),
*strip_source_roots.rules(),
*subprocess_environment.rules(),
subsystem_rule(TestOptions),
RootRule(PytestRunner),
)
def test_streaming_workunits_parent_id_and_rule_metadata(self):
rules = [RootRule(Input), rule_one_function, rule_two, rule_three, rule_four]
scheduler = self.mk_scheduler(
rules, include_trace_on_error=False, should_report_workunits=True
)
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler, callbacks=[tracker.add], report_interval_seconds=0.01
)
with handler.session():
i = Input()
scheduler.product_request(Beta, subjects=[i])
assert tracker.finished
# rule_one should complete well-after the other rules because of the artificial delay in it caused by the sleep().
assert {item["name"] for item in tracker.finished_workunit_chunks[0]} == {
"rule_two",
"rule_three",
"rule_four",
}
# Because of the artificial delay in rule_one, it should have time to be reported as
# started but not yet finished.
started = list(itertools.chain.from_iterable(tracker.started_workunit_chunks))
assert len(list(item for item in started if item["name"] == "rule_one")) > 0
assert {item["name"] for item in tracker.finished_workunit_chunks[1]} == {"rule_one"}
finished = list(itertools.chain.from_iterable(tracker.finished_workunit_chunks))
r1 = next(item for item in finished if item["name"] == "rule_one")
r2 = next(item for item in finished if item["name"] == "rule_two")
r3 = next(item for item in finished if item["name"] == "rule_three")
r4 = next(item for item in finished if item["name"] == "rule_four")
# rule_one should have no parent_id because its actual parent workunit was filted based on level
assert r1.get("parent_id", None) is None
assert r2["parent_id"] == r1["span_id"]
assert r3["parent_id"] == r1["span_id"]
assert r4["parent_id"] == r2["span_id"]
assert r3["description"] == "Rule number 3"
assert r4["description"] == "Rule number 4"
assert r4["level"] == "INFO"
def test_ruleset_with_superclass_of_selected_type_produced_fails(self):
rules = [
RootRule(C),
TaskRule(A, [Select(B)], noop),
TaskRule(B, [Select(SubA)], noop)
]
with self.assertRaises(ValueError) as cm:
create_scheduler(rules)
self.assert_equal_with_printing(dedent("""
Rules with errors: 2
(A, (Select(B),), noop):
depends on unfulfillable (B, (Select(SubA),), noop) of C with subject types: C
(B, (Select(SubA),), noop):
no matches for Select(SubA) with subject types: C
""").strip(),
str(cm.exception))
def test_ruleset_with_failure_due_to_incompatible_subject_for_singleton(self):
rules = [
RootRule(A),
TaskRule(D, [Select(C)], noop),
SingletonRule(B, B()),
]
with self.assertRaises(ValueError) as cm:
create_scheduler(rules)
# This error message could note near matches like the singleton.
self.assert_equal_with_printing(dedent("""
Rules with errors: 1
(D, (Select(C),), noop):
no matches for Select(C) with subject types: A
""").strip(),
str(cm.exception))
def test_ruleset_with_superclass_of_selected_type_produced_fails(self):
rules = [
RootRule(C),
TaskRule(A, [Select(B)], noop),
TaskRule(B, [Select(SubA)], noop)
]
with self.assertRaises(Exception) as cm:
create_scheduler(rules)
self.assert_equal_with_printing(
dedent("""
Rules with errors: 2
(A, [Select(B)], noop):
No rule was available to compute B with parameter type C
(B, [Select(SubA)], noop):
No rule was available to compute SubA with parameter type C
""").strip(), str(cm.exception))
def rules(cls):
return (
*super().rules(),
*black_rules(),
create_pex,
create_subprocess_encoding_environment,
create_pex_native_build_environment,
download_pex_bin,
RootRule(CreatePex),
RootRule(FormattablePythonTarget),
RootRule(Black),
RootRule(BlackSetup),
RootRule(PythonSetup),
RootRule(PythonNativeCode),
RootRule(SubprocessEnvironment),
)
def rules(cls):
return (
*super().rules(),
*isort_rules(),
create_pex,
create_subprocess_encoding_environment,
create_pex_native_build_environment,
download_pex_bin,
RootRule(CreatePex),
RootRule(Isort),
RootRule(IsortSetup),
RootRule(IsortTarget),
RootRule(PythonSetup),
RootRule(PythonNativeCode),
RootRule(SubprocessEnvironment),
)
def test_ruleset_with_failure_due_to_incompatible_subject_for_singleton(
self):
rules = [
RootRule(A),
TaskRule(D, [Select(C)], noop),
SingletonRule(B, B()),
]
with self.assertRaises(Exception) as cm:
create_scheduler(rules)
# This error message could note near matches like the singleton.
self.assert_equal_with_printing(
dedent("""
Rules with errors: 1
(D, [Select(C)], noop):
No rule was available to compute C with parameter type A
""").strip(), str(cm.exception))
def create_graph_rules(address_mapper, symbol_table):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table: A SymbolTable instance to provide symbols for Address lookups.
"""
symbol_table_constraint = symbol_table.constraint()
return [
TaskRule(
BuildFilesCollection,
[SelectDependencies(BuildFiles, BuildDirs, field_types=(Dir, ))],
BuildFilesCollection),
# A singleton to provide the AddressMapper.
SingletonRule(AddressMapper, address_mapper),
# Support for resolving Structs from Addresses.
TaskRule(symbol_table_constraint, [
Select(AddressMapper),
Select(UnhydratedStruct),
SelectDependencies(symbol_table_constraint,
UnhydratedStruct,
field_types=(Address, ))
], hydrate_struct),
resolve_unhydrated_struct,
TaskRule(HydratedStructs, [
SelectDependencies(symbol_table_constraint,
BuildFileAddresses,
field_types=(Address, ),
field='addresses')
], HydratedStructs),
# BUILD file parsing.
parse_address_family,
build_files,
buildfile_path_globs_for_dir,
# Spec handling: locate directories that contain build files, and request
# AddressFamilies for each of them.
addresses_from_address_families,
filter_build_dirs,
spec_to_globs,
# Root rules representing parameters that might be provided via root subjects.
RootRule(Address),
RootRule(BuildFileAddress),
RootRule(BuildFileAddresses),
RootRule(AscendantAddresses),
RootRule(DescendantAddresses),
RootRule(SiblingAddresses),
RootRule(SingleAddress),
]
def test_javac_version_example(self):
scheduler = self.mk_scheduler_in_example_fs([
RootRule(JavacVersionExecutionRequest),
process_request_from_javac_version,
get_javac_version_output,
])
request = JavacVersionExecutionRequest(
BinaryLocation('/usr/bin/javac'))
self.assertEqual(
repr(request),
"JavacVersionExecutionRequest(binary_location=BinaryLocation(bin_path='/usr/bin/javac'))"
)
results = self.execute(scheduler, JavacVersionOutput, request)
self.assertEqual(1, len(results))
javac_version_output = results[0]
self.assertIn('javac', javac_version_output.value)
def test_ruleset_with_missing_product_type(self):
@rule
def a_from_b(b: B) -> A:
pass
rules = [RootRule(SubA), a_from_b]
with self.assertRaises(Exception) as cm:
create_scheduler(rules)
self.assert_equal_with_printing(
dedent(f"""\
Rules with errors: 1
{fmt_rule(a_from_b)}:
No rule was available to compute B with parameter type SubA
""").strip(),
str(cm.exception),
)
def _init_engine(cls):
if cls._scheduler is not None:
return
# NB: This uses the long form of initialization because it needs to directly specify
# `cls.alias_groups` rather than having them be provided by bootstrap options.
graph_session = EngineInitializer.setup_legacy_graph_extended(
pants_ignore_patterns=None,
workdir=cls._pants_workdir(),
build_file_imports_behavior='allow',
native=init_native(),
build_configuration=cls.build_config(),
build_ignore_patterns=None,
# Required for sources_for:
rules=[RootRule(SourcesField)],
).new_session()
cls._scheduler = graph_session.scheduler_session
cls._build_graph, cls._address_mapper = graph_session.create_build_graph(
TargetRoots([]), cls._build_root())
def test_trace_multi(self):
# Tests that when multiple distinct failures occur, they are each rendered.
rules = [
RootRule(B),
TaskRule(D, [Select(B)], nested_raise),
TaskRule(C, [Select(B)], nested_raise),
TaskRule(A, [Select(C), Select(D)], A),
]
scheduler = self.scheduler(rules, include_trace_on_error=True)
with self.assertRaises(Exception) as cm:
list(scheduler.product_request(A, subjects=[(B())]))
self.assert_equal_with_printing(dedent('''
Received unexpected Throw state(s):
Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Computing Task(A, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Computing Task(nested_raise, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =D)
Throw(An exception for B)
Traceback (most recent call last):
File LOCATION-INFO, in call
val = func(*args)
File LOCATION-INFO, in nested_raise
fn_raises(x)
File LOCATION-INFO, in fn_raises
raise Exception('An exception for {}'.format(type(x).__name__))
Exception: An exception for B
Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Computing Task(A, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Computing Task(nested_raise, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =C)
Throw(An exception for B)
Traceback (most recent call last):
File LOCATION-INFO, in call
val = func(*args)
File LOCATION-INFO, in nested_raise
fn_raises(x)
File LOCATION-INFO, in fn_raises
raise Exception('An exception for {}'.format(type(x).__name__))
Exception: An exception for B
''').lstrip()+'\n',
remove_locations_from_traceback(str(cm.exception)))
def rules(cls):
# TODO: A convenient way to bring in all the rules needed to build a pex without
# having to enumerate them here.
return super().rules() + [
create_python_awslambda,
setup_lambdex,
create_pex,
create_pex_native_build_environment,
create_subprocess_encoding_environment,
strip_source_root,
download_pex_bin,
inject_init,
create_pex_from_target_closure,
RootRule(Digest),
RootRule(SourceRootConfig),
RootRule(PythonSetup),
RootRule(PythonNativeCode),
RootRule(SubprocessEnvironment),
RootRule(Lambdex),
RootRule(LambdexSetup),
RootRule(PythonAWSLambdaAdaptor),
]
def test_smallest_full_test(self):
@rule(A, [SubA])
def a_from_suba(suba):
pass
rules = _suba_root_rules + [
RootRule(SubA),
a_from_suba,
]
fullgraph = self.create_full_graph(rules)
self.assert_equal_with_printing(dedent("""
digraph {
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {"(A, [SubA], a_from_suba()) for SubA"}
// internal entries
"(A, [SubA], a_from_suba()) for SubA" -> {"Param(SubA)"}
}""").strip(), fullgraph)
def rules(cls):
return (
*super().rules(),
*flake8_rules(),
*download_pex_bin.rules(),
*pex.rules(),
*python_native_code.rules(),
*subprocess_environment.rules(),
RootRule(CreatePex),
RootRule(Flake8),
RootRule(Flake8Target),
RootRule(PythonSetup),
RootRule(PythonNativeCode),
RootRule(SubprocessEnvironment),
)
def test_ruleset_with_missing_product_type(self):
@rule
def a_from_b(b: B) -> A:
pass
rules = [RootRule(SubA), a_from_b]
with self.assertRaises(Exception) as cm:
create_scheduler(rules)
self.assert_equal_with_printing(
dedent(
f"""\
Rules with errors: 1
{fmt_rule(a_from_b)}:
No rule was able to compute B. No installed rules return the type B: Is the rule that you're expecting to run registered? If that type should be provided from outside the rule graph, consider declaring RootRule(B).
"""
).strip(),
str(cm.exception),
)
def rules(cls):
return super(SchedulerTest, cls).rules() + [
RootRule(A),
# B is both a RootRule and an intermediate product here.
RootRule(B),
RootRule(C),
consumes_a_and_b,
transitive_b_c,
transitive_coroutine_rule,
RootRule(UnionWrapper),
UnionRule(UnionBase, UnionA),
RootRule(UnionA),
select_union_a,
UnionRule(union_base=UnionBase, union_member=UnionB),
RootRule(UnionB),
select_union_b,
a_union_test,
a_typecheck_fail_test,
RootRule(TypeCheckFailWrapper),
]
def test_include_trace_error_raises_error_with_trace(self):
rules = [RootRule(B), TaskRule(A, [Select(B)], nested_raise)]
scheduler = self.scheduler(rules, include_trace_on_error=True)
with self.assertRaises(Exception) as cm:
list(scheduler.product_request(A, subjects=[(B())]))
self.assert_equal_with_printing(
dedent('''
Received unexpected Throw state(s):
Computing Select(<pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Computing Task(<function nested_raise at 0xEEEEEEEEE>, <pants_test.engine.test_engine.B object at 0xEEEEEEEEE>, =A)
Throw(An exception for B)
Traceback (most recent call last):
File LOCATION-INFO, in extern_invoke_runnable
val = runnable(*args)
File LOCATION-INFO, in nested_raise
fn_raises(x)
File LOCATION-INFO, in fn_raises
raise Exception('An exception for {}'.format(type(x).__name__))
Exception: An exception for B
''').lstrip() + '\n', remove_locations_from_traceback(str(cm.exception)))
def rules(cls):
return super().rules() + [
RootRule(A),
# B is both a RootRule and an intermediate product here.
RootRule(B),
RootRule(C),
RootRule(UnionX),
no_docstring_test_rule,
consumes_a_and_b,
transitive_b_c,
transitive_coroutine_rule,
RootRule(UnionWrapper),
UnionRule(UnionBase, UnionA),
UnionRule(NoDocstringUnion, UnionX),
RootRule(UnionA),
select_union_a,
UnionRule(union_base=UnionBase, union_member=UnionB),
RootRule(UnionB),
select_union_b,
a_union_test,
]
def test_include_trace_error_raises_error_with_trace(self):
rules = [
RootRule(B),
nested_raise,
]
scheduler = self.scheduler(rules, include_trace_on_error=True)
with self.assertRaises(ExecutionError) as cm:
list(scheduler.product_request(A, subjects=[(B())]))
self.assert_equal_with_printing(
dedent("""
1 Exception encountered:
Traceback (most recent call last):
File LOCATION-INFO, in nested_raise
fn_raises(x)
File LOCATION-INFO, in fn_raises
raise Exception(f"An exception for {type(x).__name__}")
Exception: An exception for B
""").lstrip(),
remove_locations_from_traceback(str(cm.exception)),
)
def test_smallest_full_test(self):
@rule
def a_from_suba(suba: SubA) -> A:
pass
rules = _suba_root_rules + [
RootRule(SubA),
a_from_suba,
]
fullgraph = self.create_full_graph(rules)
self.assert_equal_with_printing(
dedent(f"""\
digraph {{
// root subject types: SubA
// root entries
"Select(A) for SubA" [color=blue]
"Select(A) for SubA" -> {{"{fmt_rule(a_from_suba)} for SubA"}}
// internal entries
"{fmt_rule(a_from_suba)} for SubA" -> {{"Param(SubA)"}}
}}""").strip(),
fullgraph,
)
def test_streaming_workunit_log_levels(self) -> None:
rules = [
RootRule(Input), rule_one_function, rule_two, rule_three, rule_four
]
scheduler = self.mk_scheduler(rules,
include_trace_on_error=False,
should_report_workunits=True)
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler,
callbacks=[tracker.add],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.TRACE,
)
with handler.session():
i = Input()
scheduler.product_request(Beta, subjects=[i])
assert tracker.finished
finished = list(
itertools.chain.from_iterable(tracker.finished_workunit_chunks))
# With the max_workunit_verbosity set to TRACE, we should see the workunit corresponding to the Select node.
select = next(
item for item in finished if item["name"] not in {
"canonical_rule_one",
"pants.engine.internals.engine_test.rule_two",
"pants.engine.internals.engine_test.rule_three",
"pants.engine.internals.engine_test.rule_four",
})
assert select["name"] == "select"
assert select["level"] == "DEBUG"
r1 = next(item for item in finished
if item["name"] == "canonical_rule_one")
assert r1["parent_id"] == select["span_id"]
def test_engine_aware_none_case(self):
@dataclass(frozen=True)
# If level() returns None, the engine shouldn't try to set
# a new workunit level.
class ModifiedOutput(EngineAware):
_level: Optional[LogLevel]
val: int
def level(self):
return self._level
@rule(desc="a_rule")
def a_rule(n: int) -> ModifiedOutput:
return ModifiedOutput(val=n, _level=None)
rules = [a_rule, RootRule(int)]
scheduler = self.mk_scheduler(rules,
include_trace_on_error=False,
should_report_workunits=True)
tracker = WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler,
callbacks=[tracker.add],
report_interval_seconds=0.01,
max_workunit_verbosity=LogLevel.DEBUG,
)
with handler.session():
scheduler.product_request(ModifiedOutput, subjects=[0])
finished = list(
itertools.chain.from_iterable(tracker.finished_workunit_chunks))
workunit = next(
item for item in finished
if item["name"] == "pants.engine.internals.engine_test.a_rule")
assert workunit["level"] == "DEBUG"
def test_smallest_full_test(self):
@rule
def a_from_suba(suba: SubA) -> A:
pass
rules = _suba_root_rules + [
RootRule(SubA),
a_from_suba,
]
fullgraph = self.create_full_graph(rules)
assert_equal_graph_output(
self,
dedent(f"""\
digraph {{
// root subject types: SubA
// root entries
{fmt_non_param_edge(A, SubA)}
{fmt_non_param_edge(A, SubA, RuleFormatRequest(a_from_suba))}
// internal entries
{fmt_param_edge(SubA, SubA, RuleFormatRequest(a_from_suba))}
}}""").strip(),
fullgraph,
)
def test_streaming_workunits_reporting(self):
rules = [fib, RootRule(int)]
scheduler = self.mk_scheduler(
rules, include_trace_on_error=False, should_report_workunits=True
)
tracker = self.WorkunitTracker()
handler = StreamingWorkunitHandler(
scheduler, callbacks=[tracker.add], report_interval_seconds=0.01
)
with handler.session():
scheduler.product_request(Fib, subjects=[0])
# The execution of the single named @rule "fib" should be providing this one workunit.
self.assertEquals(len(tracker.workunits), 1)
tracker.workunits = []
with handler.session():
scheduler.product_request(Fib, subjects=[10])
# Requesting a bigger fibonacci number will result in more rule executions and thus more reported workunits.
# In this case, we expect 10 invocations of the `fib` rule.
assert len(tracker.workunits) == 10
assert tracker.finished
