def make_ht(nm: str) -> HydratedTarget:
if nm not in name_to_ht:
dep_hts = tuple(make_ht(dep) for dep in name_to_deps[nm])
name_to_ht[nm] = HydratedTarget(address=nm,
adaptor=TargetAdaptor(),
dependencies=dep_hts)
return name_to_ht[nm]
def test_single_non_test_target(self):
bfaddr = BuildFileAddress(None, 'bin', 'some/dir')
target_adaptor = PythonBinaryAdaptor(type_alias='python_binary')
with self.captured_logging(logging.INFO):
# Note that this is not the same error message the end user will see, as we're resolving
# union Get requests in run_rule, not the real engine. But this test still asserts that
# we error when we expect to error.
with self.assertRaisesRegex(
AssertionError,
r'Rule requested: .* which cannot be satisfied.'):
run_rule(
coordinator_of_tests,
rule_args=[
HydratedTarget(bfaddr.to_address(), target_adaptor,
()),
UnionMembership(
union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(
bfaddr_to_spec={
bfaddr:
SingleAddress(directory='some/dir',
name='bin'),
}),
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=PythonTestsAdaptor,
mock=lambda _: TestResult(status=Status.SUCCESS,
stdout='foo',
stderr=''),
),
],
)
def test_coordinator_python_test(self):
addr = Address.parse("some/target")
target_adaptor = PythonTestsAdaptor(type_alias='python_tests')
with self.captured_logging(logging.INFO):
result = run_rule(
coordinator_of_tests,
rule_args=[
HydratedTarget(addr, target_adaptor, ()),
UnionMembership(
union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(bfaddr_to_spec={}),
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=PythonTestsAdaptor,
mock=lambda _: TestResult(
status=Status.FAILURE, stdout='foo', stderr=''),
),
],
)
self.assertEqual(
result,
AddressAndTestResult(
addr, TestResult(status=Status.FAILURE,
stdout='foo',
stderr='')))
def test_globbed_non_test_target(self):
bfaddr = BuildFileAddress(None, 'bin', 'some/dir')
target_adaptor = PythonBinaryAdaptor(type_alias='python_binary')
with self.captured_logging(logging.INFO):
result = run_rule(
coordinator_of_tests,
rule_args=[
HydratedTarget(bfaddr.to_address(), target_adaptor, ()),
UnionMembership(
union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(
bfaddr_to_spec={
bfaddr: DescendantAddresses(directory='some/dir')
}),
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=PythonTestsAdaptor,
mock=lambda _: TestResult(
status=Status.SUCCESS, stdout='foo', stderr=''),
),
],
)
self.assertEqual(result,
AddressAndTestResult(bfaddr.to_address(), None))
def test_coordinator_python_test(self):
target_adaptor = PythonTestsAdaptor(type_alias='python_tests')
result = run_rule(coordinator_of_tests, HydratedTarget(Address.parse("some/target"), target_adaptor, ()), {
(PyTestResult, HydratedTarget): lambda _: PyTestResult(status=Status.FAILURE, stdout='foo'),
})
self.assertEqual(result, TestResult(status=Status.FAILURE, stdout='foo'))
def make_target(
*, name: str = "target", adaptor_type: Type[TargetAdaptor] = PythonTargetAdaptor
) -> HydratedTarget:
return HydratedTarget(
address=f"src/{name}",
adaptor=adaptor_type(sources=(), name=name),
dependencies=()
)
def test_coordinator_unknown_test(self):
target_adaptor = PythonTestsAdaptor(type_alias='unknown_tests')
with self.assertRaises(Exception) as cm:
run_rule(coordinator_of_tests, HydratedTarget(Address.parse("some/target"), target_adaptor, ()), {
(PyTestResult, HydratedTarget): lambda _: PyTestResult(status=Status.FAILURE, stdout='foo'),
})
self.assertEqual(str(cm.exception), "Didn't know how to run tests for type unknown_tests")
async def get_exporting_owner(
owned_dependency: OwnedDependency) -> ExportedTarget:
"""Find the exported target that owns the given target (and therefore exports it).
The owner of T (i.e., the exported target in whose artifact T's code is published) is:
1. An exported target that depends on T (or is T itself).
2. Is T's closest filesystem ancestor among those satisfying 1.
If there are multiple such exported targets at the same degree of ancestry, the ownership
is ambiguous and an error is raised. If there is no exported target that depends on T
and is its ancestor, then there is no owner and an error is raised.
"""
hydrated_target = owned_dependency.hydrated_target
ancestor_addrs = AscendantAddresses(
hydrated_target.adaptor.address.spec_path)
ancestor_tgts = await Get[HydratedTargets](AddressSpecs(
(ancestor_addrs, )))
# Note that addresses sort by (spec_path, target_name), and all these targets are
# ancestors of the given target, i.e., their spec_paths are all prefixes. So sorting by
# address will effectively sort by closeness of ancestry to the given target.
exported_ancestor_tgts = sorted(
[t.adaptor for t in ancestor_tgts if _is_exported(t)],
key=lambda adaptor: adaptor.address,
reverse=True,
)
exported_ancestor_iter = iter(exported_ancestor_tgts)
for exported_ancestor in exported_ancestor_iter:
tht = await Get[TransitiveHydratedTargets](Addresses(
[exported_ancestor.address]))
if hydrated_target in tht.closure:
owner = exported_ancestor
# Find any exported siblings of owner that also depend on hydrated_target. They have the
# same spec_path as it, so they must immediately follow it in ancestor_iter.
sibling_owners = []
sibling = next(exported_ancestor_iter, None)
while sibling and sibling.address.spec_path == owner.address.spec_path:
tht = await Get[TransitiveHydratedTargets](Addresses(
[sibling.address]))
if hydrated_target in tht.closure:
sibling_owners.append(sibling)
sibling = next(exported_ancestor_iter, None)
if sibling_owners:
raise AmbiguousOwnerError(
f"Exporting owners for {hydrated_target.adaptor.address.reference()} are "
f"ambiguous. Found {exported_ancestor.address.reference()} and "
f"{len(sibling_owners)} others: "
f'{", ".join(so.address.reference() for so in sibling_owners)}'
)
return ExportedTarget(HydratedTarget(owner))
raise NoOwnerError(
f"No exported target owner found for {hydrated_target.adaptor.address.reference()}"
)
def make_hydrated_target_with_origin(
*,
name: str = "target",
adaptor_type: Type[TargetAdaptor] = PythonTargetAdaptor,
include_sources: bool = True,
) -> HydratedTargetWithOrigin:
sources = Mock()
sources.snapshot = Mock()
sources.snapshot.files = ("f1", ) if include_sources else ()
ht = HydratedTarget(
adaptor_type(sources=sources, address=Address.parse(f"//:{name}")))
return HydratedTargetWithOrigin(ht,
SingleAddress(directory="", name=name))
def run_coordinator_of_tests(
self,
*,
address: Address,
origin: Optional[OriginSpec] = None,
test_target_type: bool = True,
include_sources: bool = True,
) -> AddressAndTestResult:
mocked_fileset = EagerFilesetWithSpec(
"src",
{"globs": []},
snapshot=Snapshot(
# TODO: this is not robust to set as an empty digest. Add a test util that provides
# some premade snapshots and possibly a generalized make_hydrated_target function.
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=tuple(["test.py"] if include_sources else []),
dirs=(),
),
)
adaptor_cls = PythonTestsAdaptor if test_target_type else PythonBinaryAdaptor
type_alias = "python_tests" if test_target_type else "python_binary"
adaptor = adaptor_cls(address=address,
type_alias=type_alias,
sources=mocked_fileset)
union_membership = UnionMembership(union_rules=OrderedDict(
{TestTarget: OrderedSet([PythonTestsAdaptorWithOrigin])}))
with self.captured_logging(logging.INFO):
result: AddressAndTestResult = run_rule(
coordinator_of_tests,
rule_args=[
HydratedTargetWithOrigin(
target=HydratedTarget(adaptor),
origin=(origin
or SingleAddress(directory=address.spec_path,
name=address.target_name)),
),
union_membership,
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=TestTarget,
mock=lambda _: TestResult(
status=Status.SUCCESS, stdout="foo", stderr=""),
),
],
union_membership=union_membership,
)
return result
def make_hydrated_target(
self,
*,
source_paths: List[str],
type_alias: Optional[str] = None,
) -> HydratedTarget:
adaptor = Mock()
adaptor.type_alias = type_alias
adaptor.sources = Mock()
adaptor.sources.snapshot = self.make_snapshot_of_empty_files(
source_paths)
adaptor.address = Address(spec_path=PurePath(
source_paths[0]).parent.as_posix(),
target_name="target")
return HydratedTarget(adaptor)
def test_coordinator_python_test(self):
target_adaptor = PythonTestsAdaptor(type_alias='python_tests')
with self.captured_logging(logging.INFO):
result = run_rule(
coordinator_of_tests,
HydratedTarget(Address.parse("some/target"), target_adaptor,
()),
{
(TestResult, PythonTestsAdaptor):
lambda _: TestResult(
status=Status.FAILURE, stdout='foo', stderr=''),
})
self.assertEqual(
result, TestResult(status=Status.FAILURE, stdout='foo', stderr=''))
def assert_stripped_source_file(self,
*,
original_path: str,
expected_path: str,
target_type_alias=None):
init_subsystem(SourceRootConfig)
adaptor = Mock()
adaptor.sources = Mock()
source_files = {original_path: "print('random python')"}
adaptor.sources.snapshot = self.make_snapshot(source_files)
adaptor.address = Mock()
adaptor.address.spec_path = original_path
if target_type_alias:
adaptor.type_alias = target_type_alias
target = HydratedTarget('some/target/address', adaptor, tuple())
stripped_sources = self.request_single_product(
SourceRootStrippedSources,
Params(target, SourceRootConfig.global_instance()))
self.assertEqual(stripped_sources.snapshot.files, (expected_path, ))
def run_coordinator_of_tests(
self,
*,
address: Address,
bfaddr_to_address_spec: Optional[Dict[BuildFileAddress,
AddressSpec]] = None,
test_target_type: bool = True,
include_sources: bool = True,
) -> AddressAndTestResult:
mocked_fileset = EagerFilesetWithSpec(
"src",
{"globs": []},
snapshot=Snapshot(
# TODO: this is not robust to set as an empty digest. Add a test util that provides
# some premade snapshots and possibly a generalized make_hydrated_target function.
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=tuple(["test.py"] if include_sources else []),
dirs=()))
target_adaptor = (PythonTestsAdaptor(type_alias='python_tests',
sources=mocked_fileset)
if test_target_type else PythonBinaryAdaptor(
type_alias='python_binary',
sources=mocked_fileset))
with self.captured_logging(logging.INFO):
result: AddressAndTestResult = run_rule(
coordinator_of_tests,
rule_args=[
HydratedTarget(address, target_adaptor, ()),
UnionMembership(
union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(
bfaddr_to_address_spec=bfaddr_to_address_spec or {}),
],
mock_gets=[
MockGet(
product_type=TestResult,
subject_type=PythonTestsAdaptor,
mock=lambda _: TestResult(
status=Status.SUCCESS, stdout='foo', stderr=''),
),
],
)
return result
def assert_stripped_source_file(
self,
*,
original_path: str,
expected_path: str,
target_type_alias: Optional[str] = None,
) -> None:
adaptor = Mock()
adaptor.sources = Mock()
source_files = {original_path: "print('random python')"}
adaptor.sources.snapshot = self.make_snapshot(source_files)
adaptor.address = Mock()
adaptor.address.spec_path = original_path
if target_type_alias:
adaptor.type_alias = target_type_alias
target = HydratedTarget('some/target/address', adaptor, tuple())
stripped_sources = self.request_single_product(
SourceRootStrippedSources,
Params(target, create_options_bootstrapper()))
self.assertEqual(stripped_sources.snapshot.files, (expected_path, ))
def make_hydrated_target(
*,
name: str = "target",
adaptor_type: Type[TargetAdaptor] = PythonTargetAdaptor,
include_sources: bool = True,
) -> HydratedTarget:
mocked_snapshot = Snapshot(
# TODO: this is not robust to set as an empty digest. Add a test util that provides
# some premade snapshots and possibly a generalized make_hydrated_target function.
directory_digest=EMPTY_DIRECTORY_DIGEST,
files=("formatted.txt", "fake.txt") if include_sources else (),
dirs=())
return HydratedTarget(address=f"src/{name}",
adaptor=adaptor_type(
sources=EagerFilesetWithSpec(
"src", {"globs": []},
snapshot=mocked_snapshot),
name=name,
),
dependencies=())
def test_globbed_test_target(self):
bfaddr = BuildFileAddress(None, 'tests', 'some/dir')
target_adaptor = PythonTestsAdaptor(type_alias='python_tests')
with self.captured_logging(logging.INFO):
result = run_rule(
coordinator_of_tests,
HydratedTarget(bfaddr.to_address(), target_adaptor, ()),
UnionMembership(union_rules={TestTarget: [PythonTestsAdaptor]}),
AddressProvenanceMap(bfaddr_to_spec={
bfaddr: DescendantAddresses(directory='some/dir')
}),
{
(TestResult, PythonTestsAdaptor):
lambda _: TestResult(status=Status.SUCCESS, stdout='foo', stderr=''),
})
self.assertEqual(
result,
AddressAndTestResult(bfaddr.to_address(),
TestResult(status=Status.SUCCESS, stdout='foo', stderr=''))
)
def mock_hydrated_target(self, target_address, source_files_in_dict,
dependencies):
adaptor = self.mock_target_adaptor(source_files_in_dict)
return HydratedTarget(target_address, adaptor,
tuple(d.address for d in dependencies))
async def lint(
linter: PylintLinter,
pylint: Pylint,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment,
) -> LintResult:
if pylint.options.skip:
return LintResult.noop()
adaptors_with_origins = linter.adaptors_with_origins
# Pylint needs direct dependencies in the chroot to ensure that imports are valid. However, it
# doesn't lint those direct dependencies nor does it care about transitive dependencies.
hydrated_targets = [
HydratedTarget(adaptor_with_origin.adaptor)
for adaptor_with_origin in adaptors_with_origins
]
dependencies = await MultiGet(
Get[HydratedTarget](Address, dependency)
for dependency in itertools.chain.from_iterable(
ht.adaptor.dependencies for ht in hydrated_targets))
chrooted_python_sources = await Get[ImportablePythonSources](
HydratedTargets([*hydrated_targets, *dependencies]))
# NB: Pylint output depends upon which Python interpreter version it's run with. We ensure that
# each target runs with its own interpreter constraints. See
# http://pylint.pycqa.org/en/latest/faq.html#what-versions-of-python-is-pylint-supporting.
interpreter_constraints = PexInterpreterConstraints.create_from_adaptors(
(adaptor_with_origin.adaptor
for adaptor_with_origin in adaptors_with_origins),
python_setup=python_setup,
)
requirements_pex = await Get[Pex](PexRequest(
output_filename="pylint.pex",
requirements=PexRequirements(pylint.get_requirement_specs()),
interpreter_constraints=interpreter_constraints,
entry_point=pylint.get_entry_point(),
))
config_path: Optional[str] = pylint.options.config
config_snapshot = await Get[Snapshot](PathGlobs(
globs=tuple([config_path] if config_path else []),
glob_match_error_behavior=GlobMatchErrorBehavior.error,
description_of_origin="the option `--pylint-config`",
))
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
requirements_pex.directory_digest,
config_snapshot.directory_digest,
chrooted_python_sources.snapshot.directory_digest,
)), )
specified_source_files = await Get[SourceFiles](
LegacySpecifiedSourceFilesRequest(adaptors_with_origins,
strip_source_roots=True))
address_references = ", ".join(
sorted(adaptor_with_origin.adaptor.address.reference()
for adaptor_with_origin in adaptors_with_origins))
request = requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f"./pylint.pex",
pex_args=generate_args(specified_source_files=specified_source_files,
pylint=pylint),
input_files=merged_input_files,
description=f"Run Pylint for {address_references}",
)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return LintResult.from_fallible_execute_process_result(result)
