async def find_owners(
build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
changed_request: ChangedRequest,
) -> ChangedAddresses:
direct_owners = await Get[Owners](
OwnersRequest(sources=changed_request.sources))
# If the ChangedRequest does not require dependees, then we're done.
if changed_request.include_dependees == IncludeDependeesOption.NONE:
return ChangedAddresses(direct_owners.addresses)
# Otherwise: find dependees.
all_addresses = await Get[BuildFileAddresses](AddressSpecs(
(DescendantAddresses(''), )))
all_structs = [
s.value
for s in await MultiGet(Get[HydratedStruct](Address, a.to_address())
for a in all_addresses)
]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper, all_structs)
if changed_request.include_dependees == IncludeDependeesOption.DIRECT:
return ChangedAddresses(
BuildFileAddresses(
graph.dependents_of_addresses(direct_owners.addresses)))
return ChangedAddresses(
BuildFileAddresses(
graph.transitive_dependents_of_addresses(direct_owners.addresses)))
async def find_owners(build_configuration: BuildConfiguration,
address_mapper: AddressMapper,
owners_request: OwnersRequest) -> BuildFileAddresses:
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get(HydratedTargets, Specs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target):
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
if target_sources and any_matches_filespec(sources_set,
target_sources.filespec):
return True
return False
direct_owners = tuple(
ht.adaptor.address for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(
ht.adaptor.address, sources_set) or owns_any_source(ht))
# If the OwnersRequest does not require dependees, then we're done.
if owners_request.include_dependees == 'none':
return BuildFileAddresses(direct_owners)
else:
# Otherwise: find dependees.
all_addresses = await Get(BuildFileAddresses,
Specs((DescendantAddresses(''), )))
all_hydrated_structs = await MultiGet(
Get(HydratedStruct, Address, a.to_address())
for a in all_addresses)
all_structs = [hs.value for hs in all_hydrated_structs]
bfa = build_configuration.registered_aliases()
graph = _DependentGraph.from_iterable(
target_types_from_build_file_aliases(bfa), address_mapper,
all_structs)
if owners_request.include_dependees == 'direct':
return BuildFileAddresses(
tuple(graph.dependents_of_addresses(direct_owners)))
else:
assert owners_request.include_dependees == 'transitive'
return BuildFileAddresses(
tuple(graph.transitive_dependents_of_addresses(direct_owners)))
async def get_exporting_owner(
owned_dependency: OwnedDependency) -> ExportedTarget:
"""Find the exported target that owns the given target (and therefore exports it).
The owner of T (i.e., the exported target in whose artifact T's code is published) is:
1. An exported target that depends on T (or is T itself).
2. Is T's closest filesystem ancestor among those satisfying 1.
If there are multiple such exported targets at the same degree of ancestry, the ownership
is ambiguous and an error is raised. If there is no exported target that depends on T
and is its ancestor, then there is no owner and an error is raised.
"""
hydrated_target = owned_dependency.hydrated_target
ancestor_addrs = AscendantAddresses(hydrated_target.address.spec_path)
ancestor_tgts = await Get[HydratedTargets](AddressSpecs(
(ancestor_addrs, )))
# Note that addresses sort by (spec_path, target_name), and all these targets are
# ancestors of the given target, i.e., their spec_paths are all prefixes. So sorting by
# address will effectively sort by closeness of ancestry to the given target.
exported_ancestor_tgts = sorted(
[t for t in ancestor_tgts if _is_exported(t)],
key=lambda t: t.address,
reverse=True)
exported_ancestor_iter = iter(exported_ancestor_tgts)
for exported_ancestor in exported_ancestor_iter:
tht = await Get[TransitiveHydratedTargets](BuildFileAddresses(
[exported_ancestor.address]))
if hydrated_target in tht.closure:
owner = exported_ancestor
# Find any exported siblings of owner that also depend on hydrated_target. They have the
# same spec_path as it, so they must immediately follow it in ancestor_iter.
sibling_owners = []
sibling = next(exported_ancestor_iter, None)
while sibling and sibling.address.spec_path == owner.address.spec_path:
tht = await Get[TransitiveHydratedTargets](BuildFileAddresses(
[sibling.address]))
if hydrated_target in tht.closure:
sibling_owners.append(sibling)
sibling = next(exported_ancestor_iter, None)
if sibling_owners:
raise AmbiguousOwnerError(
f'Exporting owners for {hydrated_target.address.reference()} are ambiguous. Found '
f'{exported_ancestor.address.reference()} and {len(sibling_owners)} others: '
f'{", ".join(so.address.reference() for so in sibling_owners)}'
)
return ExportedTarget(owner)
raise NoOwnerError(
f'No exported target owner found for {hydrated_target.address.reference()}'
)
async def create_python_awslambda(
lambda_tgt_adaptor: PythonAWSLambdaAdaptor, lambdex_setup: LambdexSetup,
python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> CreatedAWSLambda:
# TODO: We must enforce that everything is built for Linux, no matter the local platform.
pex_filename = f'{lambda_tgt_adaptor.address.target_name}.pex'
pex_request = CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses(
(lambda_tgt_adaptor.address, )),
entry_point=None,
output_filename=pex_filename)
pex = await Get[Pex](CreatePexFromTargetClosure, pex_request)
merged_input_files = await Get[Digest](DirectoriesToMerge(
directories=(pex.directory_digest,
lambdex_setup.requirements_pex.directory_digest)))
# NB: Lambdex modifies its input pex in-place, so the input file is also the output file.
lambdex_args = ('build', '-e', lambda_tgt_adaptor.handler, pex_filename)
process_request = lambdex_setup.requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path="./lambdex.pex",
pex_args=lambdex_args,
input_files=merged_input_files,
output_files=(pex_filename, ),
description=f'Run Lambdex for {lambda_tgt_adaptor.address.reference()}',
)
result = await Get[ExecuteProcessResult](ExecuteProcessRequest,
process_request)
return CreatedAWSLambda(digest=result.output_directory_digest,
name=pex_filename)
def addresses_from_address_families(address_mapper, address_families, spec):
"""Given a list of AddressFamilies and a Spec, return matching Addresses.
Raises a ResolveError if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
"""
if not address_families:
raise ResolveError('Path "{}" contains no BUILD files.'.format(spec.directory))
def exclude_address(address):
if address_mapper.exclude_patterns:
address_str = address.spec
return any(p.search(address_str) is not None for p in address_mapper.exclude_patterns)
return False
if type(spec) in (DescendantAddresses, SiblingAddresses, AscendantAddresses):
addresses = tuple(a
for af in address_families
for a in af.addressables.keys()
if not exclude_address(a))
elif type(spec) is SingleAddress:
# TODO Could assert len(address_families) == 1, as it should always be true in this case.
addresses = tuple(a
for af in address_families
for a in af.addressables.keys()
if a.target_name == spec.name and not exclude_address(a))
if not addresses:
if len(address_families) == 1:
_raise_did_you_mean(address_families[0], spec.name)
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
return BuildFileAddresses(addresses)
async def find_owners(owners_request: OwnersRequest) -> Owners:
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(os.path.dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = await Get[HydratedTargets](
AddressSpecs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target: HydratedTarget) -> bool:
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
return target_sources and any_matches_filespec(
paths=sources_set, spec=target_sources.filespec)
owners = BuildFileAddresses(
ht.adaptor.address for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(
ht.adaptor.address, sources_set) or owns_any_source(ht))
return Owners(owners)
async def create_python_binary(
python_binary_adaptor: PythonBinaryAdaptor) -> CreatedBinary:
#TODO(#8420) This way of calculating the entry point works but is a bit hackish.
entry_point = None
if hasattr(python_binary_adaptor, 'entry_point'):
entry_point = python_binary_adaptor.entry_point
else:
sources_snapshot = python_binary_adaptor.sources.snapshot
if len(sources_snapshot.files) == 1:
target = await Get[HydratedTarget](Address,
python_binary_adaptor.address)
output = await Get[SourceRootStrippedSources](HydratedTarget,
target)
root_filename = output.snapshot.files[0]
entry_point = PythonBinary.translate_source_path_to_py_module_specifier(
root_filename)
request = CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses(
(python_binary_adaptor.address, )),
entry_point=entry_point,
output_filename=f'{python_binary_adaptor.address.target_name}.pex')
pex = await Get[Pex](CreatePexFromTargetClosure, request)
return CreatedBinary(digest=pex.directory_digest,
binary_name=pex.output_filename)
def single_target_test(self,
result,
expected_console_output,
success=True,
debug=False):
console = MockConsole(use_colors=False)
options = MockOptions(debug=debug)
runner = InteractiveRunner(self.scheduler)
addr = self.make_build_target_address("some/target")
res = run_rule(
run_tests,
rule_args=[console, options, runner,
BuildFileAddresses([addr])],
mock_gets=[
MockGet(
product_type=AddressAndTestResult,
subject_type=Address,
mock=lambda _: AddressAndTestResult(addr, result),
),
MockGet(product_type=AddressAndDebugRequest,
subject_type=Address,
mock=lambda _: AddressAndDebugRequest(
addr,
TestDebugRequest(ipr=self.make_successful_ipr(
) if success else self.make_failure_ipr()))),
MockGet(
product_type=BuildFileAddress,
subject_type=BuildFileAddresses,
mock=lambda bfas: bfas.dependencies[0],
),
],
)
assert console.stdout.getvalue() == expected_console_output
assert (0 if success else 1) == res.exit_code
async def run_setup_pys(targets: HydratedTargets, options: SetupPyOptions, console: Console,
provenance_map: AddressProvenanceMap,
distdir: DistDir, workspace: Workspace) -> SetupPy:
"""Run setup.py commands on all exported targets addressed."""
args = tuple(options.values.args)
validate_args(args)
# Get all exported targets, ignoring any non-exported targets that happened to be
# globbed over, but erroring on any explicitly-requested non-exported targets.
exported_targets: List[ExportedTarget] = []
explicit_nonexported_targets: List[HydratedTarget] = []
for hydrated_target in targets:
if _is_exported(hydrated_target):
exported_targets.append(ExportedTarget(hydrated_target))
elif provenance_map.is_single_address(hydrated_target.address):
explicit_nonexported_targets.append(hydrated_target)
if explicit_nonexported_targets:
raise TargetNotExported(
'Cannot run setup.py on these targets, because they have no `provides=` clause: '
f'{", ".join(so.address.reference() for so in explicit_nonexported_targets)}')
if options.values.transitive:
# Expand out to all owners of the entire dep closure.
tht = await Get[TransitiveHydratedTargets](
BuildFileAddresses([et.hydrated_target.address for et in exported_targets]))
owners = await MultiGet(
Get[ExportedTarget](OwnedDependency(ht)) for ht in tht.closure if is_ownable_target(ht)
)
exported_targets = list(set(owners))
chroots = await MultiGet(Get[SetupPyChroot](SetupPyChrootRequest(target))
for target in exported_targets)
if args:
setup_py_results = await MultiGet(
Get[RunSetupPyResult](RunSetupPyRequest(exported_target, chroot, tuple(args)))
for exported_target, chroot in zip(exported_targets, chroots)
)
for exported_target, setup_py_result in zip(exported_targets, setup_py_results):
addr = exported_target.hydrated_target.address.reference()
console.print_stderr(f'Writing contents of dist dir for {addr} to {distdir.relpath}')
workspace.materialize_directory(
DirectoryToMaterialize(setup_py_result.output, path_prefix=str(distdir.relpath))
)
else:
# Just dump the chroot.
for exported_target, chroot in zip(exported_targets, chroots):
addr = exported_target.hydrated_target.address.reference()
provides = exported_target.hydrated_target.adaptor.provides
setup_py_dir = distdir.relpath / f'{provides.name}-{provides.version}'
console.print_stderr(f'Writing setup.py chroot for {addr} to {setup_py_dir}')
workspace.materialize_directory(
DirectoryToMaterialize(chroot.digest, path_prefix=str(setup_py_dir))
)
return SetupPy(0)
def addresses_from_address_families(address_families, spec):
"""Given a list of AddressFamilies and a Spec, return matching Addresses."""
if type(spec) in (DescendantAddresses, SiblingAddresses,
AscendantAddresses):
addresses = tuple(a for af in address_families
for a in af.addressables.keys())
elif type(spec) is SingleAddress:
addresses = tuple(a for af in address_families
for a in af.addressables.keys()
if a.target_name == spec.name)
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
return BuildFileAddresses(addresses)
async def get_owned_dependencies(dependency_owner: DependencyOwner) -> OwnedDependencies:
"""Find the dependencies of dependency_owner that are owned by it.
Includes dependency_owner itself.
"""
tht = await Get[TransitiveHydratedTargets](
BuildFileAddresses([dependency_owner.exported_target.hydrated_target.address]))
ownable_targets = [tgt for tgt in tht.closure
if isinstance(tgt.adaptor, (PythonTargetAdaptor, ResourcesAdaptor))]
owners = await MultiGet(Get[ExportedTarget](OwnedDependency(ht)) for ht in ownable_targets)
owned_dependencies = [tgt for owner, tgt in zip(owners, ownable_targets)
if owner == dependency_owner.exported_target]
return OwnedDependencies(OwnedDependency(t) for t in owned_dependencies)
async def get_requirements(
dep_owner: DependencyOwner) -> ExportedTargetRequirements:
tht = await Get[TransitiveHydratedTargets](BuildFileAddresses(
[dep_owner.exported_target.hydrated_target.address]))
ownable_tgts = [tgt for tgt in tht.closure if is_ownable_target(tgt)]
owners = await MultiGet(Get[ExportedTarget](OwnedDependency(ht))
for ht in ownable_tgts)
owned_by_us: Set[HydratedTarget] = set()
owned_by_others: Set[HydratedTarget] = set()
for tgt, owner in zip(ownable_tgts, owners):
(owned_by_us
if owner == dep_owner.exported_target else owned_by_others).add(tgt)
# Get all 3rdparty deps of our owned deps.
#
# Note that we need only consider requirements that are direct dependencies of our owned deps:
# If T depends on R indirectly, then it must be via some direct deps U1, U2, ... For each such U,
# if U is in the owned deps then we'll pick up R through U. And if U is not in the owned deps
# then it's owned by an exported target ET, and so R will be in the requirements for ET, and we
# will require ET.
#
# TODO: Note that this logic doesn't account for indirection via dep aggregator targets, of type
# `target`. But we don't have those in v2 (yet) anyway. Plus, as we move towards buildgen and/or
# stricter build graph hygiene, it makes sense to require that targets directly declare their
# true dependencies. Plus, in the specific realm of setup-py, since we must exclude indirect
# deps across exported target boundaries, it's not a big stretch to just insist that
# requirements must be direct deps.
direct_deps_addrs = tuple(
{dep
for ht in owned_by_us for dep in ht.dependencies})
direct_deps_tgts = await MultiGet(Get[HydratedTarget](Address, a)
for a in direct_deps_addrs)
reqs = PexRequirements.create_from_adaptors(tgt.adaptor
for tgt in direct_deps_tgts)
req_strs = list(reqs.requirements)
# Add the requirements on any exported targets on which we depend.
exported_targets_we_depend_on = await MultiGet(
Get[ExportedTarget](OwnedDependency(ht)) for ht in owned_by_others)
req_strs.extend(
sorted(et.hydrated_target.adaptor.provides.requirement
for et in set(exported_targets_we_depend_on)))
return ExportedTargetRequirements(tuple(req_strs))
def addresses_from_address_families(address_families, spec):
"""Given a list of AddressFamilies and a Spec, return matching Addresses."""
if type(spec) in (DescendantAddresses, SiblingAddresses,
AscendantAddresses):
addresses = tuple(a for af in address_families
for a in af.addressables.keys())
elif type(spec) is SingleAddress:
# TODO Could assert len(address_families) == 1, as it should always be true in this case.
addresses = tuple(a for af in address_families
for a in af.addressables.keys()
if a.target_name == spec.name)
if not addresses:
if len(address_families) == 1:
_raise_did_you_mean(address_families[0], spec.name)
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
return BuildFileAddresses(addresses)
def _inject_addresses(self, subjects):
"""Injects targets into the graph for each of the given `Address` objects, and then yields them.
TODO: See #5606 about undoing the split between `_inject_addresses` and `_inject_specs`.
"""
logger.debug('Injecting addresses to %s: %s', self, subjects)
with self._resolve_context():
addresses = tuple(subjects)
thts, = self._scheduler.product_request(TransitiveHydratedTargets,
[BuildFileAddresses(addresses)])
self._index(thts.closure)
yielded_addresses = set()
for address in subjects:
if address not in yielded_addresses:
yielded_addresses.add(address)
yield address
def _inject_addresses(self, subjects):
"""Injects targets into the graph for each of the given `Address` objects, and then yields them.
TODO: See #4533 about unifying "collection of literal Addresses" with the `Spec` types, which
would avoid the need for the independent `_inject_addresses` and `_inject_specs` codepaths.
"""
logger.debug('Injecting addresses to %s: %s', self, subjects)
with self._resolve_context():
addresses = tuple(subjects)
hydrated_targets = self._scheduler.product_request(
TransitiveHydratedTargets, [BuildFileAddresses(addresses)])
self._index(hydrated_targets)
yielded_addresses = set()
for address in subjects:
if address not in yielded_addresses:
yielded_addresses.add(address)
yield address
def addresses_from_address_families(address_mapper: AddressMapper,
specs: Specs) -> BuildFileAddresses:
"""Given an AddressMapper and list of Specs, return matching BuildFileAddresses.
:raises: :class:`ResolveError` if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
:raises: :class:`AddressLookupError` if no targets are matched for non-SingleAddress specs.
"""
# Capture a Snapshot covering all paths for these Specs, then group by directory.
snapshot = yield Get(Snapshot, PathGlobs,
_spec_to_globs(address_mapper, specs))
dirnames = {dirname(f) for f in snapshot.files}
address_families = yield [Get(AddressFamily, Dir(d)) for d in dirnames]
address_family_by_directory = {af.namespace: af for af in address_families}
matched_addresses = OrderedSet()
for spec in specs:
# NB: if a spec is provided which expands to some number of targets, but those targets match
# --exclude-target-regexp, we do NOT fail! This is why we wait to apply the tag and exclude
# patterns until we gather all the targets the spec would have matched without them.
try:
addr_families_for_spec = spec.matching_address_families(
address_family_by_directory)
except Spec.AddressFamilyResolutionError as e:
raise ResolveError(e) from e
try:
all_addr_tgt_pairs = spec.address_target_pairs_from_address_families(
addr_families_for_spec)
except Spec.AddressResolutionError as e:
raise AddressLookupError(e) from e
except SingleAddress._SingleAddressResolutionError as e:
_raise_did_you_mean(e.single_address_family, e.name, source=e)
matched_addresses.update(
addr for (addr, tgt) in all_addr_tgt_pairs
if specs.matcher.matches_target_address_pair(addr, tgt))
# NB: This may be empty, as the result of filtering by tag and exclude patterns!
yield BuildFileAddresses(tuple(matched_addresses))
async def find_build_files(addresses: Addresses) -> BuildFileAddresses:
bfas = await MultiGet(Get[BuildFileAddress](Address, address)
for address in addresses)
return BuildFileAddresses(bfas)
async def setup_pytest_for_target(
test_target: PythonTestsAdaptor,
pytest: PyTest,
test_options: TestOptions,
) -> TestTargetSetup:
# TODO: Rather than consuming the TestOptions subsystem, the TestRunner should pass on coverage
# configuration via #7490.
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
resolved_requirements_pex = await Get[Pex](
CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses((test_target.address, )),
output_filename='pytest-with-requirements.pex',
entry_point="pytest:main",
additional_requirements=pytest.get_requirement_strings(),
# NB: We set `--not-zip-safe` because Pytest plugin discovery, which uses
# `importlib_metadata` and thus `zipp`, does not play nicely when doing import magic directly
# from zip files. `zipp` has pathologically bad behavior with large zipfiles.
# TODO: this does have a performance cost as the pex must now be expanded to disk. Long term,
# it would be better to fix Zipp (whose fix would then need to be used by importlib_metadata
# and then by Pytest). See https://github.com/jaraco/zipp/pull/26.
additional_args=("--not-zip-safe", ),
include_source_files=False,
))
chrooted_sources = await Get[ChrootedPythonSources](
HydratedTargets(all_targets))
directories_to_merge = [
chrooted_sources.digest,
resolved_requirements_pex.directory_digest,
]
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get[
SourceRootStrippedSources](Address, test_target.address.to_address())
coverage_args = []
test_target_sources_file_names = source_root_stripped_test_target_sources.snapshot.files
if test_options.values.run_coverage:
coveragerc_digest = await Get[Digest](
InputFilesContent, get_coveragerc_input(DEFAULT_COVERAGE_CONFIG))
directories_to_merge.append(coveragerc_digest)
packages_to_cover = get_packages_to_cover(
test_target,
source_root_stripped_file_paths=test_target_sources_file_names,
)
coverage_args = [
'--cov-report=', # To not generate any output. https://pytest-cov.readthedocs.io/en/latest/config.html
]
for package in packages_to_cover:
coverage_args.extend(['--cov', package])
merged_input_files = await Get[Digest](
DirectoriesToMerge(directories=tuple(directories_to_merge)))
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(test_target, 'timeout', None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
return TestTargetSetup(
requirements_pex=resolved_requirements_pex,
args=(*pytest.options.args, *coverage_args,
*sorted(test_target_sources_file_names)),
input_files_digest=merged_input_files,
timeout_seconds=timeout_seconds,
)
def remove_origins(
addresses_with_origins: AddressesWithOrigins) -> BuildFileAddresses:
return BuildFileAddresses(
address_with_origin.address
for address_with_origin in addresses_with_origins)
def addresses_from_address_families(address_mapper, specs):
"""Given an AddressMapper and list of Specs, return matching BuildFileAddresses.
Raises a AddressLookupError if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
"""
# Capture a Snapshot covering all paths for these Specs, then group by directory.
snapshot = yield Get(Snapshot, PathGlobs, _spec_to_globs(address_mapper, specs))
dirnames = set(dirname(f.stat.path) for f in snapshot.files)
address_families = yield [Get(AddressFamily, Dir(d)) for d in dirnames]
# NB: `@memoized` does not work on local functions.
def by_directory():
if by_directory.cached is None:
by_directory.cached = {af.namespace: af for af in address_families}
return by_directory.cached
by_directory.cached = None
def raise_empty_address_family(spec):
raise ResolveError('Path "{}" does not contain any BUILD files.'.format(spec.directory))
def exclude_address(spec):
if specs.exclude_patterns:
return any(p.search(spec) is not None for p in specs.exclude_patterns_memo())
return False
def filter_for_tag(tag):
return lambda t: tag in map(str, t.kwargs().get("tags", []))
include_target = wrap_filters(create_filters(specs.tags if specs.tags else '', filter_for_tag))
addresses = []
included = set()
def include(address_families, predicate=None):
matched = False
for af in address_families:
for (a, t) in af.addressables.items():
if (predicate is None or predicate(a)):
if include_target(t) and (not exclude_address(a.spec)):
matched = True
if a not in included:
addresses.append(a)
included.add(a)
return matched
for spec in specs.dependencies:
if type(spec) is DescendantAddresses:
matched = include(
af
for af in address_families
if fast_relpath_optional(af.namespace, spec.directory) is not None
)
if not matched:
raise AddressLookupError(
'Spec {} does not match any targets.'.format(spec))
elif type(spec) is SiblingAddresses:
address_family = by_directory().get(spec.directory)
if not address_family:
raise_empty_address_family(spec)
include([address_family])
elif type(spec) is SingleAddress:
address_family = by_directory().get(spec.directory)
if not address_family:
raise_empty_address_family(spec)
# spec.name here is generally the root node specified on commandline. equality here implies
# a root node i.e. node specified on commandline.
if not include([address_family], predicate=lambda a: a.target_name == spec.name):
if len(addresses) == 0:
_raise_did_you_mean(address_family, spec.name)
elif type(spec) is AscendantAddresses:
include(
af
for af in address_families
if fast_relpath_optional(spec.directory, af.namespace) is not None
)
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
yield BuildFileAddresses(addresses)
def remove_provenance(
pbfas: ProvenancedBuildFileAddresses) -> BuildFileAddresses:
return BuildFileAddresses(tuple(pbfa.build_file_address for pbfa in pbfas))
def addresses_from_address_families(address_mapper, specs):
"""Given an AddressMapper and list of Specs, return matching BuildFileAddresses.
Raises a AddressLookupError if:
- there were no matching AddressFamilies, or
- the Spec matches no addresses for SingleAddresses.
"""
# Capture a Snapshot covering all paths for these Specs, then group by directory.
snapshot = yield Get(Snapshot, PathGlobs,
_spec_to_globs(address_mapper, specs))
dirnames = set(dirname(f.stat.path) for f in snapshot.files)
address_families = yield [Get(AddressFamily, Dir(d)) for d in dirnames]
# NB: `@memoized` does not work on local functions.
def by_directory():
if by_directory.cached is None:
by_directory.cached = {af.namespace: af for af in address_families}
return by_directory.cached
by_directory.cached = None
def raise_empty_address_family(spec):
raise ResolveError('Path "{}" contains no BUILD files.'.format(
spec.directory))
def exclude_address(address):
if address_mapper.exclude_patterns:
address_str = address.spec
return any(
p.search(address_str) is not None
for p in address_mapper.exclude_patterns)
return False
addresses = []
included = set()
def include(address_families, predicate=None):
matched = False
for af in address_families:
for a in af.addressables.keys():
if not exclude_address(a) and (predicate is None
or predicate(a)):
matched = True
if a not in included:
addresses.append(a)
included.add(a)
return matched
for spec in specs.dependencies:
if type(spec) is DescendantAddresses:
matched = include(af for af in address_families
if fast_relpath_optional(
af.namespace, spec.directory) is not None)
if not matched:
raise AddressLookupError(
'Spec {} does not match any targets.'.format(spec))
elif type(spec) is SiblingAddresses:
address_family = by_directory().get(spec.directory)
if not address_family:
raise_empty_address_family(spec)
include([address_family])
elif type(spec) is SingleAddress:
address_family = by_directory().get(spec.directory)
if not address_family:
raise_empty_address_family(spec)
if not include([address_family],
predicate=lambda a: a.target_name == spec.name):
_raise_did_you_mean(address_family, spec.name)
elif type(spec) is AscendantAddresses:
include(af for af in address_families if fast_relpath_optional(
spec.directory, af.namespace) is not None)
else:
raise ValueError('Unrecognized Spec type: {}'.format(spec))
yield BuildFileAddresses(addresses)
async def run_python_test(
test_target: PythonTestsAdaptor, pytest: PyTest, python_setup: PythonSetup,
subprocess_encoding_environment: SubprocessEncodingEnvironment
) -> TestResult:
"""Runs pytest for one target."""
transitive_hydrated_targets = await Get[TransitiveHydratedTargets](
BuildFileAddresses((test_target.address, )))
all_targets = transitive_hydrated_targets.closure
output_pytest_requirements_pex_filename = 'pytest-with-requirements.pex'
resolved_requirements_pex = await Get[Pex](CreatePexFromTargetClosure(
build_file_addresses=BuildFileAddresses((test_target.address, )),
output_filename=output_pytest_requirements_pex_filename,
entry_point="pytest:main",
additional_requirements=pytest.get_requirement_strings(),
include_source_files=False))
# Get the file names for the test_target, adjusted for the source root. This allows us to
# specify to Pytest which files to test and thus to avoid the test auto-discovery defined by
# https://pytest.org/en/latest/goodpractices.html#test-discovery. In addition to a performance
# optimization, this ensures that any transitive sources, such as a test project file named
# test_fail.py, do not unintentionally end up being run as tests.
source_root_stripped_test_target_sources = await Get[
SourceRootStrippedSources](Address, test_target.address.to_address())
source_root_stripped_sources = await MultiGet(
Get[SourceRootStrippedSources](HydratedTarget, hydrated_target)
for hydrated_target in all_targets)
stripped_sources_digests = tuple(
stripped_sources.snapshot.directory_digest
for stripped_sources in source_root_stripped_sources)
sources_digest = await Get[Digest](
DirectoriesToMerge(directories=stripped_sources_digests))
inits_digest = await Get[InjectedInitDigest](Digest, sources_digest)
merged_input_files = await Get[Digest](DirectoriesToMerge(directories=(
sources_digest,
inits_digest.directory_digest,
resolved_requirements_pex.directory_digest,
)), )
test_target_sources_file_names = sorted(
source_root_stripped_test_target_sources.snapshot.files)
timeout_seconds = calculate_timeout_seconds(
timeouts_enabled=pytest.options.timeouts,
target_timeout=getattr(test_target, 'timeout', None),
timeout_default=pytest.options.timeout_default,
timeout_maximum=pytest.options.timeout_maximum,
)
request = resolved_requirements_pex.create_execute_request(
python_setup=python_setup,
subprocess_encoding_environment=subprocess_encoding_environment,
pex_path=f'./{output_pytest_requirements_pex_filename}',
pex_args=(*pytest.get_args(), *test_target_sources_file_names),
input_files=merged_input_files,
description=f'Run Pytest for {test_target.address.reference()}',
timeout_seconds=timeout_seconds
if timeout_seconds is not None else 9999)
result = await Get[FallibleExecuteProcessResult](ExecuteProcessRequest,
request)
return TestResult.from_fallible_execute_process_result(result)