def fast_test(console, addresses):
test_results = yield [
Get(TestResult, Address, address.to_address()) for address in addresses
]
wrote_any_stdout = False
did_any_fail = False
for test_result in test_results:
wrote_any_stdout |= bool(test_result.stdout)
# Assume \n-terminated
console.write_stdout(test_result.stdout)
if test_result.stdout and not test_result.stdout[-1] == '\n':
console.write_stdout('\n')
if test_result.status == Status.FAILURE:
did_any_fail = True
if wrote_any_stdout:
console.write_stdout('\n')
for address, test_result in zip(addresses, test_results):
console.print_stdout('{0:80}.....{1:>10}'.format(
address.reference(), test_result.status))
if did_any_fail:
raise GracefulTerminationException("Tests failed",
exit_code=PANTS_FAILED_EXIT_CODE)
async def fmt(console: Console, targets: HydratedTargets, union_membership: UnionMembership) -> Fmt:
results = await MultiGet(
Get(FmtResult, TargetWithSources, target.adaptor)
for target in targets
# TODO: make TargetAdaptor return a 'sources' field with an empty snapshot instead of
# raising to remove the hasattr() checks here!
if union_membership.is_member(TargetWithSources, target.adaptor) and hasattr(target.adaptor, "sources")
)
for result in results:
files_content = await Get(FilesContent, Digest, result.digest)
# TODO: This is hacky and inefficient, and should be replaced by using the Workspace type
# once that is available on master.
# Blocked on: https://github.com/pantsbuild/pants/pull/8329
for file_content in files_content:
with Path(get_buildroot(), file_content.path).open('wb') as f:
f.write(file_content.content)
if result.stdout:
console.print_stdout(result.stdout)
if result.stderr:
console.print_stderr(result.stderr)
# Since we ran an ExecuteRequest, any failure would already have interrupted our flow
exit_code = 0
return Fmt(exit_code)
def wrapper(func):
if not inspect.isfunction(func):
raise ValueError(
'The @rule decorator must be applied innermost of all decorators.'
)
caller_frame = inspect.stack()[1][0]
module_ast = ast.parse(inspect.getsource(func))
def resolve_type(name):
resolved = caller_frame.f_globals.get(
name) or caller_frame.f_builtins.get(name)
if not isinstance(resolved, (TypeType, Exactly)):
# TODO(cosmicexplorer): should this say "...or Exactly instance;"?
raise ValueError(
'Expected either a `type` constructor or TypeConstraint instance; '
'got: {}'.format(name))
return resolved
gets = OrderedSet()
for node in ast.iter_child_nodes(module_ast):
if isinstance(node,
ast.FunctionDef) and node.name == func.__name__:
rule_visitor = _RuleVisitor()
rule_visitor.visit(node)
gets.update(
Get(resolve_type(p), resolve_type(s))
for p, s in rule_visitor.gets)
func._rule = TaskRule(output_type,
input_selectors,
func,
input_gets=list(gets))
return func
def hydrate_sources(sources_field):
"""Given a SourcesField, request a Snapshot for its path_globs and create an EagerFilesetWithSpec."""
snapshot = yield Get(Snapshot, PathGlobs, sources_field.path_globs)
fileset_with_spec = _eager_fileset_with_spec(
sources_field.address.spec_path, sources_field.filespecs, snapshot)
yield HydratedField(sources_field.arg, fileset_with_spec)
def download_cloc_script() -> DownloadedClocScript:
url = "https://binaries.pantsbuild.org/bin/cloc/1.80/cloc"
sha_256 = "2b23012b1c3c53bd6b9dd43cd6aa75715eed4feb2cb6db56ac3fbbd2dffeac9d"
digest = Digest(sha_256, 546279)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedClocScript(script_path=snapshot.files[0],
digest=snapshot.directory_digest)
def fast_test(console, addresses):
test_results = yield [
Get(TestResult, Address, address.to_address()) for address in addresses
]
did_any_fail = False
for address, test_result in zip(addresses, test_results):
if test_result.status == Status.FAILURE:
did_any_fail = True
if test_result.stdout:
console.write_stdout("{} stdout:\n{}\n".format(
address.reference(),
console.red(test_result.stdout) if test_result.status
== Status.FAILURE else test_result.stdout))
if test_result.stderr:
# NB: we write to stdout, rather than to stderr, to avoid potential issues interleaving the
# two streams.
console.write_stdout("{} stderr:\n{}\n".format(
address.reference(),
console.red(test_result.stderr) if test_result.status
== Status.FAILURE else test_result.stderr))
console.write_stdout("\n")
for address, test_result in zip(addresses, test_results):
console.print_stdout('{0:80}.....{1:>10}'.format(
address.reference(), test_result.status.value))
if did_any_fail:
console.print_stderr(console.red('Tests failed'))
exit_code = PANTS_FAILED_EXIT_CODE
else:
exit_code = PANTS_SUCCEEDED_EXIT_CODE
yield Test(exit_code)
def hydrate_bundles(bundles_field, glob_match_error_behavior):
"""Given a BundlesField, request Snapshots for each of its filesets and create BundleAdaptors."""
path_globs_with_match_errors = [
pg.copy(glob_match_error_behavior=glob_match_error_behavior)
for pg in bundles_field.path_globs_list
]
snapshot_list = yield [Get(Snapshot, PathGlobs, pg) for pg in path_globs_with_match_errors]
spec_path = bundles_field.address.spec_path
bundles = []
zipped = zip(bundles_field.bundles,
bundles_field.filespecs_list,
snapshot_list)
for bundle, filespecs, snapshot in zipped:
rel_spec_path = getattr(bundle, 'rel_path', spec_path)
kwargs = bundle.kwargs()
# NB: We `include_dirs=True` because bundle filesets frequently specify directories in order
# to trigger a (deprecated) default inclusion of their recursive contents. See the related
# deprecation in `pants.backend.jvm.tasks.bundle_create`.
kwargs['fileset'] = _eager_fileset_with_spec(rel_spec_path,
filespecs,
snapshot,
include_dirs=True)
bundles.append(BundleAdaptor(**kwargs))
yield HydratedField('bundles', bundles)
def create_graph_rules(address_mapper, symbol_table):
"""Creates tasks used to parse Structs from BUILD files.
:param address_mapper_key: The subject key for an AddressMapper instance.
:param symbol_table: A SymbolTable instance to provide symbols for Address lookups.
"""
symbol_table_constraint = symbol_table.constraint()
partial_hydrate_struct = functools.partial(hydrate_struct,
symbol_table_constraint)
partial_hydrate_struct.__name__ = 'hydrate_struct'
return [
# A singleton to provide the AddressMapper.
SingletonRule(AddressMapper, address_mapper),
# Support for resolving Structs from Addresses.
TaskRule(
symbol_table_constraint,
[Select(AddressMapper),
Select(UnhydratedStruct)],
partial_hydrate_struct,
input_gets=[Get(symbol_table_constraint, Address)],
),
resolve_unhydrated_struct,
# BUILD file parsing.
parse_address_family,
# Spec handling: locate directories that contain build files, and request
# AddressFamilies for each of them.
addresses_from_address_families,
# Root rules representing parameters that might be provided via root subjects.
RootRule(Address),
RootRule(BuildFileAddress),
RootRule(BuildFileAddresses),
RootRule(Specs),
]
def reify_scala_sources(sources):
"""Given a ScalaInferredDepsSources object, create ScalaSources."""
source_files_content = yield Get(FilesContent, PathGlobs, sources.path_globs)
packages = set()
import_re = re.compile(r'^import ([^;]*);?$')
for filecontent in source_files_content.dependencies:
for line in filecontent.content.splitlines():
match = import_re.search(line)
if match:
packages.add(match.group(1).rsplit('.', 1)[0])
dependency_addresses = yield [Get(Address, JVMPackageName(p)) for p in packages]
kwargs = sources._asdict()
kwargs['dependencies'] = list(set(dependency_addresses))
yield ScalaSources(**kwargs)
def all_roots(source_root_config: SourceRootConfig) -> AllSourceRoots:
source_roots = source_root_config.get_source_roots()
all_paths: Set[str] = set()
for path in source_roots.traverse():
if path.startswith("^/"):
all_paths.add(f"{path[2:]}/")
else:
all_paths.add(f"**/{path}/")
snapshot = yield Get(Snapshot, PathGlobs(include=tuple(all_paths)))
all_source_roots: Set[SourceRoot] = set()
# The globs above can match on subdirectories of the source roots.
# For instance, `src/*/` might match 'src/rust/' as well as
# 'src/rust/engine/process_execution/bazel_protos/src/gen'.
# So we use find_by_path to verify every candidate source root.
for directory in snapshot.dirs:
match: SourceRoot = source_roots.find_by_path(directory)
if match:
all_source_roots.add(match)
yield AllSourceRoots(all_source_roots)
def hydrated_targets(
build_file_addresses: BuildFileAddresses) -> HydratedTargets:
"""Requests HydratedTarget instances for BuildFileAddresses."""
targets = yield [
Get(HydratedTarget, Address, a) for a in build_file_addresses.addresses
]
yield HydratedTargets(targets)
def select_cpp_toolchain(toolchain_variant_request):
native_toolchain = toolchain_variant_request.toolchain
# TODO(#5933): make an enum exhaustiveness checking method that works with `yield Get(...)`!
use_gcc = toolchain_variant_request.variant.resolve_for_enum_variant({
'gnu':
True,
'llvm':
False,
})
if use_gcc:
toolchain_resolved = yield Get(GCCCppToolchain, NativeToolchain,
native_toolchain)
else:
toolchain_resolved = yield Get(LLVMCppToolchain, NativeToolchain,
native_toolchain)
yield toolchain_resolved.cpp_toolchain
def signature(cls):
"""Returns kwargs to construct a `TaskRule` that will construct the target Optionable.
TODO: This indirection avoids a cycle between this module and the `rules` module.
"""
partial_construct_optionable = functools.partial(
_construct_optionable, cls)
# NB: We must populate several dunder methods on the partial function because partial functions
# do not have these defined by default and the engine uses these values to visualize functions
# in error messages and the rule graph.
snake_scope = cls.options_scope.replace("-", "_")
partial_construct_optionable.__name__ = f"construct_scope_{snake_scope}"
partial_construct_optionable.__module__ = cls.__module__
_, class_definition_lineno = inspect.getsourcelines(cls)
partial_construct_optionable.__line_number__ = class_definition_lineno
return dict(
output_type=cls.optionable_cls,
input_selectors=tuple(),
func=partial_construct_optionable,
input_gets=(Get.create_statically_for_rule_graph(
ScopedOptions, Scope), ),
dependency_optionables=(cls.optionable_cls, ),
)
def wrapper(func):
if not inspect.isfunction(func):
raise ValueError('The @rule decorator must be applied innermost of all decorators.')
owning_module = sys.modules[func.__module__]
source = inspect.getsource(func)
beginning_indent = _get_starting_indent(source)
if beginning_indent:
source = "\n".join(line[beginning_indent:] for line in source.split("\n"))
module_ast = ast.parse(source)
def resolve_type(name):
resolved = getattr(owning_module, name, None) or owning_module.__builtins__.get(name, None)
if resolved is None:
raise ValueError(
f'Could not resolve type `{name}` in top level of module {owning_module.__name__}'
)
elif not isinstance(resolved, type):
raise ValueError(
f'Expected a `type` constructor for `{name}`, but got: {resolved} (type '
f'`{type(resolved).__name__}`)'
)
return resolved
gets = OrderedSet()
rule_func_node = assert_single_element(
node for node in ast.iter_child_nodes(module_ast)
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == func.__name__)
parents_table = {}
for parent in ast.walk(rule_func_node):
for child in ast.iter_child_nodes(parent):
parents_table[child] = parent
rule_visitor = _RuleVisitor()
rule_visitor.visit(rule_func_node)
gets.update(
Get.create_statically_for_rule_graph(resolve_type(p), resolve_type(s))
for p, s in rule_visitor.gets)
# Register dependencies for @goal_rule/Goal.
dependency_rules = (subsystem_rule(return_type.subsystem_cls),) if is_goal_cls else None
# Set a default name for Goal classes if one is not explicitly provided
if is_goal_cls and name is None:
effective_name = return_type.name
else:
effective_name = name
func.rule = TaskRule(
return_type,
tuple(parameter_types),
func,
input_gets=tuple(gets),
dependency_rules=dependency_rules,
cacheable=cacheable,
name=effective_name,
)
return func
def transitive_hydrated_targets(build_file_addresses):
"""Given BuildFileAddresses, kicks off recursion on expansion of TransitiveHydratedTargets.
The TransitiveHydratedTarget struct represents a structure-shared graph, which we walk
and flatten here. The engine memoizes the computation of TransitiveHydratedTarget, so
when multiple TransitiveHydratedTargets objects are being constructed for multiple
roots, their structure will be shared.
"""
transitive_hydrated_targets = yield [
Get(TransitiveHydratedTarget, Address, a)
for a in build_file_addresses.addresses
]
closure = set()
to_visit = deque(transitive_hydrated_targets)
while to_visit:
tht = to_visit.popleft()
if tht.root in closure:
continue
closure.add(tht.root)
to_visit.extend(tht.dependencies)
yield TransitiveHydratedTargets(
tuple(tht.root for tht in transitive_hydrated_targets), closure)
def generator_send(self, func, arg):
"""Given a generator, send it the given value and return a response."""
if self._do_raise_keyboardinterrupt:
raise KeyboardInterrupt("ctrl-c interrupted execution of a ffi method!")
try:
res = func.send(arg)
if Get.isinstance(res):
# Get.
return self.lib.PyGeneratorResponseGet(
res.product_type, res.subject_declared_type, res.subject,
)
elif type(res) in (tuple, list):
# GetMulti.
return self.lib.PyGeneratorResponseGetMulti(
tuple(
self.lib.PyGeneratorResponseGet(
get.product_type, get.subject_declared_type, get.subject,
)
for get in res
)
)
else:
raise ValueError(f"internal engine error: unrecognized coroutine result {res}")
except StopIteration as e:
if not e.args:
raise
# This was a `return` from a coroutine, as opposed to a `StopIteration` raised
# by calling `next()` on an empty iterator.
return self.lib.PyGeneratorResponseBreak(e.value)
async def hydrated_targets(
build_file_addresses: BuildFileAddresses) -> HydratedTargets:
"""Requests HydratedTarget instances for BuildFileAddresses."""
targets = await MultiGet(
Get(HydratedTarget, Address, a)
for a in build_file_addresses.addresses)
return HydratedTargets(targets)
def find_owners(symbol_table, address_mapper, owners_request):
sources_set = OrderedSet(owners_request.sources)
dirs_set = OrderedSet(dirname(source) for source in sources_set)
# Walk up the buildroot looking for targets that would conceivably claim changed sources.
candidate_specs = tuple(AscendantAddresses(directory=d) for d in dirs_set)
candidate_targets = yield Get(HydratedTargets, Specs(candidate_specs))
# Match the source globs against the expanded candidate targets.
def owns_any_source(legacy_target):
"""Given a `HydratedTarget` instance, check if it owns the given source file."""
target_kwargs = legacy_target.adaptor.kwargs()
# Handle `sources`-declaring targets.
# NB: Deleted files can only be matched against the 'filespec' (ie, `PathGlobs`) for a target,
# so we don't actually call `fileset.matches` here.
# TODO: This matching logic should be implemented using the rust `fs` crate for two reasons:
# 1) having two implementations isn't great
# 2) we're expanding sources via HydratedTarget, but it isn't necessary to do that to match
target_sources = target_kwargs.get('sources', None)
if target_sources and any_matches_filespec(sources_set, target_sources.filespec):
return True
return False
direct_owners = tuple(ht.adaptor.address
for ht in candidate_targets
if LegacyAddressMapper.any_is_declaring_file(ht.adaptor.address, sources_set) or
owns_any_source(ht))
# If the OwnersRequest does not require dependees, then we're done.
if owners_request.include_dependees == 'none':
yield BuildFileAddresses(direct_owners)
else:
# Otherwise: find dependees.
all_addresses = yield Get(BuildFileAddresses, Specs((DescendantAddresses(''),)))
all_structs = yield [Get(TargetAdaptorContainer, Address, a.to_address()) for a in all_addresses]
all_structs = [s.value for s in all_structs]
graph = _DependentGraph.from_iterable(target_types_from_symbol_table(symbol_table),
address_mapper,
all_structs)
if owners_request.include_dependees == 'direct':
yield BuildFileAddresses(tuple(graph.dependents_of_addresses(direct_owners)))
else:
assert owners_request.include_dependees == 'transitive'
yield BuildFileAddresses(tuple(graph.transitive_dependents_of_addresses(direct_owners)))
def wrapper(func):
if not inspect.isfunction(func):
raise ValueError('The @rule decorator must be applied innermost of all decorators.')
owning_module = sys.modules[func.__module__]
source = inspect.getsource(func)
beginning_indent = _get_starting_indent(source)
if beginning_indent:
source = "\n".join(line[beginning_indent:] for line in source.split("\n"))
module_ast = ast.parse(source)
def resolve_type(name):
resolved = getattr(owning_module, name, None) or owning_module.__builtins__.get(name, None)
if resolved is None:
raise ValueError('Could not resolve type `{}` in top level of module {}'
.format(name, owning_module.__name__))
elif not isinstance(resolved, type):
raise ValueError('Expected a `type` constructor for `{}`, but got: {} (type `{}`)'
.format(name, resolved, type(resolved).__name__))
return resolved
gets = OrderedSet()
rule_func_node = assert_single_element(
node for node in ast.iter_child_nodes(module_ast)
if isinstance(node, ast.FunctionDef) and node.name == func.__name__)
parents_table = {}
for parent in ast.walk(rule_func_node):
for child in ast.iter_child_nodes(parent):
parents_table[child] = parent
rule_visitor = _RuleVisitor(
func=func,
func_node=rule_func_node,
func_source=source,
orig_indent=beginning_indent,
parents_table=parents_table,
)
rule_visitor.visit(rule_func_node)
gets.update(
Get.create_statically_for_rule_graph(resolve_type(p), resolve_type(s))
for p, s in rule_visitor.gets)
# Register dependencies for @console_rule/Goal.
if is_goal_cls:
dependency_rules = (optionable_rule(output_type.Options),)
else:
dependency_rules = None
func.rule = TaskRule(
output_type,
tuple(input_selectors),
func,
input_gets=tuple(gets),
dependency_rules=dependency_rules,
cacheable=cacheable,
)
return func
def javac_compile_sources_execute_process_request(javac_compile_req):
sources_snapshot = yield Get(Snapshot, PathGlobs,
javac_compile_req.javac_sources.path_globs)
yield ExecuteProcessRequest.create_from_snapshot(
argv=javac_compile_req.argv_from_source_snapshot(sources_snapshot),
env=tuple(),
snapshot=sources_snapshot,
)
def create_legacy_graph_tasks(symbol_table):
"""Create tasks to recursively parse the legacy graph."""
symbol_table_constraint = symbol_table.constraint()
return [
transitive_hydrated_targets,
transitive_hydrated_target,
hydrated_targets,
TaskRule(HydratedTarget, [Select(symbol_table_constraint)],
hydrate_target,
input_gets=[
Get(HydratedField, SourcesField),
Get(HydratedField, BundlesField),
]),
hydrate_sources,
hydrate_bundles,
]
def fast_list_and_die_for_testing(console, addresses):
"""A fast and deadly variant of `./pants list`."""
options = yield Get(ScopedOptions, Scope(str('list')))
console.print_stderr('>>> Got an interesting option: {}'.format(
options.options.documented))
for address in addresses.dependencies:
console.print_stdout(address.spec)
raise GracefulTerminationException(exit_code=42)
def cat_files_process_request_input_snapshot(cat_exe_req):
cat_bin = cat_exe_req.shell_cat
cat_files_snapshot = yield Get(Snapshot, PathGlobs, cat_exe_req.path_globs)
yield ExecuteProcessRequest.create_from_snapshot(
argv=cat_bin.argv_from_snapshot(cat_files_snapshot),
env=tuple(),
snapshot=cat_files_snapshot,
)
def workspace_console_rule(console: Console, workspace: Workspace,
msg: MessageToConsoleRule) -> MockWorkspaceGoal:
digest = yield Get(Digest, InputFilesContent, msg.input_files_content)
output = workspace.materialize_directories(
(DirectoryToMaterialize(path=msg.tmp_dir, directory_digest=digest), ))
output_path = output.dependencies[0].output_paths[0]
console.print_stdout(str(Path(msg.tmp_dir, output_path)), end='')
yield MockWorkspaceGoal(exit_code=0)
def select_gcc_cpp_toolchain(platform, native_toolchain):
provided_gpp = yield Get(CppCompiler, GCC, native_toolchain._gcc)
# GCC needs an assembler, so we provide that (platform-specific) tool here.
assembler = yield Get(Assembler, NativeToolchain, native_toolchain)
if platform.normalized_os_name == 'darwin':
# GCC needs access to some headers that are only provided by the XCode toolchain
# currently (e.g. "_stdio.h"). These headers are unlikely to change across versions, so this is
# probably safe.
# TODO: we should be providing all of these (so we can eventually phase out XCodeCLITools
# entirely).
xcode_clangpp = yield Get(CppCompiler, XCodeCLITools,
native_toolchain._xcode_cli_tools)
new_include_dirs = xcode_clangpp.include_dirs + provided_gpp.include_dirs
else:
new_include_dirs = provided_gpp.include_dirs
working_cpp_compiler = CppCompiler(
path_entries=(provided_gpp.path_entries + assembler.path_entries),
exe_filename=provided_gpp.exe_filename,
library_dirs=provided_gpp.library_dirs,
include_dirs=new_include_dirs,
extra_args=([
'-x',
'c++',
'-std=c++11',
'-nostdinc++',
]))
base_linker_wrapper = yield Get(BaseLinker, NativeToolchain,
native_toolchain)
base_linker = base_linker_wrapper.linker
libc_dev = yield Get(LibcDev, NativeToolchain, native_toolchain)
working_linker = Linker(
path_entries=(working_cpp_compiler.path_entries +
base_linker.path_entries),
exe_filename=working_cpp_compiler.exe_filename,
library_dirs=(base_linker.library_dirs +
working_cpp_compiler.library_dirs),
linking_library_dirs=(base_linker.linking_library_dirs +
libc_dev.get_libc_dirs(platform)),
extra_args=base_linker.extra_args)
yield GCCCppToolchain(CppToolchain(working_cpp_compiler, working_linker))
def download_pex_bin():
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.10/pex'
digest = Digest(
'f4ac0f61f0c469537f04418e9347658340b0bce4ba61d302c5b805d194dda482',
1868106)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0],
directory_digest=snapshot.directory_digest)
def download_pex_bin() -> DownloadedPexBin:
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.11/pex'
digest = Digest(
'7a8fdfce2de22d25ba38afaa9df0282c33dd436959b3a5c3f788ded2ccc2cae9',
1867604)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0],
directory_digest=snapshot.directory_digest)
def download_pex_bin() -> DownloadedPexBin:
# TODO: Inject versions and digests here through some option, rather than hard-coding it.
url = 'https://github.com/pantsbuild/pex/releases/download/v1.6.12/pex'
digest = Digest(
'ce64cb72cd23d2123dd48126af54ccf2b718d9ecb98c2ed3045ed1802e89e7e1',
1842359)
snapshot = yield Get(Snapshot, UrlToFetch(url, digest))
yield DownloadedPexBin(executable=snapshot.files[0],
directory_digest=snapshot.directory_digest)
def create_requirements_pex(request, pex_bin, python_setup,
pex_build_environment, platform):
"""Returns a PEX with the given requirements, optional entry point, and optional
interpreter constraints."""
interpreter_search_paths = create_path_env_var(
python_setup.interpreter_search_paths)
env = {
"PATH": interpreter_search_paths,
**pex_build_environment.invocation_environment_dict
}
interpreter_constraint_args = []
for constraint in request.interpreter_constraints:
interpreter_constraint_args.extend(
["--interpreter-constraint", constraint])
# NB: we use the hardcoded and generic bin name `python`, rather than something dynamic like
# `sys.executable`, to ensure that the interpreter may be discovered both locally and in remote
# execution (so long as `env` is populated with a `PATH` env var and `python` is discoverable
# somewhere on that PATH). This is only used to run the downloaded PEX tool; it is not
# necessarily the interpreter that PEX will use to execute the generated .pex file.
# TODO(#7735): Set --python-setup-interpreter-search-paths differently for the host and target
# platforms, when we introduce platforms in https://github.com/pantsbuild/pants/issues/7735.
argv = [
"python", f"./{pex_bin.executable}", "--output-file",
request.output_filename
]
if request.entry_point is not None:
argv.extend(["--entry-point", request.entry_point])
argv.extend(interpreter_constraint_args + list(request.requirements))
# NOTE
# PEX outputs are platform dependent so in order to get a PEX that we can use locally, without cross-building
# we specify that out PEX command be run on the current local platform. When we support cross-building
# through CLI flags we can configure requests that build a PEX for out local platform that are
# able to execute on a different platform, but for now in order to guarantee correct build we need
# to restrict this command to execute on the same platform type that the output is intended for.
# The correct way to interpret the keys (execution_platform_constraint, target_platform_constraint)
# of this dictionary is "The output of this command is intended for `target_platform_constraint` iff
# it is run on `execution_platform_constraint`".
execute_process_request = MultiPlatformExecuteProcessRequest(
{
(PlatformConstraint(platform.value),
PlatformConstraint(platform.value)):
ExecuteProcessRequest(
argv=tuple(argv),
env=env,
input_files=pex_bin.directory_digest,
description=
f"Create a requirements PEX: {', '.join(request.requirements)}",
output_files=(request.output_filename, ))
})
result = yield Get(ExecuteProcessResult,
MultiPlatformExecuteProcessRequest,
execute_process_request)
yield RequirementsPex(directory_digest=result.output_directory_digest)
def get_javac_version_output(javac_version_command):
javac_version_proc_req = yield Get(ExecuteProcessRequest,
JavacVersionExecutionRequest,
javac_version_command)
javac_version_proc_result = yield Get(ExecuteProcessResult,
ExecuteProcessRequest,
javac_version_proc_req)
exit_code = javac_version_proc_result.exit_code
if exit_code != 0:
stdout = javac_version_proc_result.stdout
stderr = javac_version_proc_result.stderr
# TODO(cosmicexplorer): We should probably make this automatic for most
# process invocations (see #5719).
raise ProcessExecutionFailure(exit_code, stdout, stderr,
'obtaining javac version')
yield JavacVersionOutput(str(javac_version_proc_result.stderr))
def signature(cls):
"""Returns kwargs to construct a `TaskRule` that will construct the target Optionable.
TODO: This indirection avoids a cycle between this module and the `rules` module.
"""
snake_scope = cls.options_scope.replace('-', '_')
partial_construct_optionable = functools.partial(_construct_optionable, cls)
partial_construct_optionable.__name__ = 'construct_scope_{}'.format(snake_scope)
return dict(
output_type=cls.optionable_cls,
input_selectors=tuple(),
func=partial_construct_optionable,
input_gets=(Get.create_statically_for_rule_graph(ScopedOptions, Scope),),
dependency_optionables=(cls.optionable_cls,),
)
def test_create(self):
# Test the equivalence of the 2-arg and 3-arg versions.
self.assertEqual(Get(AClass, BClass()),
Get(AClass, BClass, BClass()))
with self.assertRaises(TypeError) as cm:
Get(AClass, BClass)
self.assertEqual("""\
The two-argument form of Get does not accept a type as its second argument.
args were: Get(({a!r}, {b!r}))
Get.create_statically_for_rule_graph() should be used to generate a Get() for
the `input_gets` field of a rule. If you are using a `yield Get(...)` in a rule
and a type was intended, use the 3-argument version:
Get({a!r}, {t!r}, {b!r})
""".format(a=AClass, t=type(BClass), b=BClass), str(cm.exception))
with self.assertRaises(ValueError) as cm:
Get(1)
self.assertEqual("Expected either two or three arguments to Get; got (1,).",
str(cm.exception))
def test_extract_constraints(self):
parsed_two_arg_call = self._get_call_node("Get(A, B(x))")
self.assertEqual(('A', 'B'),
Get.extract_constraints(parsed_two_arg_call))
with self.assertRaises(ValueError) as cm:
Get.extract_constraints(self._get_call_node("Get(1, 2)"))
self.assertEqual(str(cm.exception), """\
Two arg form of Get expected (product_type, subject_type(subject)), but got: (Num, Num)""")
parsed_three_arg_call = self._get_call_node("Get(A, B, C(x))")
self.assertEqual(('A', 'B'),
Get.extract_constraints(parsed_three_arg_call))
with self.assertRaises(ValueError) as cm:
Get.extract_constraints(self._get_call_node("Get(A, 'asdf', C(x))"))
self.assertEqual(str(cm.exception), """\
Three arg form of Get expected (product_type, subject_declared_type, subject), but got: (A, Str, Call)""")
def test_create_statically_for_rule_graph(self):
self.assertEqual(Get(AClass, BClass, None),
Get.create_statically_for_rule_graph(AClass, BClass))
def visit_Call(self, node):
if not isinstance(node.func, ast.Name) or node.func.id != Get.__name__:
return
self.gets.append(Get.extract_constraints(node))
