def get_asdf_paths(env: Environment,
*,
asdf_local: bool = False) -> list[str]:
"""Returns a list of paths to Python interpreters managed by ASDF.
:param env: The environment to use to look up ASDF.
:param bool asdf_local: If True, only use the interpreter specified by
'.tool-versions' file under `build_root`.
"""
asdf_dir = get_asdf_data_dir(env)
if not asdf_dir:
return []
asdf_dir = Path(asdf_dir)
# Ignore ASDF if the python plugin isn't installed.
asdf_python_plugin = asdf_dir / "plugins" / "python"
if not asdf_python_plugin.exists():
return []
# Ignore ASDF if no python versions have ever been installed (the installs folder is
# missing).
asdf_installs_dir = asdf_dir / "installs" / "python"
if not asdf_installs_dir.exists():
return []
# Find all installed versions.
asdf_installed_paths: list[str] = []
for child in asdf_installs_dir.iterdir():
# Aliases, and non-cpython installs may have odd names.
# Make sure that the entry is a subdirectory of the installs directory.
if child.is_dir():
# Make sure that the subdirectory has a bin directory.
bin_dir = child / "bin"
if bin_dir.exists():
asdf_installed_paths.append(str(bin_dir))
# Ignore ASDF if there are no installed versions.
if not asdf_installed_paths:
return []
asdf_paths: list[str] = []
asdf_versions: OrderedDict[str, str] = OrderedDict()
tool_versions_file = None
# Support "shell" based ASDF configuration
ASDF_PYTHON_VERSION = env.get("ASDF_PYTHON_VERSION")
if ASDF_PYTHON_VERSION:
asdf_versions.update([
(v, "ASDF_PYTHON_VERSION")
for v in re.split(r"\s+", ASDF_PYTHON_VERSION)
])
# Target the local tool-versions file.
if asdf_local:
tool_versions_file = Path(get_buildroot(), ".tool-versions")
if not tool_versions_file.exists():
logger.warning(
"No `.tool-versions` file found in the build root, but <ASDF_LOCAL> was set in"
" `[python-bootstrap].search_paths`.")
tool_versions_file = None
# Target the home directory tool-versions file.
else:
home = env.get("HOME")
if home:
tool_versions_file = Path(home) / ".tool-versions"
if not tool_versions_file.exists():
tool_versions_file = None
if tool_versions_file:
# Parse the tool-versions file.
# A tool-versions file contains multiple lines, one or more per tool.
# Standardize that the last line for each tool wins.
#
# The definition of a tool-versions file can be found here:
# https://asdf-vm.com/#/core-configuration?id=tool-versions
tool_versions_lines = tool_versions_file.read_text().splitlines()
last_line = None
for line in tool_versions_lines:
# Find the last python line.
if line.lower().startswith("python"):
last_line = line
if last_line:
_, _, versions = last_line.partition("python")
for v in re.split(r"\s+", versions.strip()):
if ":" in v:
key, _, value = v.partition(":")
if key.lower() == "path":
asdf_paths.append(value)
elif key.lower() == "ref":
asdf_versions[value] = str(tool_versions_file)
else:
logger.warning(
f"Unknown version format `{v}` from ASDF configured by "
"`[python-bootstrap].search_path`, ignoring. This "
"version will not be considered when determining which Python "
f"interpreters to use. Please check that `{tool_versions_file}` "
"is accurate.")
elif v == "system":
logger.warning(
"System python set by ASDF configured by "
"`[python-bootstrap].search_path` is unsupported, ignoring. "
"This version will not be considered when determining which Python "
"interpreters to use. Please remove 'system' from "
f"`{tool_versions_file}` to disable this warning.")
else:
asdf_versions[v] = str(tool_versions_file)
for version, source in asdf_versions.items():
install_dir = asdf_installs_dir / version / "bin"
if install_dir.exists():
asdf_paths.append(str(install_dir))
else:
logger.warning(
f"Trying to use ASDF version `{version}` configured by "
f"`[python-bootstrap].search_path` but `{install_dir}` does not "
"exist. This version will not be considered when determining which Python "
f"interpreters to use. Please check that `{source}` is accurate."
)
# For non-local, if no paths have been defined, fallback to every version installed
if not asdf_local and len(asdf_paths) == 0:
# This could be appended to asdf_paths, but there isn't any reason to
return asdf_installed_paths
else:
return asdf_paths
def get_environment_paths(env: Environment):
"""Returns a list of paths specified by the PATH env var."""
pathstr = env.get("PATH")
if pathstr:
return pathstr.split(os.pathsep)
return []
async def package_python_dist(
field_set: PythonDistributionFieldSet,
python_setup: PythonSetup,
union_membership: UnionMembership,
) -> BuiltPackage:
transitive_targets = await Get(TransitiveTargets, TransitiveTargetsRequest([field_set.address]))
exported_target = ExportedTarget(transitive_targets.roots[0])
dist_tgt = exported_target.target
wheel = dist_tgt.get(WheelField).value
sdist = dist_tgt.get(SDistField).value
if not wheel and not sdist:
raise NoDistTypeSelected(
softwrap(
f"""
In order to package {dist_tgt.address.spec} at least one of {WheelField.alias!r} or
{SDistField.alias!r} must be `True`.
"""
)
)
wheel_config_settings = dist_tgt.get(WheelConfigSettingsField).value or FrozenDict()
sdist_config_settings = dist_tgt.get(SDistConfigSettingsField).value or FrozenDict()
backend_env_vars = dist_tgt.get(BuildBackendEnvVarsField).value
if backend_env_vars:
extra_build_time_env = await Get(Environment, EnvironmentRequest(sorted(backend_env_vars)))
else:
extra_build_time_env = Environment()
interpreter_constraints = InterpreterConstraints.create_from_targets(
transitive_targets.closure, python_setup
) or InterpreterConstraints(python_setup.interpreter_constraints)
chroot = await Get(
DistBuildChroot,
DistBuildChrootRequest(
exported_target,
interpreter_constraints=interpreter_constraints,
),
)
# Find the source roots for the build-time 1stparty deps (e.g., deps of setup.py).
source_roots_result = await Get(
SourceRootsResult,
SourceRootsRequest(
files=[], dirs={PurePath(tgt.address.spec_path) for tgt in transitive_targets.closure}
),
)
source_roots = tuple(sorted({sr.path for sr in source_roots_result.path_to_root.values()}))
# Get any extra build-time environment (e.g., native extension requirements).
build_env_requests = []
build_env_request_types = union_membership.get(DistBuildEnvironmentRequest)
for build_env_request_type in build_env_request_types:
if build_env_request_type.is_applicable(dist_tgt):
build_env_requests.append(
build_env_request_type(
tuple(tt.address for tt in transitive_targets.closure), interpreter_constraints
)
)
build_envs = await MultiGet(
[
Get(DistBuildEnvironment, DistBuildEnvironmentRequest, build_env_request)
for build_env_request in build_env_requests
]
)
extra_build_time_requirements = tuple(
itertools.chain.from_iterable(
build_env.extra_build_time_requirements for build_env in build_envs
)
)
input_digest = await Get(
Digest,
MergeDigests(
[chroot.digest, *(build_env.extra_build_time_inputs for build_env in build_envs)]
),
)
# We prefix the entire chroot, and run with this prefix as the cwd, so that we can capture
# any changes setup made within it without also capturing other artifacts of the pex
# process invocation.
chroot_prefix = "chroot"
working_directory = os.path.join(chroot_prefix, chroot.working_directory)
prefixed_input = await Get(Digest, AddPrefix(input_digest, chroot_prefix))
build_system = await Get(BuildSystem, BuildSystemRequest(prefixed_input, working_directory))
setup_py_result = await Get(
DistBuildResult,
DistBuildRequest(
build_system=build_system,
interpreter_constraints=interpreter_constraints,
build_wheel=wheel,
build_sdist=sdist,
input=prefixed_input,
working_directory=working_directory,
build_time_source_roots=source_roots,
target_address_spec=exported_target.target.address.spec,
wheel_config_settings=wheel_config_settings,
sdist_config_settings=sdist_config_settings,
extra_build_time_requirements=extra_build_time_requirements,
extra_build_time_env=extra_build_time_env,
),
)
dist_snapshot = await Get(Snapshot, Digest, setup_py_result.output)
return BuiltPackage(
setup_py_result.output,
tuple(BuiltPackageArtifact(path) for path in dist_snapshot.files),
)
def test_expand_interpreter_search_paths(rule_runner: RuleRunner) -> None:
local_pyenv_version = "3.5.5"
all_python_versions = [
"2.7.14", local_pyenv_version, "3.7.10", "3.9.4", "3.9.5"
]
asdf_home_versions = [0, 1, 2]
asdf_local_versions = [2, 1, 4]
asdf_local_versions_str = " ".join(
materialize_indices(all_python_versions, asdf_local_versions))
rule_runner.write_files({
".python-version":
f"{local_pyenv_version}\n",
".tool-versions": ("nodejs 16.0.1\n"
"java current\n"
f"python {asdf_local_versions_str}\n"
"rust 1.52.0\n"),
})
with setup_pexrc_with_pex_python_path(["/pexrc/path1:/pexrc/path2"]):
with fake_asdf_root(all_python_versions, asdf_home_versions,
asdf_local_versions) as (
home_dir,
asdf_dir,
expected_asdf_paths,
expected_asdf_home_paths,
expected_asdf_local_paths,
), fake_pyenv_root(all_python_versions,
local_pyenv_version) as (
pyenv_root,
expected_pyenv_paths,
expected_pyenv_local_paths,
):
paths = [
"/foo",
"<PATH>",
"/bar",
"<PEXRC>",
"/baz",
"<ASDF>",
"<ASDF_LOCAL>",
"<PYENV>",
"<PYENV_LOCAL>",
"/qux",
]
env = Environment({
"HOME": home_dir,
"PATH": "/env/path1:/env/path2",
"PYENV_ROOT": pyenv_root,
"ASDF_DATA_DIR": asdf_dir,
})
expanded_paths = PythonBootstrap.expand_interpreter_search_paths(
paths,
env,
)
expected = [
"/foo",
"/env/path1",
"/env/path2",
"/bar",
"/pexrc/path1",
"/pexrc/path2",
"/baz",
*expected_asdf_home_paths,
*expected_asdf_local_paths,
*expected_pyenv_paths,
*expected_pyenv_local_paths,
"/qux",
]
assert expected == expanded_paths
def test_get_environment_paths() -> None:
paths = PythonBootstrap.get_environment_paths(
Environment({"PATH": "foo/bar:baz:/qux/quux"}))
assert ["foo/bar", "baz", "/qux/quux"] == paths
def create(
cls,
env: Mapping[str, str],
) -> DockerBuildEnvironment:
return cls(Environment(env))