def test_resolve_from_pex_not_found(
pex_repository, # type: str
py36, # type: PythonInterpreter
):
# type: (...) -> None
with pytest.raises(Unsatisfiable) as exec_info:
resolve_from_pex(
pex=pex_repository,
requirements=["pex"],
interpreters=[py36],
)
assert "A distribution for pex could not be resolved in this environment." in str(
exec_info.value)
with pytest.raises(Unsatisfiable) as exec_info:
resolve_from_pex(
pex=pex_repository,
requirements=["requests==1.0.0"],
interpreters=[py36],
)
message = str(exec_info.value)
assert ("Failed to resolve requirements from PEX environment @ {}".format(
pex_repository) in message)
assert "Needed {} compatible dependencies for:".format(
py36.platform) in message
assert "1: requests==1.0.0" in message
assert "But this pex only contains:" in message
assert "requests-2.25.1-py2.py3-none-any.whl" in message
def test_resolve_from_pex_intransitive(
pex_repository, # type: str
py27, # type: PythonInterpreter
py36, # type: PythonInterpreter
foreign_platform, # type: Platform
manylinux, # type: Optional[str]
):
# type: (...) -> None
resolved_distributions = resolve_from_pex(
pex=pex_repository,
requirements=["requests"],
transitive=False,
interpreters=[py27, py36],
platforms=[foreign_platform],
manylinux=manylinux,
)
assert 3 == len(
resolved_distributions
), "Expected one resolved distribution per distribution target."
assert 1 == len(
frozenset(resolved_distribution.distribution.location
for resolved_distribution in resolved_distributions)
), ("Expected one underlying resolved universal distribution usable on Linux and macOs by "
"both Python 2.7 and Python 3.6.")
for resolved_distribution in resolved_distributions:
assert (Requirement.parse("requests==2.25.1") ==
resolved_distribution.distribution.as_requirement())
assert Requirement.parse(
"requests") == resolved_distribution.direct_requirement
def test_resolve_from_pex_constraints(
pex_repository, # type: str
py27, # type: PythonInterpreter
):
# type: (...) -> None
with pytest.raises(Unsatisfiable) as exec_info:
resolve_from_pex(
pex=pex_repository,
requirements=["requests"],
constraint_files=[create_constraints_file("urllib3==1.26.2")],
interpreters=[py27],
)
message = str(exec_info.value)
assert "The following constraints were not satisfied by " in message
assert " resolved from {}:".format(pex_repository) in message
assert "urllib3==1.26.2" in message
def test_resolve_from_pex(
pex_repository, # type: str
py27, # type: PythonInterpreter
py36, # type: PythonInterpreter
foreign_platform, # type: Platform
manylinux, # type: Optional[str]
):
# type: (...) -> None
pex_info = PexInfo.from_pex(pex_repository)
direct_requirements = pex_info.requirements
assert 1 == len(direct_requirements)
resolved_distributions = resolve_from_pex(
pex=pex_repository,
requirements=direct_requirements,
interpreters=[py27, py36],
platforms=[foreign_platform],
manylinux=manylinux,
)
distribution_locations_by_key = defaultdict(
set) # type: DefaultDict[str, Set[str]]
for resolved_distribution in resolved_distributions:
distribution_locations_by_key[
resolved_distribution.distribution.key].add(
resolved_distribution.distribution.location)
assert {
os.path.basename(location)
for locations in distribution_locations_by_key.values()
for location in locations
} == set(pex_info.distributions.keys()), (
"Expected to resolve the same full set of distributions from the pex repository as make "
"it up when using the same requirements.")
assert "requests" in distribution_locations_by_key
assert 1 == len(distribution_locations_by_key["requests"])
assert "pysocks" in distribution_locations_by_key
assert 2 == len(distribution_locations_by_key["pysocks"]), (
"PySocks has a non-platform-specific Python 2.7 distribution and a non-platform-specific "
"Python 3 distribution; so we expect to resolve two distributions - one covering "
"Python 2.7 and one covering local Python 3.6 and our cp36 foreign platform."
)
assert "cryptography" in distribution_locations_by_key
assert 3 == len(distribution_locations_by_key["cryptography"]), (
"The cryptography requirement of the security extra is platform specific; so we expect a "
"unique distribution to be resolved for each of the three distribution targets"
)
def test_resolve_from_pex_subset(
pex_repository, # type: str
foreign_platform, # type: Platform
manylinux, # type: Optional[str]
):
# type: (...) -> None
resolved_distributions = resolve_from_pex(
pex=pex_repository,
requirements=["cffi"],
platforms=[foreign_platform],
manylinux=manylinux,
)
assert {"cffi", "pycparser"} == {
resolved_distribution.distribution.key
for resolved_distribution in resolved_distributions
}
def test_resolve_from_pex_ignore_errors(
pex_repository, # type: str
py27, # type: PythonInterpreter
):
# type: (...) -> None
# See test_resolve_from_pex_constraints above for the failure this would otherwise cause.
resolved_distributions = resolve_from_pex(
pex=pex_repository,
requirements=["requests"],
constraint_files=[create_constraints_file("urllib3==1.26.2")],
interpreters=[py27],
ignore_errors=True,
)
resolved_distributions_by_key = {
resolved_distribution.distribution.key:
resolved_distribution.distribution.as_requirement()
for resolved_distribution in resolved_distributions
}
assert len(resolved_distributions_by_key
) > 1, "We should resolve at least requests and urllib3"
assert "requests" in resolved_distributions_by_key
assert Requirement.parse(
"urllib3==1.26.1") == resolved_distributions_by_key["urllib3"]
def build_pex(reqs, options, cache=None):
interpreters = None # Default to the current interpreter.
pex_python_path = options.python_path # If None, this will result in using $PATH.
# TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag.
if pex_python_path is None and (options.rc_file
or not ENV.PEX_IGNORE_RCFILES):
rc_variables = Variables(rc=options.rc_file)
pex_python_path = rc_variables.PEX_PYTHON_PATH
# NB: options.python and interpreter constraints cannot be used together.
if options.python:
with TRACER.timed("Resolving interpreters", V=2):
def to_python_interpreter(full_path_or_basename):
if os.path.isfile(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interp = PythonInterpreter.from_env(full_path_or_basename)
if interp is None:
die("Failed to find interpreter: %s" %
full_path_or_basename)
return interp
interpreters = [
to_python_interpreter(interp) for interp in options.python
]
elif options.interpreter_constraint:
with TRACER.timed("Resolving interpreters", V=2):
constraints = options.interpreter_constraint
validate_constraints(constraints)
try:
interpreters = list(
iter_compatible_interpreters(
path=pex_python_path,
interpreter_constraints=constraints))
except UnsatisfiableInterpreterConstraintsError as e:
die(
e.create_message(
"Could not find a compatible interpreter."),
CANNOT_SETUP_INTERPRETER,
)
platforms = OrderedSet(options.platforms)
interpreters = interpreters or []
if options.platforms and options.resolve_local_platforms:
with TRACER.timed(
"Searching for local interpreters matching {}".format(
", ".join(map(str, platforms)))):
candidate_interpreters = OrderedSet(
iter_compatible_interpreters(path=pex_python_path))
candidate_interpreters.add(PythonInterpreter.get())
for candidate_interpreter in candidate_interpreters:
resolved_platforms = candidate_interpreter.supported_platforms.intersection(
platforms)
if resolved_platforms:
for resolved_platform in resolved_platforms:
TRACER.log("Resolved {} for platform {}".format(
candidate_interpreter, resolved_platform))
platforms.remove(resolved_platform)
interpreters.append(candidate_interpreter)
if platforms:
TRACER.log(
"Could not resolve a local interpreter for {}, will resolve only binary distributions "
"for {}.".format(
", ".join(map(str, platforms)),
"this platform"
if len(platforms) == 1 else "these platforms",
))
interpreter = (PythonInterpreter.latest_release_of_min_compatible_version(
interpreters) if interpreters else None)
try:
with open(options.preamble_file) as preamble_fd:
preamble = preamble_fd.read()
except TypeError:
# options.preamble_file is None
preamble = None
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
preamble=preamble,
copy_mode=CopyMode.SYMLINK,
include_tools=options.include_tools or options.venv,
)
if options.resources_directory:
pex_warnings.warn(
"The `-R/--resources-directory` option is deprecated. Resources should be added via "
"`-D/--sources-directory` instead.")
for directory in OrderedSet(options.sources_directory +
options.resources_directory):
src_dir = os.path.normpath(directory)
for root, _, files in os.walk(src_dir):
for f in files:
src_file_path = os.path.join(root, f)
dst_path = os.path.relpath(src_file_path, src_dir)
pex_builder.add_source(src_file_path, dst_path)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.unzip = options.unzip
pex_info.venv = bool(options.venv)
pex_info.venv_bin_path = options.venv
pex_info.venv_copies = options.venv_copies
pex_info.pex_path = options.pex_path
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.emit_warnings = options.emit_warnings
pex_info.inherit_path = InheritPath.for_value(options.inherit_path)
pex_info.pex_root = options.runtime_pex_root
pex_info.strip_pex_env = options.strip_pex_env
if options.interpreter_constraint:
for ic in options.interpreter_constraint:
pex_builder.add_interpreter_constraint(ic)
indexes = compute_indexes(options)
for requirements_pex in options.requirements_pexes:
pex_builder.add_from_requirements_pex(requirements_pex)
with TRACER.timed(
"Resolving distributions ({})".format(reqs +
options.requirement_files)):
if options.cache_ttl:
pex_warnings.warn(
"The --cache-ttl option is deprecated and no longer has any effect."
)
if options.headers:
pex_warnings.warn(
"The --header option is deprecated and no longer has any effect."
)
network_configuration = NetworkConfiguration(
retries=options.retries,
timeout=options.timeout,
proxy=options.proxy,
cert=options.cert,
client_cert=options.client_cert,
)
try:
if options.pex_repository:
with TRACER.timed("Resolving requirements from PEX {}.".format(
options.pex_repository)):
resolveds = resolve_from_pex(
pex=options.pex_repository,
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
network_configuration=network_configuration,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
manylinux=options.manylinux,
ignore_errors=options.ignore_errors,
)
else:
with TRACER.timed("Resolving requirements."):
resolveds = resolve_multi(
requirements=reqs,
requirement_files=options.requirement_files,
constraint_files=options.constraint_files,
allow_prereleases=options.allow_prereleases,
transitive=options.transitive,
interpreters=interpreters,
platforms=list(platforms),
indexes=indexes,
find_links=options.find_links,
resolver_version=ResolverVersion.for_value(
options.resolver_version),
network_configuration=network_configuration,
cache=cache,
build=options.build,
use_wheel=options.use_wheel,
compile=options.compile,
manylinux=options.manylinux,
max_parallel_jobs=options.max_parallel_jobs,
ignore_errors=options.ignore_errors,
)
for resolved_dist in resolveds:
pex_builder.add_distribution(resolved_dist.distribution)
if resolved_dist.direct_requirement:
pex_builder.add_requirement(
resolved_dist.direct_requirement)
except Unsatisfiable as e:
die(str(e))
if options.entry_point and options.script:
die("Must specify at most one entry point or script.", INVALID_OPTIONS)
if options.entry_point:
pex_builder.set_entry_point(options.entry_point)
elif options.script:
pex_builder.set_script(options.script)
if options.python_shebang:
pex_builder.set_shebang(options.python_shebang)
return pex_builder