def check_resolve_venv(real_interpreter):
# type: (PythonInterpreter) -> None
tmpdir = safe_mkdtemp()
def create_venv(
interpreter, # type: PythonInterpreter
rel_path, # type: str
):
# type: (...) -> List[str]
venv_dir = os.path.join(tmpdir, rel_path)
interpreter.execute(["-m", "venv", venv_dir])
return glob.glob(os.path.join(venv_dir, "bin", "python*"))
assert not real_interpreter.is_venv
assert real_interpreter is real_interpreter.resolve_base_interpreter()
for index, python in enumerate(create_venv(real_interpreter,
"first-level")):
venv_interpreter = PythonInterpreter.from_binary(python)
assert venv_interpreter.is_venv
assert venv_interpreter != real_interpreter.binary
assert real_interpreter == venv_interpreter.resolve_base_interpreter()
for nested_python in create_venv(venv_interpreter,
"second-level{}".format(index)):
nested_venv_interpreter = PythonInterpreter.from_binary(
nested_python)
assert nested_venv_interpreter.is_venv
assert nested_venv_interpreter != venv_interpreter
assert nested_venv_interpreter != real_interpreter
assert real_interpreter == nested_venv_interpreter.resolve_base_interpreter(
)
def test_venv_copies(tmpdir):
# type: (Any) -> None
python36 = ensure_python_interpreter(PY36)
pex_file = os.path.join(str(tmpdir), "venv.pex")
result = run_pex_command(args=["-o", pex_file, "--include-tools"],
python=python36)
result.assert_success()
PEX_TOOLS = make_env(PEX_TOOLS=1)
venv_symlinks = os.path.join(str(tmpdir), "venv.symlinks")
subprocess.check_call(args=[python36, pex_file, "venv", venv_symlinks],
env=PEX_TOOLS)
venv_symlinks_interpreter = PythonInterpreter.from_binary(
os.path.join(venv_symlinks, "bin", "python"))
assert os.path.islink(venv_symlinks_interpreter.binary)
venv_copies = os.path.join(str(tmpdir), "venv.copies")
subprocess.check_call(
args=[python36, pex_file, "venv", "--copies", venv_copies],
env=PEX_TOOLS)
venv_copies_interpreter = PythonInterpreter.from_binary(
os.path.join(venv_copies, "bin", "python"))
assert not os.path.islink(venv_copies_interpreter.binary)
def _do_test_exact_requirements_interpreter_change(self, sdist: bool) -> None:
python36 = PythonInterpreter.from_binary(python_interpreter_path(PY_36))
python37 = PythonInterpreter.from_binary(python_interpreter_path(PY_37))
with self.plugin_resolution(
interpreter=python36, plugins=[("jake", "1.2.3"), ("jane", "3.4.5")], sdist=sdist
) as results:
working_set, chroot, repo_dir, cache_dir = results
safe_rmtree(repo_dir)
with pytest.raises(Unsatisfiable):
with self.plugin_resolution(
interpreter=python37,
chroot=chroot,
plugins=[("jake", "1.2.3"), ("jane", "3.4.5")],
):
pytest.fail(
"Plugin re-resolution is expected for an incompatible interpreter and it is "
"expected to fail since we removed the dist `repo_dir` above."
)
# But for a compatible interpreter the exact resolve results should be re-used and load
# directly from the still in-tact cache.
with self.plugin_resolution(
interpreter=python36, chroot=chroot, plugins=[("jake", "1.2.3"), ("jane", "3.4.5")]
) as results2:
working_set2, _, _, _ = results2
assert list(working_set) == list(working_set2)
def test_detect_pyvenv(tmpdir):
# type: (Any) -> None
venv = str(tmpdir)
py35 = ensure_python_interpreter(PY35)
real_interpreter = PythonInterpreter.from_binary(py35)
real_interpreter.execute(["-m", "venv", venv])
with pytest.raises(Executor.NonZeroExit):
real_interpreter.execute(["-c", "import colors"])
venv_bin_dir = os.path.join(venv, "bin")
subprocess.check_call(
[os.path.join(venv_bin_dir, "pip"), "install", "ansicolors==1.1.8"])
canonical_to_python = defaultdict(set)
for python in glob.glob(os.path.join(venv_bin_dir, "python*")):
venv_interpreter = PythonInterpreter.from_binary(python)
canonical_to_python[venv_interpreter.binary].add(python)
venv_interpreter.execute(["-c", "import colors"])
assert (len(canonical_to_python) == 1
), "Expected exactly one canonical venv python, found: {}".format(
canonical_to_python)
canonical, pythons = canonical_to_python.popitem()
real_python = os.path.realpath(py35)
assert canonical != real_python
assert os.path.dirname(canonical) == venv_bin_dir
assert os.path.realpath(canonical) == real_python
assert len(
pythons
) >= 2, "Expected at least two virtualenv python binaries, found: {}".format(
pythons)
def test_iter_interpreter_some(self, test_interpreter1, test_interpreter2):
# type: (str, str) -> None
assert [
PythonInterpreter.from_binary(test_interpreter1),
PythonInterpreter.from_binary(test_interpreter2),
] == list(
PythonInterpreter.iter_candidates(
paths=[test_interpreter1, test_interpreter2]))
def test_interpreter_caching(self, test_interpreter1, test_interpreter2):
# type: (str, str) -> None
py_interpreter1 = PythonInterpreter.from_binary(test_interpreter1)
py_interpreter2 = PythonInterpreter.from_binary(test_interpreter2)
assert py_interpreter1 is not py_interpreter2
assert py_interpreter2.identity.version == self.TEST_INTERPRETER2_VERSION_TUPLE
py_interpreter3 = PythonInterpreter.from_binary(test_interpreter1)
assert py_interpreter1 is py_interpreter3
def test_resolve_current_and_foreign_platforms(p537_resolve_cache):
# type: (str) -> None
foreign_platform = "macosx-10.13-x86_64-cp-37-m" if IS_LINUX else "manylinux1_x86_64-cp-37-m"
resolve_current_and_foreign = functools.partial(
resolve_p537_wheel_names,
cache=p537_resolve_cache,
platforms=["current", foreign_platform])
assert 2 == len(resolve_current_and_foreign())
other_python_version = PY36 if PY_VER == (3, 5) else PY35
other_python = PythonInterpreter.from_binary(
ensure_python_interpreter(other_python_version))
current_python = PythonInterpreter.get()
assert 2 == len(resolve_current_and_foreign(interpreters=[current_python]))
assert 2 == len(resolve_current_and_foreign(interpreters=[other_python]))
assert 2 == len(
resolve_current_and_foreign(
interpreters=[current_python, current_python]))
# Here we have 2 local interpreters, satisfying current, but with different platforms and thus
# different dists and then the foreign platform for 3 total dists.
assert 3 == len(
resolve_current_and_foreign(
interpreters=[current_python, other_python]))
def wheel_installer(*mixins):
bare_interpreter = PythonInterpreter.from_binary(
ensure_python_interpreter(PY36))
with make_installer(installer_impl=OrderableInstaller,
interpreter=bare_interpreter,
mixins=list(mixins)) as installer:
yield installer
def get_interpreter(python_interpreter, interpreter_cache_dir, repos,
use_wheel):
interpreter = None
if python_interpreter:
if os.path.exists(python_interpreter):
interpreter = PythonInterpreter.from_binary(python_interpreter)
else:
interpreter = PythonInterpreter.from_env(python_interpreter)
if interpreter is None:
die('Failed to find interpreter: %s' % python_interpreter)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, interpreter_cache_dir,
repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def fake_interpreter(python_tag: str, abi_tag: str, version: Tuple[int, int, int]):
interpreter_dir = safe_mkdtemp()
binary = os.path.join(interpreter_dir, "python")
values = dict(
binary=binary,
python_tag=python_tag,
abi_tag=abi_tag,
platform_tag="",
version=version,
supported_tags=[],
env_markers={},
)
id_str = json.dumps(values)
with open(binary, "w") as fp:
fp.write(
dedent(
f"""
#!{PythonInterpreter.get().binary}
from __future__ import print_function
print({id_str!r})
"""
).strip()
)
chmod_plus_x(binary)
return PythonInterpreter.from_binary(binary)
def find_compatible_interpreters(pex_python_path, compatibility_constraints):
"""Find all compatible interpreters on the system within the supplied constraints and use
PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
"""
if pex_python_path:
interpreters = []
for binary in pex_python_path.split(os.pathsep):
try:
interpreters.append(PythonInterpreter.from_binary(binary))
except Executor.ExecutionError:
print(
"Python interpreter %s in PEX_PYTHON_PATH failed to load properly."
% binary,
file=sys.stderr)
if not interpreters:
die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.'
)
else:
if not os.getenv('PATH', ''):
# no $PATH, use sys.executable
interpreters = [PythonInterpreter.get()]
else:
# get all qualifying interpreters found in $PATH
interpreters = PythonInterpreter.all()
return list(
matched_interpreters(interpreters, compatibility_constraints
) if compatibility_constraints else interpreters)
def bare_interpreter():
with temporary_dir() as interpreter_cache:
yield setup_interpreter(interpreter=PythonInterpreter.from_binary(
ensure_python_interpreter(PY36)),
interpreter_cache_dir=interpreter_cache,
repos=None,
use_wheel=True)
def test_resolve_current_platform(p537_resolve_cache):
# type: (str) -> None
resolve_current = functools.partial(resolve_p537_wheel_names,
cache=p537_resolve_cache,
platforms=["current"])
other_python_version = PY36 if PY_VER == (3, 5) else PY35
other_python = PythonInterpreter.from_binary(
ensure_python_interpreter(other_python_version))
current_python = PythonInterpreter.get()
resolved_other = resolve_current(interpreters=[other_python])
resolved_current = resolve_current()
assert 1 == len(resolved_other)
assert 1 == len(resolved_current)
assert resolved_other != resolved_current
assert resolved_current == resolve_current(interpreters=[current_python])
assert resolved_current == resolve_current(
interpreters=[current_python, current_python])
# Here we have 2 local interpreters satisfying current but with different platforms and thus
# different dists for 2 total dists.
assert 2 == len(
resolve_current(interpreters=[current_python, other_python]))
def find_compatible_interpreters(pex_python_path=None, compatibility_constraints=None):
"""Find all compatible interpreters on the system within the supplied constraints and use
PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
"""
if pex_python_path:
interpreters = []
for binary in pex_python_path.split(os.pathsep):
try:
interpreters.append(PythonInterpreter.from_binary(binary))
except Executor.ExecutionError:
print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary,
file=sys.stderr)
if not interpreters:
die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.')
else:
# We may have been invoked with a specific interpreter not on the $PATH, make sure our
# sys.executable is included as a candidate in this case.
interpreters = OrderedSet([PythonInterpreter.get()])
# Add all qualifying interpreters found in $PATH.
interpreters.update(PythonInterpreter.all())
return list(
matched_interpreters(interpreters, compatibility_constraints)
if compatibility_constraints
else interpreters
)
def test_issues_892():
python27 = ensure_python_interpreter(PY27)
program = dedent("""\
from __future__ import print_function
import os
import sys
# This puts python3.6 stdlib on PYTHONPATH.
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
from pex import resolver
from pex.interpreter import PythonInterpreter
python27 = PythonInterpreter.from_binary({python27!r})
result = resolver.resolve(requirements=['packaging==19.2'], interpreter=python27)
print('Resolved: {{}}'.format(result))
""".format(python27=python27))
python36 = ensure_python_interpreter(PY36)
cmd, process = PythonInterpreter.from_binary(python36).open_process(
args=["-c", program], stderr=subprocess.PIPE)
_, stderr = process.communicate()
assert process.returncode == 0, dedent("""
Command {cmd} failed with {returncode}.
STDERR
======
{stderr}
""".format(cmd=cmd,
returncode=process.returncode,
stderr=stderr.decode("utf8")))
def test_pex_run_conflicting_custom_setuptools_useable():
# Here we use an older setuptools to build the pex which has a newer setuptools requirement.
# These setuptools dists have different pkg_resources APIs:
# $ diff \
# <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \
# <(zipinfo -1 setuptools-40.4.3-py2.py3-none-any.whl | grep pkg_resources/ | sort)
# 2a3,4
# > pkg_resources/py31compat.py
# > pkg_resources/_vendor/appdirs.py
with temporary_dir() as resolve_cache:
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)]
interpreter = PythonInterpreter.from_binary(sys.executable,
path_extras=[dist.location for dist in dists],
include_site_extras=False)
dists = [resolved_dist.distribution
for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)]
with temporary_dir() as temp_dir:
pex = write_simple_pex(
temp_dir,
'from pkg_resources import appdirs, py31compat',
dists=dists,
interpreter=interpreter
)
rc = PEX(pex.path()).run()
assert rc == 0
def test_activate_interpreter_different_from_current():
with temporary_dir() as pex_root:
interp_version = PY36 if PY2 else PY27
custom_interpreter = setup_interpreter(
interpreter=PythonInterpreter.from_binary(ensure_python_interpreter(interp_version)),
interpreter_cache_dir=os.path.join(pex_root, 'interpreters'),
repos=None, # Default to PyPI.
use_wheel=True
)
pex_info = PexInfo.default(custom_interpreter)
pex_info.pex_root = pex_root
with temporary_dir() as pex_chroot:
pex_builder = PEXBuilder(path=pex_chroot,
interpreter=custom_interpreter,
pex_info=pex_info)
with make_bdist(installer_impl=WheelInstaller, interpreter=custom_interpreter) as bdist:
pex_builder.add_distribution(bdist)
pex_builder.set_entry_point('sys:exit')
pex_builder.freeze()
pex = PEX(pex_builder.path(), interpreter=custom_interpreter)
try:
pex._activate()
except SystemExit as e:
pytest.fail('PEX activation of %s failed with %s' % (pex, e))
def test_iter_candidates_empty_paths(self, test_interpreter1):
# type: (str) -> None
# Whereas `paths=None` should inspect $PATH, `paths=[]` means to search nothing.
with environment_as(PATH=test_interpreter1):
assert [] == list(PythonInterpreter.iter_candidates(paths=[]))
assert [PythonInterpreter.from_binary(test_interpreter1)
] == list(PythonInterpreter.iter_candidates(paths=None))
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter,
options.interpreter_cache_dir,
options.repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and options.use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def to_python_interpreter(full_path_or_basename):
if os.path.isfile(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interp = PythonInterpreter.from_env(full_path_or_basename)
if interp is None:
die("Failed to find interpreter: %s" % full_path_or_basename)
return interp
def resolve_pytest(python_version, pytest_version):
interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(python_version))
resolved_dists = resolve_multi(
interpreters=[interpreter], requirements=["pytest=={}".format(pytest_version)]
)
project_to_version = {rd.requirement.key: rd.distribution.version for rd in resolved_dists}
assert project_to_version["pytest"] == pytest_version
return project_to_version
def test_iter_interpreter_path_filter(self, test_interpreter1,
test_interpreter2):
# type: (str, str) -> None
assert [PythonInterpreter.from_binary(test_interpreter2)] == list(
PythonInterpreter.iter_candidates(
paths=[test_interpreter1, test_interpreter2],
path_filter=lambda path: path == test_interpreter2,
))
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
def to_python_interpreter(full_path_or_basename):
if os.path.exists(full_path_or_basename):
return PythonInterpreter.from_binary(full_path_or_basename)
else:
interpreter = PythonInterpreter.from_env(full_path_or_basename)
if interpreter is None:
die('Failed to find interpreter: %s' % full_path_or_basename)
return interpreter
def _validate_good_interpreter_path_file(path):
with open(path, 'r') as fp:
lines = fp.readlines()
binary = lines[0].strip()
try:
interpreter = PythonInterpreter.from_binary(binary)
return True if interpreter else False
except Executor.ExecutableNotFound:
return False
def _validate_good_interpreter_path_file(path):
with open(path, 'r') as fp:
lines = fp.readlines()
binary = lines[0].strip()
try:
interpreter = PythonInterpreter.from_binary(binary)
return True if interpreter else False
except Executor.ExecutableNotFound:
return False
def _get_interpreter(interpreter_path_file):
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary = lines[0].strip()
interpreter = PythonInterpreter.from_binary(binary, include_site_extras=False)
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split('\t')
interpreter = interpreter.with_extra(dist_name, dist_version, location)
return interpreter
def _interpreter_from_path(self, path, filters):
try:
executable = os.readlink(os.path.join(path, 'python'))
except OSError:
return None
interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False)
if self._matches(interpreter, filters):
return self._resolve(interpreter)
return None
def _get_interpreter(interpreter_path_file):
with open(interpreter_path_file, 'r') as infile:
lines = infile.readlines()
binary = lines[0].strip()
interpreter = PythonInterpreter.from_binary(
binary, include_site_extras=False)
for line in lines[1:]:
dist_name, dist_version, location = line.strip().split('\t')
interpreter = interpreter.with_extra(dist_name, dist_version,
location)
return interpreter
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
else:
interpreter = PythonInterpreter.get()
return interpreter
def _get_interpreter(self, interpreter_path_file, targets):
if os.path.exists(interpreter_path_file):
with open(interpreter_path_file, "r") as infile:
binary = infile.read().strip()
try:
return PythonInterpreter.from_binary(binary)
except PythonInterpreter.Error:
self.context.log.info(
"Stale interpreter reference detected: {}, removing reference and "
"selecting a new interpreter.".format(binary))
os.remove(interpreter_path_file)
return self._select_interpreter(interpreter_path_file, targets)
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
else:
interpreter = PythonInterpreter.get()
return interpreter
def create_bare_interpreter(binary_path): """Creates an interpreter for python binary at the given path. The interpreter is bare in that it has no extras associated with it. :returns: A bare python interpreter with no extras. :rtype: :class:`pex.interpreter.PythonInterpreter` """ # TODO(John Sirois): Replace with a more direct PythonInterpreter construction API call when # https://github.com/pantsbuild/pex/issues/510 is fixed. interpreter_with_extras = PythonInterpreter.from_binary(binary_path) return PythonInterpreter(binary_path, interpreter_with_extras.identity, extras=None)
def fake_interpreter(id_str):
interpreter_dir = safe_mkdtemp()
binary = os.path.join(interpreter_dir, 'binary')
with open(binary, 'w') as fp:
fp.write(dedent("""
#!{}
from __future__ import print_function
print({!r})
""".format(PythonInterpreter.get().binary, id_str)).strip())
chmod_plus_x(binary)
return PythonInterpreter.from_binary(binary)
def create_bare_interpreter(binary_path): """Creates an interpreter for python binary at the given path. The interpreter is bare in that it has no extras associated with it. :returns: A bare python interpreter with no extras. :rtype: :class:`pex.interpreter.PythonInterpreter` """ # TODO(John Sirois): Replace with a more direct PythonInterpreter construction API call when # https://github.com/pantsbuild/pex/issues/510 is fixed. interpreter_with_extras = PythonInterpreter.from_binary(binary_path) return PythonInterpreter(binary_path, interpreter_with_extras.identity, extras=None)
def fake_interpreter(id_str):
interpreter_dir = safe_mkdtemp()
binary = os.path.join(interpreter_dir, 'binary')
with open(binary, 'w') as fp:
fp.write(dedent("""
#!{}
from __future__ import print_function
print({!r})
""".format(PythonInterpreter.get().binary, id_str)).strip())
chmod_plus_x(binary)
return PythonInterpreter.from_binary(binary)
def _select_pex_python_interpreter(target_python, compatibility_constraints=None):
target = find_in_path(target_python)
if not target:
die('Failed to find interpreter specified by PEX_PYTHON: %s' % target)
if compatibility_constraints:
pi = PythonInterpreter.from_binary(target)
if not list(matched_interpreters([pi], compatibility_constraints)):
die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified '
'interpreter constraints: %s' % (target, str(compatibility_constraints)))
if not os.path.exists(target):
die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target)
return target
def _interpreter_from_relpath(self, path, filters=()):
path = os.path.join(self._cache_dir, path)
try:
executable = os.readlink(os.path.join(path, 'python'))
if not os.path.exists(executable):
self._purge_interpreter(path)
return None
except OSError:
return None
interpreter = PythonInterpreter.from_binary(executable)
if self._matches(interpreter, filters=filters):
return interpreter
return None
def _get_interpreter(self, interpreter_path_file, targets):
if os.path.exists(interpreter_path_file):
with open(interpreter_path_file, 'r') as infile:
binary = infile.read().strip()
try:
return PythonInterpreter.from_binary(binary)
except Executor.ExecutableNotFound:
# TODO(John Sirois): Trap a more appropriate exception once available:
# https://github.com/pantsbuild/pex/issues/672
self.context.log.info('Stale interpreter reference detected: {}, removing reference and '
'selecting a new interpreter.'.format(binary))
os.remove(interpreter_path_file)
return self._select_interpreter(interpreter_path_file, targets)
def _establish_interpreter(args):
if args.python:
if os.path.exists(args.python):
interpreter = PythonInterpreter.from_binary(args.python)
else:
interpreter = PythonInterpreter.from_env(args.python)
if interpreter is None:
die('Failed to find interpreter: %s' % args.python)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, args.interpreter_cache_dir, args.repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and args.use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die("Failed to find interpreter: %s" % options.python)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed("Setting up interpreter %s" % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and options.use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel):
interpreter = None
if python_interpreter:
if os.path.exists(python_interpreter):
interpreter = PythonInterpreter.from_binary(python_interpreter)
else:
interpreter = PythonInterpreter.from_env(python_interpreter)
if interpreter is None:
die('Failed to find interpreter: %s' % python_interpreter)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def main():
""" Main """
# Options that this wrapper will accept from the bazel rule
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-pypi', action='store_false',
dest='pypi', default=True)
parser.add_option('--not-zip-safe', action='store_false',
dest='zip_safe', default=True)
parser.add_option('--python', default="python2.7")
parser.add_option('--find-links', dest='find_links', default='')
parser.add_option('--no-use-wheel', action='store_false',
dest='use_wheel', default=True)
parser.add_option('--pex-root', dest='pex_root', default=".pex")
options, args = parser.parse_args()
# The manifest is passed via stdin or a file, as it can sometimes get too
# large to be passed as a CLA.
if len(args) == 2:
output = args[0]
manifest_text = open(args[1], 'r').read()
elif len(args) == 1:
output = args[0]
manifest_text = sys.stdin.read()
else:
parser.error("'output' positional argument is required")
return 1
if manifest_text.startswith('"') and manifest_text.endswith('"'):
manifest_text = manifest_text[1:len(manifest_text) - 1]
manifest = parse_manifest(manifest_text)
# These are the options that pex will use
pparser, resolver_options_builder = configure_clp()
poptions, preqs = pparser.parse_args(sys.argv)
poptions.entry_point = options.entry_point
poptions.find_links = options.find_links
poptions.pypi = options.pypi
poptions.python = options.python
poptions.use_wheel = options.use_wheel
poptions.zip_safe = options.zip_safe
poptions.pex_root = options.pex_root
poptions.cache_dir = options.pex_root + "/build"
poptions.interpreter_cache_dir = options.pex_root + "/interpreters"
# sys.stderr.write("pex options: %s\n" % poptions)
os.environ["PATH"] = os.getenv("PATH",
"%s:/bin:/usr/bin" % poptions.python)
if os.path.exists(options.python):
pybin = poptions.python
else:
pybin = distutils.spawn.find_executable(options.python)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in and force it into the
# process by constructing a custom PythonInterpreter instance using it.
# interpreter = PythonInterpreter.from_binary(pybin,
# [SETUPTOOLS_PATH,
# WHEEL_PATH])
interpreter = PythonInterpreter(
pybin,
PythonInterpreter.from_binary(pybin).identity,
extras={
# TODO: Fix this to resolve automatically
('setuptools', '18.0.1'): SETUPTOOLS_PATH,
# FIXME: I don't think this accomplishes anything at all.
('wheel', '0.23.0'): WHEEL_PATH,
})
# resolve setuptools
interpreter = resolve_or_die(interpreter,
SETUPTOOLS_REQUIREMENT,
poptions)
# possibly resolve wheel
if interpreter and poptions.use_wheel:
interpreter = resolve_or_die(interpreter,
WHEEL_REQUIREMENT,
poptions)
# Add prebuilt libraries listed in the manifest.
reqs = manifest.get('requirements', {}).keys()
# if len(reqs) > 0:
# sys.stderr.write("pex requirements: %s" % reqs)
pex_builder = build_pex(reqs, poptions,
resolver_options_builder,
interpreter=interpreter)
# Set whether this PEX is zip-safe, meaning everything will stay zipped
# up and we'll rely on python's zip-import mechanism to load modules
# from the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
pex_builder.add_source(
dereference_symlinks(PKG_RESOURCES_PATH),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
raise Exception("Failed to add {}: {}".format(src, err))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as err:
raise Exception("Failed to add {}: {}".format(req, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option("--entry-point", default="__main__")
parser.add_option("--directory", action="store_true", default=False)
parser.add_option(
"--no-zip-safe", action="store_false", dest="zip_safe", default=True
)
parser.add_option("--python", default="")
parser.add_option("--python-version", default="")
parser.add_option("--python-shebang", default=None)
parser.add_option("--preload", action="append", default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace(".", " ").split()
if len(python_version) == 3:
python_version.append("0")
identity = PythonIdentity.from_id_string(" ".join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None, interpreter=interpreter
)
if options.python_shebang is not None:
pex_builder.set_shebang(options.python_shebang)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest["modules"].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest["resources"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add resources listed in the manifest.
for dst, src in manifest["nativeLibraries"].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def main():
# These are the options that this class will accept from the rule
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True)
parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default="/usr/bin/python2.7")
parser.add_option('--find-links', dest='find_links', default='')
options, args = parser.parse_args()
if len(args) == 2:
output = args[0]
manifest_text = open(args[1], 'r').read()
elif len(args) == 1:
output = args[0]
manifest_text = sys.stdin.read()
else:
parser.error("'output' positional argument is required")
return 1
if manifest_text.startswith('"') and manifest_text.endswith('"'):
manifest_text = manifest_text[1:len(manifest_text) - 1]
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
x.write(manifest_text)
manifest = parse_manifest(manifest_text)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# These are the options that pex will use
pparser, resolver_options_builder = configure_clp()
# Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
# Enabling Trace logging in pex/tracer.py shows this upon failure:
# pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
# is not compatible with CPython-2.7.3 / linux-x86_64
poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
poptions.entry_point = options.entry_point
poptions.find_links = options.find_links
poptions.pypi = options.pypi
poptions.python = options.python
poptions.zip_safe = options.zip_safe
print("pex options: %s" % poptions)
os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python
# The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
# we keep a recent version in and force it into the process by constructing a custom
# PythonInterpreter instance using it.
interpreter = PythonInterpreter(
poptions.python,
PythonInterpreter.from_binary(options.python).identity,
extras={
# TODO: Fix this to resolve automatically
('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg'
})
# resolve setuptools
interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions)
# possibly resolve wheel
if interpreter and poptions.use_wheel:
interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions)
# Add prebuilt libraries listed in the manifest.
reqs = manifest.get('requirements', {}).keys()
if len(reqs) > 0:
print("pex requirements: %s" % reqs)
pex_builder = build_pex(reqs, poptions,
resolver_options_builder, interpreter=interpreter)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
pex_builder.add_source(
dereference_symlinks(pkg_resources_py),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
