def __init__(self, pex, pex_info, interpreter=None, **kw):
self._internal_cache = os.path.join(pex, pex_info.internal_cache)
self._pex = pex
self._pex_info = pex_info
self._activated = False
self._working_set = None
self._interpreter = interpreter or PythonInterpreter.get()
self._inherit_path = pex_info.inherit_path
self._supported_tags = frozenset(
self._interpreter.identity.supported_tags)
self._target_interpreter_env = self._interpreter.identity.env_markers
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._interpreter.identity.python_tag.startswith(
'pp') and zipfile.is_zipfile(self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
super(PEXEnvironment, self).__init__(
search_path=[] if pex_info.inherit_path == 'false' else sys.path,
platform=self._interpreter.identity.platform_tag,
**kw)
TRACER.log('E: tags for %r x %r -> %s' %
(self.platform, self._interpreter, self._supported_tags),
V=9)
def test_conflict_via_config(self):
# Tests that targets with compatibility conflict with targets with default compatibility.
# NB: Passes empty `args` to avoid having the default CLI args override the config.
config = {
"python-setup": {
"interpreter_constraints": ["CPython<2.7"],
}
}
binary_target = f"{self.testproject}:echo_interpreter_version"
pants_run = self._build_pex(binary_target, config=config, args=[])
self.assert_failure(
pants_run, f"Unexpected successful build of {binary_target}.")
self.assertIn(
"Unable to detect a suitable interpreter for compatibilities",
pants_run.stdout_data)
self.assertIn("CPython<2.7", pants_run.stdout_data,
"Did not output requested compatibiility.")
self.assertIn(f"Conflicting targets: {binary_target}",
pants_run.stdout_data)
# NB: we expect the error message to print *all* interpreters resolved by Pants. However,
# to simplify the tests and for hermicity, here we only test that the current interpreter
# gets printed as a proxy for the overall behavior.
self.assertIn(
PythonInterpreter.get().version_string,
pants_run.stdout_data,
"Did not output interpreters discoved by Pants.",
)
def find_compatible_interpreters(pex_python_path, compatibility_constraints):
"""Find all compatible interpreters on the system within the supplied constraints and use
PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
"""
if pex_python_path:
interpreters = []
for binary in pex_python_path.split(os.pathsep):
try:
interpreters.append(PythonInterpreter.from_binary(binary))
except Executor.ExecutionError:
print(
"Python interpreter %s in PEX_PYTHON_PATH failed to load properly."
% binary,
file=sys.stderr)
if not interpreters:
die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.'
)
else:
if not os.getenv('PATH', ''):
# no $PATH, use sys.executable
interpreters = [PythonInterpreter.get()]
else:
# get all qualifying interpreters found in $PATH
interpreters = PythonInterpreter.all()
return list(
matched_interpreters(interpreters, compatibility_constraints
) if compatibility_constraints else interpreters)
def __init__(self, target, run_tracker, interpreter=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.from_cache()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath, pex_info=pexinfo, interpreter=self.interpreter)
self.chroot = PythonChroot(
targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter)
def test_find_compatible_interpreters():
py27 = ensure_python_interpreter(PY27)
py35 = ensure_python_interpreter(PY35)
py36 = ensure_python_interpreter(PY36)
path = [py27, py35, py36]
assert [py35, py36] == find_interpreters(path, '>3')
assert [py27] == find_interpreters(path, '<3')
assert [py36] == find_interpreters(path, '>{}'.format(PY35))
assert [py35] == find_interpreters(path, '>{}, <{}'.format(PY27, PY36))
assert [py36] == find_interpreters(path, '>=3.6')
assert [] == find_interpreters(path, '<2')
assert [] == find_interpreters(path, '>4')
assert [] == find_interpreters(path, '>{}, <{}'.format(PY27, PY35))
# All interpreters on PATH including whatever interpreter is currently running.
all_known_interpreters = set(PythonInterpreter.all())
all_known_interpreters.add(PythonInterpreter.get())
interpreters = set(
iter_compatible_interpreters(compatibility_constraints=['<3']))
i_rendered = '\n '.join(sorted(map(repr, interpreters)))
aki_rendered = '\n '.join(sorted(map(repr, all_known_interpreters)))
assert interpreters.issubset(all_known_interpreters), dedent("""
interpreters '<3':
{interpreters}
all known interpreters:
{all_known_interpreters}
""".format(interpreters=i_rendered, all_known_interpreters=aki_rendered))
def __init__(self,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None,
conn_timeout=None):
self._config = Config.from_cache()
self._targets = targets
self._extra_requirements = list(
extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(
tempfile.mkdtemp()),
interpreter=self._interpreter)
self._conn_timeout = conn_timeout
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
PythonSetup(self._config).scratch_dir('artifact_cache',
default_name='artifacts'),
str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator(self._egg_cache_root)
def _gather_sources(self, target_roots):
with temporary_dir() as cache_dir:
interpreter = PythonInterpreter.get()
context = self.context(
target_roots=target_roots,
for_subsystems=[PythonInterpreterCache],
options={
PythonSetup.options_scope: {
'interpreter_cache_dir':
cache_dir,
'interpreter_search_paths':
[os.path.dirname(interpreter.binary)],
}
})
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter,
lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(GatherSources.PYTHON_SOURCES)
def _iter_interpreters():
# type: () -> Iterator[InterpreterOrError]
seen = set()
normalized_paths = (OrderedSet(
os.path.realpath(p)
for p in path.split(os.pathsep)) if path else None)
# Prefer the current interpreter, if valid.
current_interpreter = PythonInterpreter.get()
if not _valid_path or _valid_path(current_interpreter.binary):
if normalized_paths:
candidate_paths = frozenset(
(current_interpreter.binary,
os.path.dirname(current_interpreter.binary)))
candidate_paths_in_path = candidate_paths.intersection(
normalized_paths)
if candidate_paths_in_path:
for p in candidate_paths_in_path:
normalized_paths.remove(p)
seen.add(current_interpreter)
yield current_interpreter
else:
seen.add(current_interpreter)
yield current_interpreter
for interp in PythonInterpreter.iter_candidates(
paths=normalized_paths, path_filter=_valid_path):
if interp not in seen:
seen.add(interp)
yield interp
def test_conflict_via_config(self):
# Tests that targets with compatibility conflict with targets with default compatibility.
# NB: Passes empty `args` to avoid having the default CLI args override the config.
config = {
'python-setup': {
'interpreter_constraints': ['CPython<2.7'],
}
}
binary_target = '{}:echo_interpreter_version'.format(self.testproject)
pants_run = self._build_pex(binary_target, config=config, args=[])
self.assert_failure(
pants_run,
'Unexpected successful build of {binary}.'.format(binary=binary_target)
)
self.assertIn(
"Unable to detect a suitable interpreter for compatibilities",
pants_run.stdout_data
)
self.assertIn(
"CPython<2.7",
pants_run.stdout_data,
"Did not output requested compatibiility."
)
self.assertIn("Conflicting targets: {}".format(binary_target), pants_run.stdout_data)
# NB: we expect the error message to print *all* interpreters resolved by Pants. However,
# to simplify the tests and for hermicity, here we only test that the current interpreter
# gets printed as a proxy for the overall behavior.
self.assertIn(
PythonInterpreter.get().version_string,
pants_run.stdout_data,
"Did not output interpreters discoved by Pants."
)
def execute(self):
tool_req_lib = self._create_requirements(self.context, self.workdir)
with self.invalidated(targets=[tool_req_lib]) as invalidation_check:
pex_name = self._tool_subsystem().options_scope
interpreter = PythonInterpreter.get()
if len(invalidation_check.all_vts) != 1:
raise TaskError(
'Expected exactly one versioned target found {}: {}'.
format(len(invalidation_check.all_vts),
invalidation_check.all_vts))
vt = invalidation_check.all_vts[0]
pex_path = os.path.join(vt.results_dir, '{}.pex'.format(pex_name))
if invalidation_check.invalid_vts:
with self.context.new_workunit(
name='create-{}-pex'.format(pex_name),
labels=[WorkUnitLabel.PREP]):
self._build_tool_pex(context=self.context,
interpreter=interpreter,
pex_path=pex_path,
requirements_lib=tool_req_lib)
tool_instance = self.tool_instance_cls(pex_path, interpreter)
self.context.products.register_data(self.tool_instance_cls,
tool_instance)
def test_resolve_current_platform(p537_resolve_cache):
# type: (str) -> None
resolve_current = functools.partial(resolve_p537_wheel_names,
cache=p537_resolve_cache,
platforms=["current"])
other_python_version = PY36 if PY_VER == (3, 5) else PY35
other_python = PythonInterpreter.from_binary(
ensure_python_interpreter(other_python_version))
current_python = PythonInterpreter.get()
resolved_other = resolve_current(interpreters=[other_python])
resolved_current = resolve_current()
assert 1 == len(resolved_other)
assert 1 == len(resolved_current)
assert resolved_other != resolved_current
assert resolved_current == resolve_current(interpreters=[current_python])
assert resolved_current == resolve_current(
interpreters=[current_python, current_python])
# Here we have 2 local interpreters satisfying current but with different platforms and thus
# different dists for 2 total dists.
assert 2 == len(
resolve_current(interpreters=[current_python, other_python]))
def create(
cls,
path, # type: str
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> Pip
"""Creates a pip tool with PEX isolation at path.
:param path: The path to assemble the pip tool at.
:param interpreter: The interpreter to run Pip with. The current interpreter by default.
:return: The path of a PEX that can be used to execute Pip in isolation.
"""
pip_interpreter = interpreter or PythonInterpreter.get()
pip_pex_path = os.path.join(path, isolated().pex_hash)
with atomic_directory(pip_pex_path, exclusive=True) as chroot:
if not chroot.is_finalized:
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=chroot.work_dir)
isolated_pip_builder.info.venv = True
for dist_location in third_party.expose(
["pip", "setuptools", "wheel"]):
isolated_pip_builder.add_dist_location(dist=dist_location)
isolated_pip_builder.set_script("pip")
isolated_pip_builder.freeze()
pex_info = PexInfo.from_pex(pip_pex_path)
pex_info.add_interpreter_constraint(
str(pip_interpreter.identity.requirement))
return cls(
ensure_venv(
PEX(pip_pex_path,
interpreter=pip_interpreter,
pex_info=pex_info)))
def test_resolve_current_and_foreign_platforms(p537_resolve_cache):
# type: (str) -> None
foreign_platform = "macosx-10.13-x86_64-cp-37-m" if IS_LINUX else "manylinux1_x86_64-cp-37-m"
resolve_current_and_foreign = functools.partial(
resolve_p537_wheel_names,
cache=p537_resolve_cache,
platforms=["current", foreign_platform])
assert 2 == len(resolve_current_and_foreign())
other_python_version = PY36 if PY_VER == (3, 5) else PY35
other_python = PythonInterpreter.from_binary(
ensure_python_interpreter(other_python_version))
current_python = PythonInterpreter.get()
assert 2 == len(resolve_current_and_foreign(interpreters=[current_python]))
assert 2 == len(resolve_current_and_foreign(interpreters=[other_python]))
assert 2 == len(
resolve_current_and_foreign(
interpreters=[current_python, current_python]))
# Here we have 2 local interpreters, satisfying current, but with different platforms and thus
# different dists and then the foreign platform for 3 total dists.
assert 3 == len(
resolve_current_and_foreign(
interpreters=[current_python, other_python]))
def __init__(self, pex, pex_info, interpreter=None, **kw):
self._internal_cache = os.path.join(pex, pex_info.internal_cache)
self._pex = pex
self._pex_info = pex_info
self._activated = False
self._working_set = None
self._interpreter = interpreter or PythonInterpreter.get()
self._inherit_path = pex_info.inherit_path
self._supported_tags = []
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._interpreter.identity.abbr_impl == 'pp' and zipfile.is_zipfile(self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
platform = Platform.current()
platform_name = platform.platform
super(PEXEnvironment, self).__init__(
search_path=[] if pex_info.inherit_path == 'false' else sys.path,
# NB: Our pkg_resources.Environment base-class wants the platform name string and not the
# pex.platform.Platform object.
platform=platform_name,
**kw
)
self._target_interpreter_env = self._interpreter.identity.pkg_resources_env(platform_name)
self._supported_tags.extend(platform.supported_tags(self._interpreter))
TRACER.log(
'E: tags for %r x %r -> %s' % (self.platform, self._interpreter, self._supported_tags),
V=9
)
def test_find_compatible_interpreters():
py27 = ensure_python_interpreter(PY27)
py35 = ensure_python_interpreter(PY35)
py36 = ensure_python_interpreter(PY36)
pex_python_path = ':'.join([py27, py35, py36])
def find_interpreters(*constraints):
return [
interp.binary for interp in find_compatible_interpreters(
path=pex_python_path, compatibility_constraints=constraints)
]
assert [py35, py36] == find_interpreters('>3')
assert [py27] == find_interpreters('<3')
assert [py36] == find_interpreters('>{}'.format(PY35))
assert [py35] == find_interpreters('>{}, <{}'.format(PY27, PY36))
assert [py36] == find_interpreters('>=3.6')
assert [] == find_interpreters('<2')
assert [] == find_interpreters('>4')
assert [] == find_interpreters('>{}, <{}'.format(PY27, PY35))
# All interpreters on PATH including whatever interpreter is currently running.
all_known_interpreters = set(PythonInterpreter.all())
all_known_interpreters.add(PythonInterpreter.get())
interpreters = find_compatible_interpreters(
compatibility_constraints=['<3'])
assert set(interpreters).issubset(all_known_interpreters)
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter,
options.interpreter_cache_dir,
options.repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and options.use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def _iter_pex_python(pex_python):
def try_create(try_path):
try:
return PythonInterpreter.from_binary(try_path)
except Executor.ExecutionError:
return None
interpreter = try_create(pex_python)
if interpreter:
# If the target interpreter specified in PEX_PYTHON is an existing absolute path - use it.
yield interpreter
else:
# Otherwise scan the PATH for matches:
try_paths = OrderedSet(
os.path.realpath(os.path.join(directory, pex_python))
for directory in os.getenv('PATH', '').split(os.pathsep))
# Prefer the current interpreter if present in the `path`.
current_interpreter = PythonInterpreter.get()
if current_interpreter.binary in try_paths:
try_paths.remove(current_interpreter.binary)
yield current_interpreter
for try_path in try_paths:
interpreter = try_create(try_path)
if interpreter:
yield interpreter
def build(self, targets, args, interpreter=None, fast_tests=False, debug=False):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
for target in targets:
assert target.is_python, "PythonBuilder can only build PythonTargets, given %s" % str(target)
# PythonBuilder supports PythonTests and PythonBinaries
for target in targets:
if isinstance(target, PythonTests):
test_targets.append(target)
elif isinstance(target, PythonBinary):
binary_targets.append(target)
rv = PythonTestBuilder(
test_targets,
args,
interpreter=interpreter,
fast=fast_tests,
debug=debug).run()
if rv != 0:
return rv
for binary_target in binary_targets:
rv = PythonBinaryBuilder(
binary_target,
self._run_tracker,
interpreter=interpreter).run()
if rv != 0:
return rv
return 0
def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None,
copy=False):
"""Initialize a pex builder.
:keyword path: The path to write the PEX as it is built. If ``None`` is specified,
a temporary directory will be created.
:keyword interpreter: The interpreter to use to build this PEX environment. If ``None``
is specified, the current interpreter is used.
:keyword chroot: If specified, preexisting :class:`Chroot` to use for building the PEX.
:keyword pex_info: A preexisting PexInfo to use to build the PEX.
:keyword preamble: If supplied, execute this code prior to bootstrapping this PEX
environment.
:type preamble: str
:keyword copy: If False, attempt to create the pex environment via hard-linking, falling
back to copying across devices. If True, always copy.
.. versionchanged:: 0.8
The temporary directory created when ``path`` is not specified is now garbage collected on
interpreter exit.
"""
self._interpreter = interpreter or PythonInterpreter.get()
self._chroot = chroot or Chroot(path or safe_mkdtemp())
self._pex_info = pex_info or PexInfo.default(self._interpreter)
self._preamble = preamble or ''
self._copy = copy
self._shebang = self._interpreter.identity.hashbang()
self._logger = logging.getLogger(__name__)
self._frozen = False
self._distributions = set()
def test_resolve_multiplatform_requirements(self):
cffi_tgt = self._fake_target('cffi', ['cffi==1.9.1'])
pex = self._resolve_requirements([cffi_tgt], {
'python-setup': {
# We have 'current' so we can import the module in order to get the path to it.
# The other platforms (one of which may happen to be the same as current) are what we
# actually test the presence of.
'platforms': ['current', 'macosx-10.10-x86_64', 'manylinux1_i686', 'win_amd64']
}
})
stdout_data, stderr_data = self._exercise_module(pex, 'cffi')
self.assertEquals('', stderr_data.strip())
path = stdout_data.strip()
wheel_dir = os.path.join(path[0:path.find('{sep}.deps{sep}'.format(sep=os.sep))], '.deps')
wheels = set(os.listdir(wheel_dir))
def name_and_platform(whl):
# The wheel filename is of the format
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
# See https://www.python.org/dev/peps/pep-0425/.
# We don't care about the python or abi versions (they depend on what we're currently
# running on), we just want to make sure we have all the platforms we expect.
parts = os.path.splitext(whl)[0].split('-')
return '{}-{}'.format(parts[0], parts[1]), parts[-1]
names_and_platforms = set(name_and_platform(w) for w in wheels)
expected_name_and_platforms = {
# Note that we don't check for 'current' because if there's no published wheel for the
# current platform we may end up with a wheel for a compatible platform (e.g., if there's no
# wheel for macosx_10_11_x86_64, 'current' will be satisfied by macosx_10_10_x86_64).
# This is technically also true for the hard-coded platforms we list below, but we chose
# those and we happen to know that cffi wheels exist for them. Whereas we have no such
# advance knowledge for the current platform, whatever that might be in the future.
('cffi-1.9.1', 'macosx_10_10_x86_64'),
('cffi-1.9.1', 'manylinux1_i686'),
('cffi-1.9.1', 'win_amd64'),
}
# pycparser is a dependency of cffi only on CPython. We might as well check for it,
# as extra verification that we correctly fetch transitive dependencies.
if PythonInterpreter.get().identity.interpreter == 'CPython':
# N.B. Since pycparser is a floating transitive dep of cffi, we do a version-agnostic
# check here to avoid master breakage as new pycparser versions are released on pypi.
self.assertTrue(
any(
(package.startswith('pycparser-') and platform == 'any')
for package, platform
in names_and_platforms
),
'could not find pycparser in transitive dependencies!'
)
self.assertTrue(expected_name_and_platforms.issubset(names_and_platforms),
'{} is not a subset of {}'.format(expected_name_and_platforms,
names_and_platforms))
# Check that the path is under the test's build root, so we know the pex was created there.
self.assertTrue(path.startswith(os.path.realpath(get_buildroot())))
def _resolve_requirements(self, target_roots, options=None):
with temporary_dir() as cache_dir:
options = options or {}
python_setup_opts = options.setdefault(PythonSetup.options_scope,
{})
python_setup_opts['interpreter_cache_dir'] = cache_dir
interpreter = PythonInterpreter.get()
python_setup_opts['interpreter_search_paths'] = [
os.path.dirname(interpreter.binary)
]
context = self.context(target_roots=target_roots,
options=options,
for_subsystems=[PythonInterpreterCache])
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter,
lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(
ResolveRequirements.REQUIREMENTS_PEX)
def build(self, targets, args, interpreter=None, conn_timeout=None, fast_tests=False):
test_targets = []
binary_targets = []
interpreter = interpreter or PythonInterpreter.get()
for target in targets:
assert target.is_python, "PythonBuilder can only build PythonTargets, given %s" % str(target)
# PythonBuilder supports PythonTests and PythonBinaries
for target in targets:
if isinstance(target, PythonTests):
test_targets.append(target)
elif isinstance(target, PythonBinary):
binary_targets.append(target)
rv = PythonTestBuilder(
test_targets,
args,
interpreter=interpreter,
conn_timeout=conn_timeout,
fast=fast_tests).run()
if rv != 0:
return rv
for binary_target in binary_targets:
rv = PythonBinaryBuilder(
binary_target,
self._run_tracker,
interpreter=interpreter,
conn_timeout=conn_timeout).run()
if rv != 0:
return rv
return 0
def test_run_pex():
# type: () -> None
def assert_run_pex(python=None, pex_args=None):
# type: (Optional[str], Optional[List[str]]) -> List[Text]
pex_args = list(pex_args) if pex_args else []
results = run_pex_command(
python=python,
args=pex_args + [
"ansicolors==1.1.8", "--", "-c",
'import colors; print(" ".join(colors.COLORS))'
],
quiet=True,
)
results.assert_success()
assert "black red green yellow blue magenta cyan white" == results.output.strip(
)
return results.error.splitlines()
incompatible_platforms_warning_msg = (
"WARNING: attempting to run PEX with incompatible platforms!")
assert incompatible_platforms_warning_msg not in assert_run_pex()
assert incompatible_platforms_warning_msg not in assert_run_pex(
pex_args=["--platform=current"])
assert incompatible_platforms_warning_msg not in assert_run_pex(
pex_args=["--platform={}".format(PythonInterpreter.get().platform)])
py27 = ensure_python_interpreter(PY27)
stderr_lines = assert_run_pex(
python=py27, pex_args=["--platform=macosx-10.13-x86_64-cp-37-m"])
assert incompatible_platforms_warning_msg in stderr_lines
def __init__(self,
target,
run_tracker,
interpreter=None,
conn_timeout=None):
self.target = target
self.interpreter = interpreter or PythonInterpreter.get()
if not isinstance(target, PythonBinary):
raise PythonBinaryBuilder.NotABinaryTargetException(
"Target %s is not a PythonBinary!" % target)
config = Config.load()
self.distdir = config.getdefault('pants_distdir')
distpath = tempfile.mktemp(dir=self.distdir, prefix=target.name)
run_info = run_tracker.run_info
build_properties = {}
build_properties.update(
run_info.add_basic_info(run_id=None, timestamp=time.time()))
build_properties.update(run_info.add_scm_info())
pexinfo = target.pexinfo.copy()
pexinfo.build_properties = build_properties
builder = PEXBuilder(distpath,
pex_info=pexinfo,
interpreter=self.interpreter)
self.chroot = PythonChroot(targets=[target],
builder=builder,
platforms=target.platforms,
interpreter=self.interpreter,
conn_timeout=conn_timeout)
def _iter_interpreters():
# type: () -> Iterator[InterpreterOrError]
seen = set()
normalized_paths = (OrderedSet(
PythonInterpreter.canonicalize_path(p)
for p in path.split(os.pathsep)) if path else None)
# Prefer the current interpreter, if valid.
current_interpreter = preferred_interpreter or PythonInterpreter.get()
if not _valid_path or _valid_path(current_interpreter.binary):
if normalized_paths:
candidate_paths = frozenset(
(current_interpreter.binary,
os.path.dirname(current_interpreter.binary)))
candidate_paths_in_path = candidate_paths.intersection(
normalized_paths)
if candidate_paths_in_path:
# In case the full path of the current interpreter binary was in the
# `normalized_paths` we're searching, remove it to prevent identifying it again
# just to then skip it as `seen`.
normalized_paths.discard(current_interpreter.binary)
seen.add(current_interpreter)
yield current_interpreter
else:
seen.add(current_interpreter)
yield current_interpreter
for interp in PythonInterpreter.iter_candidates(
paths=normalized_paths, path_filter=_valid_path):
if interp not in seen:
seen.add(interp)
yield interp
def test_find_compatible_interpreters():
py27 = ensure_python_interpreter(PY27)
py35 = ensure_python_interpreter(PY35)
py36 = ensure_python_interpreter(PY36)
pex_python_path = ':'.join([py27, py35, py36])
def find_interpreters(*constraints):
return [interp.binary for interp in
find_compatible_interpreters(pex_python_path=pex_python_path,
compatibility_constraints=constraints)]
assert [py35, py36] == find_interpreters('>3')
assert [py27] == find_interpreters('<3')
assert [py36] == find_interpreters('>{}'.format(PY35))
assert [py35] == find_interpreters('>{}, <{}'.format(PY27, PY36))
assert [py36] == find_interpreters('>=3.6')
assert [] == find_interpreters('<2')
assert [] == find_interpreters('>4')
assert [] == find_interpreters('>{}, <{}'.format(PY27, PY35))
# All interpreters on PATH including whatever interpreter is currently running.
all_known_interpreters = set(PythonInterpreter.all())
all_known_interpreters.add(PythonInterpreter.get())
interpreters = find_compatible_interpreters(compatibility_constraints=['<3'])
assert set(interpreters).issubset(all_known_interpreters)
def __init__(self, pex, pex_info, interpreter=None, **kw):
self._internal_cache = os.path.join(pex, pex_info.internal_cache)
self._pex = pex
self._pex_info = pex_info
self._activated = False
self._working_set = None
self._interpreter = interpreter or PythonInterpreter.get()
self._inherit_path = pex_info.inherit_path
self._supported_tags = []
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._interpreter.identity.abbr_impl == 'pp' and zipfile.is_zipfile(
self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
platform = Platform.current()
platform_name = platform.platform
super(PEXEnvironment, self).__init__(
search_path=[] if pex_info.inherit_path == 'false' else sys.path,
# NB: Our pkg_resources.Environment base-class wants the platform name string and not the
# pex.platform.Platform object.
platform=platform_name,
**kw)
self._target_interpreter_env = self._interpreter.identity.pkg_resources_env(
platform_name)
self._supported_tags.extend(platform.supported_tags(self._interpreter))
TRACER.log('E: tags for %r x %r -> %s' %
(self.platform, self._interpreter, self._supported_tags),
V=9)
def test_resolve_multiplatform_requirements(self):
cffi_tgt = self._fake_target('cffi', ['cffi==1.9.1'])
pex = self._resolve_requirements([cffi_tgt], {
'python-setup': {
# We have 'current' so we can import the module in order to get the path to it.
# The other platforms (one of which may happen to be the same as current) are what we
# actually test the presence of.
'platforms': ['current', 'macosx-10.10-x86_64', 'manylinux1_i686', 'win_amd64']
}
})
stdout_data, stderr_data = self._exercise_module(pex, 'cffi')
self.assertEquals('', stderr_data.strip())
path = stdout_data.strip()
wheel_dir = os.path.join(path[0:path.find('{sep}.deps{sep}'.format(sep=os.sep))], '.deps')
wheels = set(os.listdir(wheel_dir))
def name_and_platform(whl):
# The wheel filename is of the format
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
# See https://www.python.org/dev/peps/pep-0425/.
# We don't care about the python or abi versions (they depend on what we're currently
# running on), we just want to make sure we have all the platforms we expect.
parts = os.path.splitext(whl)[0].split('-')
return '{}-{}'.format(parts[0], parts[1]), parts[-1]
names_and_platforms = set(name_and_platform(w) for w in wheels)
expected_name_and_platforms = {
# Note that we don't check for 'current' because if there's no published wheel for the
# current platform we may end up with a wheel for a compatible platform (e.g., if there's no
# wheel for macosx_10_11_x86_64, 'current' will be satisfied by macosx_10_10_x86_64).
# This is technically also true for the hard-coded platforms we list below, but we chose
# those and we happen to know that cffi wheels exist for them. Whereas we have no such
# advance knowledge for the current platform, whatever that might be in the future.
('cffi-1.9.1', 'macosx_10_10_x86_64'),
('cffi-1.9.1', 'manylinux1_i686'),
('cffi-1.9.1', 'win_amd64'),
}
# pycparser is a dependency of cffi only on CPython. We might as well check for it,
# as extra verification that we correctly fetch transitive dependencies.
if PythonInterpreter.get().identity.interpreter == 'CPython':
# N.B. Since pycparser is a floating transitive dep of cffi, we do a version-agnostic
# check here to avoid master breakage as new pycparser versions are released on pypi.
self.assertTrue(
any(
(package.startswith('pycparser-') and platform == 'any')
for package, platform
in names_and_platforms
),
'could not find pycparser in transitive dependencies!'
)
self.assertTrue(expected_name_and_platforms.issubset(names_and_platforms),
'{} is not a subset of {}'.format(expected_name_and_platforms,
names_and_platforms))
# Check that the path is under the test's build root, so we know the pex was created there.
self.assertTrue(path.startswith(os.path.realpath(get_buildroot())))
def execute(self):
isort_requirement_lib = self.Isort.Factory.create_requirements(
self.context, self.workdir)
with self.invalidated(
targets=[isort_requirement_lib]) as invalidation_check:
interpreter = PythonInterpreter.get()
assert len(invalidation_check.all_vts) == 1, (
'Expected exactly one versioned target found {}: {}'.format(
len(invalidation_check.all_vts),
invalidation_check.all_vts))
vt = invalidation_check.all_vts[0]
pex_path = os.path.join(vt.results_dir, 'isort.pex')
if invalidation_check.invalid_vts:
with self.context.new_workunit(name='create-isort-pex',
labels=[WorkUnitLabel.PREP]):
self.Isort.Factory.build_isort_pex(
context=self.context,
interpreter=interpreter,
pex_path=pex_path,
requirements_lib=isort_requirement_lib)
isort = self.Isort(pex_path, interpreter=interpreter)
self.context.products.register_data(self.Isort, isort)
def get_interpreter(python_interpreter, interpreter_cache_dir, repos,
use_wheel):
interpreter = None
if python_interpreter:
if os.path.exists(python_interpreter):
interpreter = PythonInterpreter.from_binary(python_interpreter)
else:
interpreter = PythonInterpreter.from_env(python_interpreter)
if interpreter is None:
die('Failed to find interpreter: %s' % python_interpreter)
else:
interpreter = PythonInterpreter.get()
with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2):
resolve = functools.partial(resolve_interpreter, interpreter_cache_dir,
repos)
# resolve setuptools
interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT)
# possibly resolve wheel
if interpreter and use_wheel:
interpreter = resolve(interpreter, WHEEL_REQUIREMENT)
return interpreter
def __init__(
self,
pex, # type: str
pex_info=None, # type: Optional[PexInfo]
interpreter=None, # type: Optional[PythonInterpreter]
):
# type: (...) -> None
self._pex = pex
self._pex_info = pex_info or PexInfo.from_pex(pex)
self._internal_cache = os.path.join(self._pex, self._pex_info.internal_cache)
self._activated = False
self._working_set = None
self._interpreter = interpreter or PythonInterpreter.get()
self._inherit_path = self._pex_info.inherit_path
self._supported_tags = frozenset(self._interpreter.identity.supported_tags)
self._target_interpreter_env = self._interpreter.identity.env_markers
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._interpreter.identity.python_tag.startswith("pp") and zipfile.is_zipfile(self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
super(PEXEnvironment, self).__init__(
search_path=[] if self._pex_info.inherit_path == InheritPath.FALSE else sys.path,
platform=self._interpreter.identity.platform_tag,
)
TRACER.log(
"E: tags for %r x %r -> %s" % (self.platform, self._interpreter, self._supported_tags),
V=9,
)
def __init__(self, options_bootstrapper, *, interpreter=None):
self._options_bootstrapper = options_bootstrapper
self._interpreter = interpreter or PythonInterpreter.get()
bootstrap_options = self._options_bootstrapper.get_bootstrap_options().for_global_scope()
self._plugin_requirements = bootstrap_options.plugins
self._plugin_cache_dir = bootstrap_options.plugin_cache_dir
def create(
cls,
venv_dir, # type: str
interpreter=None, # type: Optional[PythonInterpreter]
force=False, # type: bool
):
# type: (...) -> Virtualenv
venv_dir = os.path.abspath(venv_dir)
safe_mkdir(venv_dir, clean=force)
interpreter = interpreter or PythonInterpreter.get()
if interpreter.is_venv:
base_interpreter = interpreter.resolve_base_interpreter()
TRACER.log(
"Ignoring enclosing venv {} and using its base interpreter {} to create venv at {}"
" instead.".format(interpreter.prefix, base_interpreter.binary, venv_dir),
V=3,
)
interpreter = base_interpreter
if interpreter.version[0] >= 3 and not interpreter.identity.interpreter == "PyPy":
# N.B.: PyPy3 comes equipped with a venv module but it does not seem to work.
interpreter.execute(args=["-m", "venv", "--without-pip", venv_dir])
else:
virtualenv_py = resource_string(__name__, "virtualenv_16.7.10_py")
with named_temporary_file(mode="wb") as fp:
fp.write(virtualenv_py)
fp.close()
interpreter.execute(
args=[fp.name, "--no-pip", "--no-setuptools", "--no-wheel", venv_dir],
)
return cls(venv_dir)
def find_compatible_interpreters(pex_python_path=None, compatibility_constraints=None):
"""Find all compatible interpreters on the system within the supplied constraints and use
PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH.
"""
if pex_python_path:
interpreters = []
for binary in pex_python_path.split(os.pathsep):
try:
interpreters.append(PythonInterpreter.from_binary(binary))
except Executor.ExecutionError:
print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary,
file=sys.stderr)
if not interpreters:
die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.')
else:
# We may have been invoked with a specific interpreter not on the $PATH, make sure our
# sys.executable is included as a candidate in this case.
interpreters = OrderedSet([PythonInterpreter.get()])
# Add all qualifying interpreters found in $PATH.
interpreters.update(PythonInterpreter.all())
return list(
matched_interpreters(interpreters, compatibility_constraints)
if compatibility_constraints
else interpreters
)
def __init__(self,
context,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None):
self.context = context
# TODO: These should come from the caller, and we should not know about config.
self._python_setup = PythonSetup(self.context.config)
self._python_repos = PythonRepos(self.context.config)
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._python_setup.scratch_dir, 'artifacts', str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def _select_path_interpreter(
path=None, # type: Optional[str]
valid_basenames=None, # type: Optional[Tuple[str, ...]]
interpreter_constraints=None, # type: Optional[Iterable[str]]
):
# type: (...) -> Optional[PythonInterpreter]
candidate_interpreters_iter = iter_compatible_interpreters(
path=path,
valid_basenames=valid_basenames,
interpreter_constraints=interpreter_constraints,
)
current_interpreter = PythonInterpreter.get() # type: PythonInterpreter
candidate_interpreters = []
for interpreter in candidate_interpreters_iter:
if current_interpreter == interpreter:
# Always prefer continuing with the current interpreter when possible to avoid re-exec
# overhead.
return current_interpreter
else:
candidate_interpreters.append(interpreter)
if not candidate_interpreters:
return None
# TODO: Allow the selection strategy to be parameterized:
# https://github.com/pantsbuild/pex/issues/430
return PythonInterpreter.latest_release_of_min_compatible_version(
candidate_interpreters)
def __init__(self, path=None, interpreter=None, chroot=None, pex_info=None, preamble=None,
copy=False):
"""Initialize a pex builder.
:keyword path: The path to write the PEX as it is built. If ``None`` is specified,
a temporary directory will be created.
:keyword interpreter: The interpreter to use to build this PEX environment. If ``None``
is specified, the current interpreter is used.
:keyword chroot: If specified, preexisting :class:`Chroot` to use for building the PEX.
:keyword pex_info: A preexisting PexInfo to use to build the PEX.
:keyword preamble: If supplied, execute this code prior to bootstrapping this PEX
environment.
:type preamble: str
:keyword copy: If False, attempt to create the pex environment via hard-linking, falling
back to copying across devices. If True, always copy.
.. versionchanged:: 0.8
The temporary directory created when ``path`` is not specified is now garbage collected on
interpreter exit.
"""
self._interpreter = interpreter or PythonInterpreter.get()
self._chroot = chroot or Chroot(path or safe_mkdtemp())
self._pex_info = pex_info or PexInfo.default(self._interpreter)
self._preamble = preamble or ''
self._copy = copy
self._shebang = self._interpreter.identity.hashbang()
self._logger = logging.getLogger(__name__)
self._frozen = False
self._distributions = set()
def _iter_interpreters():
seen = set()
paths = None
current_interpreter = PythonInterpreter.get()
if path:
paths = OrderedSet(
os.path.realpath(p) for p in path.split(os.pathsep))
# Prefer the current interpreter if present on the `path`.
candidate_paths = frozenset(
(current_interpreter.binary,
os.path.dirname(current_interpreter.binary)))
if candidate_paths.intersection(paths):
for p in candidate_paths:
paths.remove(p)
seen.add(current_interpreter)
yield current_interpreter
else:
# We may have been invoked with a specific interpreter, make sure our sys.executable is
# included as a candidate in this case.
seen.add(current_interpreter)
yield current_interpreter
for interp in PythonInterpreter.iter(paths=paths):
if interp not in seen:
seen.add(interp)
yield interp
def __init__(self,
context,
python_setup,
python_repos,
targets,
extra_requirements=None,
builder=None,
platforms=None,
interpreter=None):
self.context = context
self._python_setup = python_setup
self._python_repos = python_repos
self._targets = targets
self._extra_requirements = list(extra_requirements) if extra_requirements else []
self._platforms = platforms
self._interpreter = interpreter or PythonInterpreter.get()
self._builder = builder or PEXBuilder(os.path.realpath(tempfile.mkdtemp()),
interpreter=self._interpreter)
# Note: unrelated to the general pants artifact cache.
self._egg_cache_root = os.path.join(
self._python_setup.scratch_dir, 'artifacts', str(self._interpreter.identity))
self._key_generator = CacheKeyGenerator()
self._build_invalidator = BuildInvalidator( self._egg_cache_root)
def compilation(valid_paths=None, invalid_paths=None, compile_paths=None):
with temporary_dir() as root:
for path in valid_paths:
write_source(os.path.join(root, path))
for path in invalid_paths:
write_source(os.path.join(root, path), valid=False)
compiler = Compiler(PythonInterpreter.get())
yield root, compiler.compile(root, compile_paths)
def __init__(self, targets, args, interpreter=None, conn_timeout=None, fast=False):
self.targets = targets
self.args = args
self.interpreter = interpreter or PythonInterpreter.get()
self._conn_timeout = conn_timeout
# If fast is true, we run all the tests in a single chroot. This is MUCH faster than
# creating a chroot for each test target. However running each test separately is more
# correct, as the isolation verifies that its dependencies are correctly declared.
self._fast = fast
def __init__(self, pex=sys.argv[0], interpreter=None, env=ENV, verify_entry_point=False):
self._pex = pex
self._interpreter = interpreter or PythonInterpreter.get()
self._pex_info = PexInfo.from_pex(self._pex)
self._pex_info_overrides = PexInfo.from_env(env=env)
self._vars = env
self._envs = []
self._working_set = None
if verify_entry_point:
self._do_entry_point_verification()
def test_resolve_multiplatform_requirements(self):
cffi_tgt = self._fake_target("cffi", ["cffi==1.9.1"])
pex = self._resolve_requirements(
[cffi_tgt],
{
"python-setup": {
# We have 'current' so we can import the module in order to get the path to it.
# The other platforms (one of which may happen to be the same as current) are what we
# actually test the presence of.
"platforms": ["current", "macosx-10.10-x86_64", "manylinux1_i686", "win_amd64"]
}
},
)
stdout_data, stderr_data = self._exercise_module(pex, "cffi")
self.assertEquals("", stderr_data.strip())
path = stdout_data.strip()
wheel_dir = os.path.join(path[0 : path.find("{sep}.deps{sep}".format(sep=os.sep))], ".deps")
wheels = set(os.listdir(wheel_dir))
def name_and_platform(whl):
# The wheel filename is of the format
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
# See https://www.python.org/dev/peps/pep-0425/.
# We don't care about the python or abi versions (they depend on what we're currently
# running on), we just want to make sure we have all the platforms we expect.
parts = os.path.splitext(whl)[0].split("-")
return "{}-{}".format(parts[0], parts[1]), parts[-1]
names_and_platforms = set(name_and_platform(w) for w in wheels)
expected_name_and_platforms = {
# Note that we don't check for 'current' because if there's no published wheel for the
# current platform we may end up with a wheel for a compatible platform (e.g., if there's no
# wheel for macosx_10_11_x86_64, 'current' will be satisfied by macosx_10_10_x86_64).
# This is technically also true for the hard-coded platforms we list below, but we chose
# those and we happen to know that cffi wheels exist for them. Whereas we have no such
# advance knowledge for the current platform, whatever that might be in the future.
("cffi-1.9.1", "macosx_10_10_x86_64"),
("cffi-1.9.1", "manylinux1_i686"),
("cffi-1.9.1", "win_amd64"),
}
# pycparser is a dependency of cffi only on CPython. We might as well check for it,
# as extra verification that we correctly fetch transitive dependencies.
if PythonInterpreter.get().identity.interpreter == "CPython":
expected_name_and_platforms.add(("pycparser-2.17", "any"))
self.assertTrue(
expected_name_and_platforms.issubset(names_and_platforms),
"{} is not a subset of {}".format(expected_name_and_platforms, names_and_platforms),
)
# Check that the path is under the test's build root, so we know the pex was created there.
self.assertTrue(path.startswith(os.path.realpath(get_buildroot())))
def test_iter_ordering():
pi = PythonInterpreter.get()
tgz = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py%s-%s.egg' % (pi.python, get_build_platform()))
whl = WheelPackage('psutil-0.6.1-cp%s-none-%s.whl' % (
pi.python.replace('.', ''),
get_build_platform().replace('-', '_').replace('.', '_').lower()))
req = Requirement.parse('psutil')
assert list(FakeObtainer([tgz, egg, whl]).iter(req)) == [whl, egg, tgz]
assert list(FakeObtainer([egg, tgz, whl]).iter(req)) == [whl, egg, tgz]
def test_resolvable_directory():
builder = ResolverOptionsBuilder()
interpreter = PythonInterpreter.get()
with make_source_dir(name='my_project') as td:
rdir = ResolvableDirectory.from_string(td, builder, interpreter)
assert rdir.name == pkg_resources.safe_name('my_project')
assert rdir.extras() == []
rdir = ResolvableDirectory.from_string(td + '[extra1,extra2]', builder, interpreter)
assert rdir.name == pkg_resources.safe_name('my_project')
assert rdir.extras() == ['extra1', 'extra2']
def interpreter_from_options(options):
interpreter = None
if options.python:
if os.path.exists(options.python):
interpreter = PythonInterpreter.from_binary(options.python)
else:
interpreter = PythonInterpreter.from_env(options.python)
if interpreter is None:
die('Failed to find interpreter: %s' % options.python)
else:
interpreter = PythonInterpreter.get()
return interpreter
def fake_interpreter(id_str):
interpreter_dir = safe_mkdtemp()
binary = os.path.join(interpreter_dir, 'binary')
with open(binary, 'w') as fp:
fp.write(dedent("""
#!{}
from __future__ import print_function
print({!r})
""".format(PythonInterpreter.get().binary, id_str)).strip())
chmod_plus_x(binary)
return PythonInterpreter.from_binary(binary)
def _cache_current_interpreter(self):
cache = PythonInterpreterCache(self.config())
# We only need to cache the current interpreter, avoid caching for every interpreter on the
# PATH.
current_interpreter = PythonInterpreter.get()
current_id = (current_interpreter.binary, current_interpreter.identity)
for cached_interpreter in cache.setup(filters=[current_interpreter.identity.requirement]):
# TODO(John Sirois): Revert to directly comparing interpreters when
# https://github.com/pantsbuild/pex/pull/31 is in, released and consumed by pants.
if (cached_interpreter.binary, cached_interpreter.identity) == current_id:
return cached_interpreter
raise RuntimeError('Could not find suitable interpreter to run tests.')
def __init__(self, allow_prereleases=None, interpreter=None, platform=None, use_manylinux=None):
self._interpreter = interpreter or PythonInterpreter.get()
self._platform = self._maybe_expand_platform(self._interpreter, platform)
self._allow_prereleases = allow_prereleases
platform_name = self._platform.platform
self._target_interpreter_env = self._interpreter.identity.pkg_resources_env(platform_name)
self._supported_tags = self._platform.supported_tags(
self._interpreter,
use_manylinux
)
TRACER.log(
'R: tags for %r x %r -> %s' % (self._platform, self._interpreter, self._supported_tags),
V=9
)
def bootstrap_conan(self):
pex_info = PexInfo.default()
pex_info.entry_point = 'conans.conan'
conan_bootstrap_dir = os.path.join(get_pants_cachedir(), 'conan_support')
conan_pex_path = os.path.join(conan_bootstrap_dir, 'conan_binary')
interpreter = PythonInterpreter.get()
if not os.path.exists(conan_pex_path):
with safe_concurrent_creation(conan_pex_path) as safe_path:
builder = PEXBuilder(safe_path, interpreter, pex_info=pex_info)
reqs = [PythonRequirement(req) for req in self.get_options().conan_requirements]
dump_requirements(builder, interpreter, reqs, logger)
builder.freeze()
conan_binary = PEX(conan_pex_path, interpreter)
return self.ConanBinary(pex=conan_binary)
def resolve_multi(config,
requirements,
interpreter=None,
platforms=None,
conn_timeout=None,
ttl=3600):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param config: Pants :class:`Config` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param conn_timeout: Optional connection timeout for any remote fetching.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got %s' % type(interpreter))
install_cache = PythonSetup(config).scratch_dir('install_cache', default_name='eggs')
platforms = get_platforms(platforms or config.getlist('python-setup', 'platforms', ['current']))
for platform in platforms:
translator = Translator.default(
install_cache=install_cache,
interpreter=interpreter,
platform=platform,
conn_timeout=conn_timeout)
obtainer = PantsObtainer(
install_cache=install_cache,
crawler=crawler_from_config(config, conn_timeout=conn_timeout),
fetchers=fetchers_from_config(config) or [PyPIFetcher()],
translators=translator)
distributions[platform] = resolve(requirements=requirements,
obtainer=obtainer,
interpreter=interpreter,
platform=platform)
return distributions
def test_sorter_sort():
pi = PythonInterpreter.get()
tgz = SourcePackage('psutil-0.6.1.tar.gz')
egg = EggPackage('psutil-0.6.1-py%s-%s.egg' % (pi.python, get_build_platform()))
whl = WheelPackage('psutil-0.6.1-cp%s-none-%s.whl' % (
pi.python.replace('.', ''),
get_build_platform().replace('-', '_').replace('.', '_').lower()))
assert Sorter().sort([tgz, egg, whl]) == [whl, egg, tgz]
assert Sorter().sort([egg, tgz, whl]) == [whl, egg, tgz]
# test unknown type
sorter = Sorter(precedence=(EggPackage, WheelPackage))
assert sorter.sort([egg, tgz, whl], filter=False) == [egg, whl, tgz]
assert sorter.sort([egg, tgz, whl], filter=True) == [egg, whl]
def resolve_multi(python_setup,
python_repos,
requirements,
interpreter=None,
platforms=None,
ttl=3600,
find_links=None):
"""Multi-platform dependency resolution for PEX files.
Given a pants configuration and a set of requirements, return a list of distributions
that must be included in order to satisfy them. That may involve distributions for
multiple platforms.
:param python_setup: Pants :class:`PythonSetup` object.
:param python_repos: Pants :class:`PythonRepos` object.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param interpreter: :class:`PythonInterpreter` for which requirements should be resolved.
If None specified, defaults to current interpreter.
:param platforms: Optional list of platforms against requirements will be resolved. If
None specified, the defaults from `config` will be used.
:param ttl: Time in seconds before we consider re-resolving an open-ended requirement, e.g.
"flask>=0.2" if a matching distribution is available on disk. Defaults
to 3600.
:param find_links: Additional paths to search for source packages during resolution.
"""
distributions = dict()
interpreter = interpreter or PythonInterpreter.get()
if not isinstance(interpreter, PythonInterpreter):
raise TypeError('Expected interpreter to be a PythonInterpreter, got {}'.format(type(interpreter)))
cache = os.path.join(python_setup.scratch_dir, 'eggs')
platforms = get_platforms(platforms or python_setup.platforms)
fetchers = python_repos.get_fetchers()
if find_links:
fetchers.extend(Fetcher([path]) for path in find_links)
context = python_repos.get_network_context()
for platform in platforms:
distributions[platform] = resolve(
requirements=requirements,
interpreter=interpreter,
fetchers=fetchers,
platform=platform,
context=context,
cache=cache,
cache_ttl=ttl)
return distributions
def expand_and_maybe_adjust_platform(interpreter, platform):
"""Adjusts `platform` if it is 'current' and does not match the given `interpreter` platform.
:param interpreter: The target interpreter for the given `platform`.
:type interpreter: :class:`pex.interpreter.PythonInterpreter`
:param platform: The platform name to expand and maybe adjust.
:type platform: text
:returns: The `platform`, potentially adjusted.
:rtype: :class:`pex.platforms.Platform`
"""
# TODO(John Sirois): Kill all usages when https://github.com/pantsbuild/pex/issues/511 is fixed.
cur_plat = Platform.current()
if cur_plat.platform != Platform.create(platform).platform:
# IE: Say we're on OSX and platform was 'linux-x86_64' or 'linux_x86_64-cp-27-cp27mu'.
return Platform.create(platform)
ii = interpreter.identity
if (ii.abbr_impl, ii.impl_ver, ii.abi_tag) == (cur_plat.impl, cur_plat.version, cur_plat.abi):
# IE: Say we're on Linux and platform was 'current' or 'linux-x86_64' or
# 'linux_x86_64-cp-27-cp27mu'and the current extended platform info matches the given
# interpreter exactly.
return cur_plat
# Otherwise we need to adjust the platform to match a local interpreter different from the
# currently executing interpreter.
interpreter_platform = Platform(platform=cur_plat.platform,
impl=ii.abbr_impl,
version=ii.impl_ver,
abi=ii.abi_tag)
logger.debug("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=_interpreter_str(PythonInterpreter.get()),
given_interpreter=_interpreter_str(interpreter),
calculated_platform=interpreter_platform)
)
return interpreter_platform
def _gather_sources(self, target_roots):
context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])
# We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
# to ensure that the interpreter has setuptools and wheel support.
interpreter = PythonInterpreter.get()
interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
PythonRepos.global_instance(),
logger=context.log.debug)
interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(GatherSources.PYTHON_SOURCES)
def _resolve_requirements(self, target_roots, options=None):
context = self.context(target_roots=target_roots, options=options)
# We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
# to ensure that the interpreter has setuptools and wheel support.
interpreter = PythonInterpreter.get()
interpreter_cache = PythonInterpreterCache(
PythonSetup.global_instance(), PythonRepos.global_instance(), logger=context.log.debug
)
interpreters = interpreter_cache.setup(
paths=[os.path.dirname(interpreter.binary)], filters=[str(interpreter.identity.requirement)]
)
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
def setup_interpreter(distributions, interpreter=None):
"""Return an interpreter configured with vendored distributions as extras.
Any distributions that are present in the vendored set will be added to the interpreter as extras.
:param distributions: The names of distributions to setup the interpreter with.
:type distributions: list of str
:param interpreter: An optional interpreter to configure. If ``None``, the current interpreter is
used.
:type interpreter: :class:`pex.interpreter.PythonInterpreter`
:return: An bare interpreter configured with vendored extras.
:rtype: :class:`pex.interpreter.PythonInterpreter`
"""
from pex.interpreter import PythonInterpreter
interpreter = interpreter or PythonInterpreter.get()
for dist in _vendored_dists(OrderedSet(distributions)):
interpreter = interpreter.with_extra(dist.key, dist.version, dist.location)
return interpreter
def _resolve_requirements(self, target_roots, options=None):
with temporary_dir() as cache_dir:
options = options or {}
python_setup_opts = options.setdefault(PythonSetup.options_scope, {})
python_setup_opts['interpreter_cache_dir'] = cache_dir
interpreter = PythonInterpreter.get()
python_setup_opts['interpreter_search_paths'] = [os.path.dirname(interpreter.binary)]
context = self.context(target_roots=target_roots, options=options,
for_subsystems=[PythonInterpreterCache])
# We must get an interpreter via the cache, instead of using the value of
# PythonInterpreter.get() directly, to ensure that the interpreter has setuptools and
# wheel support.
interpreter_cache = PythonInterpreterCache.global_instance()
interpreters = interpreter_cache.setup(filters=[str(interpreter.identity.requirement)])
context.products.get_data(PythonInterpreter, lambda: interpreters[0])
task = self.create_task(context)
task.execute()
return context.products.get_data(ResolveRequirements.REQUIREMENTS_PEX)
def expand_platform():
expanded_platform = Platform(platform=cur_plat.platform,
impl=interpreter.identity.abbr_impl,
version=interpreter.identity.impl_ver,
abi=interpreter.identity.abi_tag)
TRACER.log("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=PythonInterpreter.get(),
given_interpreter=interpreter,
calculated_platform=expanded_platform),
V=9
)
return expanded_platform
