def test_platform_create():
assert Platform.create('linux-x86_64') == ('linux_x86_64', None, None,
None)
assert Platform.create('linux-x86_64-cp-27-cp27mu') == ('linux_x86_64',
'cp', '27',
'cp27mu')
assert Platform.create('linux-x86_64-cp-27-mu') == ('linux_x86_64', 'cp',
'27', 'cp27mu')
assert Platform.create('macosx-10.4-x86_64-cp-27-m') == (
'macosx_10_4_x86_64', 'cp', '27', 'cp27m')
def test_platform_create():
assert Platform.create("linux-x86_64-cp-27-cp27mu") == ("linux_x86_64",
"cp", "27",
"cp27mu")
assert Platform.create("linux-x86_64-cp-27-mu") == ("linux_x86_64", "cp",
"27", "cp27mu")
assert Platform.create("macosx-10.4-x86_64-cp-27-m") == (
"macosx_10_4_x86_64",
"cp",
"27",
"cp27m",
)
def _build_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
self.context.log.debug(" Dumping requirement: {}".format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
self.context.log.debug("Skipping {} based on version filter".format(req))
# Resolve the requirements into distributions.
distributions = self._resolve_multi(interpreter, reqs_to_build, find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self.context.log.debug(" Dumping distribution: .../{}".format(os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
builder.freeze()
def __init__(self, pex, pex_info, interpreter=None, **kw):
self._internal_cache = os.path.join(pex, pex_info.internal_cache)
self._pex = pex
self._pex_info = pex_info
self._activated = False
self._working_set = None
self._interpreter = interpreter or PythonInterpreter.get()
self._inherit_path = pex_info.inherit_path
self._supported_tags = []
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._interpreter.identity.abbr_impl == 'pp' and zipfile.is_zipfile(
self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
platform = Platform.current()
platform_name = platform.platform
super(PEXEnvironment, self).__init__(
search_path=[] if pex_info.inherit_path == 'false' else sys.path,
# NB: Our pkg_resources.Environment base-class wants the platform name string and not the
# pex.platform.Platform object.
platform=platform_name,
**kw)
self._target_interpreter_env = self._interpreter.identity.pkg_resources_env(
platform_name)
self._supported_tags.extend(platform.supported_tags(self._interpreter))
TRACER.log('E: tags for %r x %r -> %s' %
(self.platform, self._interpreter, self._supported_tags),
V=9)
def test_platform_supported_tags_abi3():
tags = Platform.create('linux-x86_64-cp-37-m').supported_tags()
expected_tags = [
('cp37', 'cp37m', 'linux_x86_64'),
('cp37', 'cp37m', 'manylinux1_x86_64'),
('cp37', 'abi3', 'linux_x86_64'),
('cp37', 'abi3', 'manylinux1_x86_64'),
('cp37', 'none', 'linux_x86_64'),
('cp37', 'none', 'manylinux1_x86_64'),
('cp36', 'abi3', 'linux_x86_64'),
('cp36', 'abi3', 'manylinux1_x86_64'),
('cp35', 'abi3', 'linux_x86_64'),
('cp35', 'abi3', 'manylinux1_x86_64'),
('cp34', 'abi3', 'linux_x86_64'),
('cp34', 'abi3', 'manylinux1_x86_64'),
('cp33', 'abi3', 'linux_x86_64'),
('cp33', 'abi3', 'manylinux1_x86_64'),
('cp32', 'abi3', 'linux_x86_64'),
('cp32', 'abi3', 'manylinux1_x86_64'),
('py3', 'none', 'linux_x86_64'),
('py3', 'none', 'manylinux1_x86_64'),
('cp37', 'none', 'any'),
('cp3', 'none', 'any'),
('py37', 'none', 'any'),
('py3', 'none', 'any'),
('py36', 'none', 'any'),
('py35', 'none', 'any'),
('py34', 'none', 'any'),
('py33', 'none', 'any'),
('py32', 'none', 'any'),
('py31', 'none', 'any'),
('py30', 'none', 'any'),
]
assert expected_tags == tags
def main():
parser = configure_clp()
options, args = parser.parse_args()
verbosity = 5 if options.verbosity else -1
with Tracer.env_override(PEX_VERBOSE=verbosity, PEX_HTTP=verbosity):
pex_builder = build_pex(args, options)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
return pex.run(args=list(args))
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main():
parser = configure_clp()
options, args = parser.parse_args()
with TraceLogger.env_override(PEX_VERBOSE=options.verbosity):
pex_builder = build_pex(args, options)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
return pex.run(args=list(args))
def test_platform_supported_tags_abi3():
tags = Platform.create('linux-x86_64-cp-37-m').supported_tags()
expected_tags = [
('cp37', 'cp37m', 'linux_x86_64'),
('cp37', 'cp37m', 'manylinux1_x86_64'),
('cp37', 'abi3', 'linux_x86_64'),
('cp37', 'abi3', 'manylinux1_x86_64'),
('cp37', 'none', 'linux_x86_64'),
('cp37', 'none', 'manylinux1_x86_64'),
('cp36', 'abi3', 'linux_x86_64'),
('cp36', 'abi3', 'manylinux1_x86_64'),
('cp35', 'abi3', 'linux_x86_64'),
('cp35', 'abi3', 'manylinux1_x86_64'),
('cp34', 'abi3', 'linux_x86_64'),
('cp34', 'abi3', 'manylinux1_x86_64'),
('cp33', 'abi3', 'linux_x86_64'),
('cp33', 'abi3', 'manylinux1_x86_64'),
('cp32', 'abi3', 'linux_x86_64'),
('cp32', 'abi3', 'manylinux1_x86_64'),
('py3', 'none', 'linux_x86_64'),
('py3', 'none', 'manylinux1_x86_64'),
('cp37', 'none', 'any'),
('cp3', 'none', 'any'),
('py37', 'none', 'any'),
('py3', 'none', 'any'),
('py36', 'none', 'any'),
('py35', 'none', 'any'),
('py34', 'none', 'any'),
('py33', 'none', 'any'),
('py32', 'none', 'any'),
('py31', 'none', 'any'),
('py30', 'none', 'any'),
]
assert expected_tags == tags
def __init__(self, pex, pex_info, interpreter=None, **kw):
self._internal_cache = os.path.join(pex, pex_info.internal_cache)
self._pex = pex
self._pex_info = pex_info
self._activated = False
self._working_set = None
self._interpreter = interpreter or PythonInterpreter.get()
self._inherit_path = pex_info.inherit_path
self._supported_tags = []
# For the bug this works around, see: https://bitbucket.org/pypy/pypy/issues/1686
# NB: This must be installed early before the underlying pex is loaded in any way.
if self._interpreter.identity.abbr_impl == 'pp' and zipfile.is_zipfile(self._pex):
self._install_pypy_zipimporter_workaround(self._pex)
platform = Platform.current()
platform_name = platform.platform
super(PEXEnvironment, self).__init__(
search_path=[] if pex_info.inherit_path == 'false' else sys.path,
# NB: Our pkg_resources.Environment base-class wants the platform name string and not the
# pex.platform.Platform object.
platform=platform_name,
**kw
)
self._target_interpreter_env = self._interpreter.identity.pkg_resources_env(platform_name)
self._supported_tags.extend(platform.supported_tags(self._interpreter))
TRACER.log(
'E: tags for %r x %r -> %s' % (self.platform, self._interpreter, self._supported_tags),
V=9
)
def expand_and_maybe_adjust_platform(interpreter, platform):
"""Adjusts `platform` if it is 'current' and does not match the given `interpreter` platform.
:param interpreter: The target interpreter for the given `platform`.
:type interpreter: :class:`pex.interpreter.PythonInterpreter`
:param platform: The platform name to expand and maybe adjust.
:type platform: text
:returns: The `platform`, potentially adjusted.
:rtype: :class:`pex.platforms.Platform`
"""
# TODO(John Sirois): Kill all usages when https://github.com/pantsbuild/pex/issues/511 is fixed.
cur_plat = Platform.current()
if cur_plat.platform != Platform.create(platform).platform:
# IE: Say we're on OSX and platform was 'linux-x86_64' or 'linux_x86_64-cp-27-cp27mu'.
return Platform.create(platform)
ii = interpreter.identity
if (ii.abbr_impl, ii.impl_ver,
ii.abi_tag) == (cur_plat.impl, cur_plat.version, cur_plat.abi):
# IE: Say we're on Linux and platform was 'current' or 'linux-x86_64' or
# 'linux_x86_64-cp-27-cp27mu'and the current extended platform info matches the given
# interpreter exactly.
return cur_plat
# Otherwise we need to adjust the platform to match a local interpreter different from the
# currently executing interpreter.
interpreter_platform = Platform(platform=cur_plat.platform,
impl=ii.abbr_impl,
version=ii.impl_ver,
abi=ii.abi_tag)
logger.debug("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=_interpreter_str(PythonInterpreter.get()),
given_interpreter=_interpreter_str(interpreter),
calculated_platform=interpreter_platform))
return interpreter_platform
def expand_and_maybe_adjust_platform(interpreter, platform):
"""Adjusts `platform` if it is 'current' and does not match the given `interpreter` platform.
:param interpreter: The target interpreter for the given `platform`.
:type interpreter: :class:`pex.interpreter.PythonInterpreter`
:param platform: The platform name to expand and maybe adjust.
:type platform: text
:returns: The `platform`, potentially adjusted.
:rtype: :class:`pex.platforms.Platform`
"""
# TODO(John Sirois): Kill all usages when https://github.com/pantsbuild/pex/issues/511 is fixed.
cur_plat = Platform.current()
if cur_plat.platform != Platform.create(platform).platform:
# IE: Say we're on OSX and platform was 'linux-x86_64' or 'linux_x86_64-cp-27-cp27mu'.
return Platform.create(platform)
ii = interpreter.identity
if (ii.abbr_impl, ii.impl_ver, ii.abi_tag) == (cur_plat.impl, cur_plat.version, cur_plat.abi):
# IE: Say we're on Linux and platform was 'current' or 'linux-x86_64' or
# 'linux_x86_64-cp-27-cp27mu'and the current extended platform info matches the given
# interpreter exactly.
return cur_plat
# Otherwise we need to adjust the platform to match a local interpreter different from the
# currently executing interpreter.
interpreter_platform = Platform(platform=cur_plat.platform,
impl=ii.abbr_impl,
version=ii.impl_ver,
abi=ii.abi_tag)
logger.debug("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=_interpreter_str(PythonInterpreter.get()),
given_interpreter=_interpreter_str(interpreter),
calculated_platform=interpreter_platform)
)
return interpreter_platform
def _maybe_expand_platform(interpreter, platform=None):
# Expands `platform` if it is 'current' and abbreviated.
#
# IE: If we're on linux and handed a platform of `None`, 'current', or 'linux_x86_64', we expand
# the platform to an extended platform matching the given interpreter's abi info, eg:
# 'linux_x86_64-cp-27-cp27mu'.
cur_plat = Platform.current()
def expand_platform():
expanded_platform = Platform(platform=cur_plat.platform,
impl=interpreter.identity.abbr_impl,
version=interpreter.identity.impl_ver,
abi=interpreter.identity.abi_tag)
TRACER.log("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=PythonInterpreter.get(),
given_interpreter=interpreter,
calculated_platform=expanded_platform),
V=9)
return expanded_platform
if platform in (None, 'current'):
# Always expand the default local (abbreviated) platform to the given interpreter.
return expand_platform()
else:
given_platform = Platform.create(platform)
if given_platform.is_extended:
# Always respect an explicit extended platform.
return given_platform
elif given_platform.platform != cur_plat.platform:
# IE: Say we're on OSX and platform was 'linux-x86_64'; we can't expand a non-local
# platform so we leave as-is.
return given_platform
else:
# IE: Say we're on 64 bit linux and platform was 'linux-x86_64'; ie: the abbreviated local
# platform.
return expand_platform()
def iter_supported_platforms(self):
# type: () -> Iterator[Platform]
"""All platforms supported by the associated interpreter ordered from most specific to
least."""
for tags in self._supported_tags:
yield Platform.from_tags(platform=tags.platform,
python=tags.interpreter,
abi=tags.abi)
def _maybe_expand_platform(interpreter, platform=None):
# Expands `platform` if it is 'current' and abbreviated.
#
# IE: If we're on linux and handed a platform of `None`, 'current', or 'linux_x86_64', we expand
# the platform to an extended platform matching the given interpreter's abi info, eg:
# 'linux_x86_64-cp-27-cp27mu'.
cur_plat = Platform.current()
def expand_platform():
expanded_platform = Platform(platform=cur_plat.platform,
impl=interpreter.identity.abbr_impl,
version=interpreter.identity.impl_ver,
abi=interpreter.identity.abi_tag)
TRACER.log("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=PythonInterpreter.get(),
given_interpreter=interpreter,
calculated_platform=expanded_platform),
V=9
)
return expanded_platform
if platform in (None, 'current'):
# Always expand the default local (abbreviated) platform to the given interpreter.
return expand_platform()
else:
given_platform = Platform.create(platform)
if given_platform.is_extended:
# Always respect an explicit extended platform.
return given_platform
elif given_platform.platform != cur_plat.platform:
# IE: Say we're on OSX and platform was 'linux-x86_64'; we can't expand a non-local
# platform so we leave as-is.
return given_platform
else:
# IE: Say we're on 64 bit linux and platform was 'linux-x86_64'; ie: the abbreviated local
# platform.
return expand_platform()
def dump(self):
self.debug('Building chroot for {}:'.format(self._targets))
targets = self.resolve(self._targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
find_links = []
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping {} based upon version filter'.format(req))
continue
reqs_to_build.add(req)
self._dump_requirement(req.requirement)
if req.repository:
find_links.append(req.repository)
distributions = resolve_multi(
self._python_setup,
self._python_repos,
reqs_to_build,
interpreter=self._interpreter,
platforms=self._platforms,
ttl=self.context.options.for_global_scope().python_chroot_requirements_ttl,
find_links=find_links)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
return self._builder
def dump(self):
self.debug('Building chroot for %s:' % self._targets)
targets = self.resolve(self._targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[
thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
find_links = []
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python,
Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req.requirement)
if req.repository:
find_links.append(req.repository)
distributions = resolve_multi(self._config,
reqs_to_build,
interpreter=self._interpreter,
platforms=self._platforms,
find_links=find_links)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!',
file=sys.stderr)
return self._builder
def get_local_platform(): """Returns the name of the local platform; eg: 'linux_x86_64' or 'macosx_10_8_x86_64'. :returns: The local platform name. :rtype: str """ # TODO(John Sirois): Kill some or all usages when https://github.com/pantsbuild/pex/issues/511 # is fixed. current_platform = Platform.current() return current_platform.platform
def get_local_platform():
"""Returns the name of the local platform; eg: 'linux_x86_64' or 'macosx_10_8_x86_64'.
:returns: The local platform name.
:rtype: str
"""
# TODO(John Sirois): Kill some or all usages when https://github.com/pantsbuild/pex/issues/511
# is fixed.
current_platform = Platform.current()
return current_platform.platform
def iter_supported_platforms(self):
"""All platforms supported by the associated interpreter ordered from most specific to
least.
:rtype: iterator of :class:`Platform`
"""
for tags in self._supported_tags:
yield Platform.from_tags(platform=tags.platform,
python=tags.interpreter,
abi=tags.abi)
def test_platform_supported_tags_manylinux():
platform = Platform.create("linux-x86_64-cp-37-cp37m")
tags = frozenset(platform.supported_tags())
manylinux1_tags = frozenset(
platform.supported_tags(manylinux="manylinux1"))
manylinux2010_tags = frozenset(
platform.supported_tags(manylinux="manylinux2010"))
manylinux2014_tags = frozenset(
platform.supported_tags(manylinux="manylinux2014"))
assert manylinux2014_tags > manylinux2010_tags > manylinux1_tags > tags
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=options.pex_root) as patched_env:
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!',
V=1)
log('Running on platform {} but built for {}'.format(
Platform.current(), ', '.join(map(str,
options.platforms))),
V=1)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def test_platform_supported_tags():
platform = Platform.create("macosx-10.13-x86_64-cp-36-m")
# A golden file test. This could break if we upgrade Pip and it upgrades packaging which, from
# time to time, corrects omissions in tag sets.
assert (tuple(
itertools.chain.from_iterable(
tags.parse_tag(tag) for tag in pkgutil.get_data(
__name__, "data/platforms/macosx_10_13_x86_64-cp-36-m.tags.txt"
).decode("utf-8").splitlines()
if not tag.startswith("#"))) == platform.supported_tags())
def test_unknown(self):
with pytest.raises(Platform.UnknownPlatformError):
Platform.compatible('macosx-10.0-morfgorf', 'macosx-10.1-morfgorf')
with pytest.raises(Platform.UnknownPlatformError):
Platform.compatible('macosx-10.0-x86_64', 'macosx-10.1-morfgorf')
with pytest.raises(Platform.UnknownPlatformError):
Platform.compatible('macosx-10.0-morfgorf', 'macosx-10.1-x86_64')
def test_unknown(self):
with pytest.raises(Platform.UnknownPlatformError):
Platform.compatible("macosx-10.0-morfgorf", "macosx-10.1-morfgorf")
with pytest.raises(Platform.UnknownPlatformError):
Platform.compatible("macosx-10.0-x86_64", "macosx-10.1-morfgorf")
with pytest.raises(Platform.UnknownPlatformError):
Platform.compatible("macosx-10.0-morfgorf", "macosx-10.1-x86_64")
def checker_pex(self, interpreter):
# TODO(John Sirois): Formalize in pants.base?
pants_dev_mode = os.environ.get('PANTS_DEV')
if pants_dev_mode:
checker_id = self.checker_target.transitive_invalidation_hash()
else:
checker_id = hash_all([self._CHECKER_REQ])
pex_path = os.path.join(self.workdir, 'checker', checker_id, str(interpreter.identity))
if not os.path.exists(pex_path):
with self.context.new_workunit(name='build-checker'):
with safe_concurrent_creation(pex_path) as chroot:
pex_builder = PexBuilderWrapper.Factory.create(
builder=PEXBuilder(path=chroot, interpreter=interpreter),
log=self.context.log)
# Constraining is required to guard against the case where the user
# has a pexrc file set.
pex_builder.add_interpreter_constraint(str(interpreter.identity.requirement))
if pants_dev_mode:
pex_builder.add_sources_from(self.checker_target)
req_libs = [tgt for tgt in self.checker_target.closure()
if isinstance(tgt, PythonRequirementLibrary)]
pex_builder.add_requirement_libs_from(req_libs=req_libs)
else:
try:
# The checker is already on sys.path, eg: embedded in pants.pex.
platform = Platform.current()
platform_name = platform.platform
env = Environment(search_path=sys.path,
platform=platform_name,
python=interpreter.version_string)
working_set = WorkingSet(entries=sys.path)
for dist in working_set.resolve([Requirement.parse(self._CHECKER_REQ)], env=env):
pex_builder.add_direct_requirements(dist.requires())
# NB: We add the dist location instead of the dist itself to make sure its a
# distribution style pex knows how to package.
pex_builder.add_dist_location(dist.location)
pex_builder.add_direct_requirements([self._CHECKER_REQ])
except (DistributionNotFound, PEXBuilder.InvalidDistribution):
# We need to resolve the checker from a local or remote distribution repo.
pex_builder.add_resolved_requirements(
[PythonRequirement(self._CHECKER_REQ)])
pex_builder.set_entry_point(self._CHECKER_ENTRYPOINT)
pex_builder.freeze()
return PEX(pex_path, interpreter=interpreter)
def parsed_platform(platform=None):
"""Parse the given platform into a `Platform` object.
Unlike `Platform.create`, this function supports the special platform of 'current' or `None`. This
maps to the platform of any local python interpreter.
:param platform: The platform string to parse. If `None` or 'current', return `None`. If already a
`Platform` object, return it.
:type platform: str or :class:`Platform`
:return: The parsed platform or `None` for the current platform.
:rtype: :class:`Platform` or :class:`NoneType`
"""
return Platform.create(platform) if platform and platform != 'current' else None
def test_run_pex():
incompatible_platforms_warning_msg = 'WARNING: attempting to run PEX with incompatible platforms!'
assert incompatible_platforms_warning_msg not in assert_run_pex()
assert incompatible_platforms_warning_msg not in assert_run_pex(
pex_args=['--platform=current'])
assert incompatible_platforms_warning_msg not in assert_run_pex(
pex_args=['--platform={}'.format(Platform.current())])
py27 = ensure_python_interpreter(PY27)
stderr_lines = assert_run_pex(
python=py27, pex_args=['--platform=macosx-10.13-x86_64-cp-37-m'])
assert incompatible_platforms_warning_msg in stderr_lines
def test_resolve_multiplatform_requirements(self):
cffi_tgt = self._fake_target('cffi', ['cffi==1.9.1'])
pex = self._resolve_requirements(
[cffi_tgt],
{
'python-setup': {
# We have 'current' so we can import the module in order to get the path to it.
# The other platforms (one of which may happen to be the same as current) are what we
# actually test the presence of.
'platforms': [
'current', 'macosx-10.10-x86_64', 'manylinux1_i686',
'win_amd64'
]
}
})
stdout_data, stderr_data = self._exercise_module(pex, 'cffi')
self.assertEquals('', stderr_data.strip())
path = stdout_data.strip()
wheel_dir = os.path.join(
path[0:path.find('{sep}.deps{sep}'.format(sep=os.sep))], '.deps')
wheels = set(os.listdir(wheel_dir))
def name_and_platform(whl):
# The wheel filename is of the format
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
# See https://www.python.org/dev/peps/pep-0425/.
# We don't care about the python or abi versions (they depend on what we're currently
# running on), we just want to make sure we have all the platforms we expect.
parts = os.path.splitext(whl)[0].split('-')
return '{}-{}'.format(parts[0], parts[1]), parts[-1]
names_and_platforms = set(name_and_platform(w) for w in wheels)
expected_name_and_platforms = {
# Note that Platform.current() may happen to be the same as one of the other platforms.
('cffi-1.9.1', Platform.current().replace('-', '_')),
('cffi-1.9.1', 'macosx_10_10_x86_64'),
('cffi-1.9.1', 'manylinux1_i686'),
('cffi-1.9.1', 'win_amd64'),
}
# pycparser is a dependency of cffi only on CPython. We might as well check for it,
# as extra verification that we correctly fetch transitive dependencies.
if PythonInterpreter.get().identity.interpreter == 'CPython':
expected_name_and_platforms.add(('pycparser-2.17', 'any'))
self.assertEquals(expected_name_and_platforms, names_and_platforms)
# Check that the path is under the test's build root, so we know the pex was created there.
self.assertTrue(path.startswith(os.path.realpath(get_buildroot())))
def dump(self):
self.debug('Building chroot for %s:' % self._targets)
targets = self.resolve(self._targets)
for lib in targets['libraries'] | targets['binaries']:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets['thrifts']:
for thr in set(targets['thrifts']):
if thr not in self.MEMOIZED_THRIFTS:
self.MEMOIZED_THRIFTS[thr] = self._generate_thrift_requirement(thr)
generated_reqs.add(self.MEMOIZED_THRIFTS[thr])
generated_reqs.add(PythonRequirement('thrift', use_2to3=True))
for antlr in targets['antlrs']:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets['reqs']:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug('Skipping %s based upon version filter' % req)
continue
reqs_to_build.add(req)
self._dump_requirement(req._requirement, False, req._repository)
distributions = resolve_multi(
self._config,
reqs_to_build,
interpreter=self._interpreter,
platforms=self._platforms,
conn_timeout=self._conn_timeout)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets['binaries']) > 1:
print('WARNING: Target has multiple python_binary targets!', file=sys.stderr)
return self._builder
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform and Platform.current() not in options.platform:
log('WARNING: attempting to run PEX with incompatible platforms!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def test_versioning(self):
# Major versions incompatible
assert not Platform.compatible("macosx-9.1-x86_64", "macosx-10.0-x86_64")
assert not Platform.compatible("macosx-10.0-x86_64", "macosx-9.1-x86_64")
# Platforms equal
assert Platform.compatible("macosx-10.0-x86_64", "macosx-10.0-x86_64")
# Minor versions less than
assert Platform.compatible("macosx-10.0-x86_64", "macosx-10.1-x86_64")
assert not Platform.compatible("macosx-10.1-x86_64", "macosx-10.0-x86_64")
assert Platform.compatible("macosx-10.9-x86_64", "macosx-10.10-x86_64")
assert not Platform.compatible("macosx-10.10-x86_64", "macosx-10.9-x86_64")
def dump(self):
self.debug("Building chroot for {}:".format(self._targets))
targets = self.resolve(self._targets)
for lib in targets["libraries"] | targets["binaries"]:
self._dump_library(lib)
generated_reqs = OrderedSet()
if targets["thrifts"]:
for thr in targets["thrifts"]:
generated_reqs.add(self._generate_thrift_requirement(thr))
generated_reqs.add(PythonRequirement("thrift", use_2to3=True))
for antlr in targets["antlrs"]:
generated_reqs.add(self._generate_antlr_requirement(antlr))
reqs_from_libraries = OrderedSet()
for req_lib in targets["reqs"]:
for req in req_lib.payload.requirements:
reqs_from_libraries.add(req)
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs_from_libraries | generated_reqs | self._extra_requirements:
if not req.should_build(self._interpreter.python, Platform.current()):
self.debug("Skipping {} based upon version filter".format(req))
continue
reqs_to_build.add(req)
self._dump_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
distributions = self._resolve_multi(reqs_to_build, find_links)
locations = set()
for platform, dist_set in distributions.items():
for dist in dist_set:
if dist.location not in locations:
self._dump_distribution(dist)
locations.add(dist.location)
if len(targets["binaries"]) > 1:
print("WARNING: Target has multiple python_binary targets!", file=sys.stderr)
return self._builder
def dump_requirements(builder, interpreter, req_libs, log, platforms=None):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
log.debug(' Dumping requirement: {}'.format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
log.debug(' Skipping {} based on version filter'.format(req))
# Resolve the requirements into distributions.
distributions = _resolve_multi(interpreter, reqs_to_build, platforms,
find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
log.debug(' Dumping distribution: .../{}'.format(
os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform and Platform.current() not in options.platform:
log('WARNING: attempting to run PEX with incompatible platforms!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def _resolve(self, working_set, reqs):
reqs = reqs[:]
unresolved_reqs = set()
resolveds = set()
environment = self._target_interpreter_env.copy()
environment['extra'] = list(set(itertools.chain(*(req.extras for req in reqs))))
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
unresolved_reqs.add(e.req.project_name)
if e.requirers:
unresolved_reqs.update(e.requirers)
unresolved_reqs = set([req.lower() for req in unresolved_reqs])
if unresolved_reqs:
TRACER.log('Unresolved requirements:')
for req in unresolved_reqs:
TRACER.log(' - %s' % req)
TRACER.log('Distributions contained within this pex:')
if not self._pex_info.distributions:
TRACER.log(' None')
else:
for dist in self._pex_info.distributions:
TRACER.log(' - %s' % dist)
if not self._pex_info.ignore_errors:
die(
'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % (
Platform.current(),
'\n'.join(str(r) for r in unresolved_reqs)
)
)
return resolveds
def dump_requirements(builder, interpreter, req_libs, log, platforms=None):
"""Multi-platform dependency resolution for PEX files.
Returns a list of distributions that must be included in order to satisfy a set of requirements.
That may involve distributions for multiple platforms.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
log.debug(' Dumping requirement: {}'.format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
log.debug(' Skipping {} based on version filter'.format(req))
# Resolve the requirements into distributions.
distributions = _resolve_multi(interpreter, reqs_to_build, platforms, find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
log.debug(' Dumping distribution: .../{}'.format(os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
def _resolve(self, working_set, reqs):
reqs = reqs[:]
unresolved_reqs = set()
resolveds = set()
environment = self._target_interpreter_env.copy()
environment['extra'] = list(
set(itertools.chain(*(req.extras for req in reqs))))
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
'Skipping activation of `%s` due to environment marker de-selection'
% req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
unresolved_reqs.add(e.req.project_name)
if e.requirers:
unresolved_reqs.update(e.requirers)
unresolved_reqs = set([req.lower() for req in unresolved_reqs])
if unresolved_reqs:
TRACER.log('Unresolved requirements:')
for req in unresolved_reqs:
TRACER.log(' - %s' % req)
TRACER.log('Distributions contained within this pex:')
if not self._pex_info.distributions:
TRACER.log(' None')
else:
for dist in self._pex_info.distributions:
TRACER.log(' - %s' % dist)
if not self._pex_info.ignore_errors:
die('Failed to execute PEX file, missing %s compatible dependencies for:\n%s'
% (Platform.current(), '\n'.join(
str(r) for r in unresolved_reqs)))
return resolveds
def test_versioning(self):
# Major versions incompatible
assert not Platform.compatible('macosx-9.1-x86_64',
'macosx-10.0-x86_64')
assert not Platform.compatible('macosx-10.0-x86_64',
'macosx-9.1-x86_64')
# Platforms equal
assert Platform.compatible('macosx-10.0-x86_64', 'macosx-10.0-x86_64')
# Minor versions less than
assert Platform.compatible('macosx-10.0-x86_64', 'macosx-10.1-x86_64')
assert not Platform.compatible('macosx-10.1-x86_64',
'macosx-10.0-x86_64')
assert Platform.compatible('macosx-10.9-x86_64', 'macosx-10.10-x86_64')
assert not Platform.compatible('macosx-10.10-x86_64',
'macosx-10.9-x86_64')
def expand_platform():
expanded_platform = Platform(platform=cur_plat.platform,
impl=interpreter.identity.abbr_impl,
version=interpreter.identity.impl_ver,
abi=interpreter.identity.abi_tag)
TRACER.log("""
Modifying given platform of {given_platform!r}:
Using the current platform of {current_platform!r}
Under current interpreter {current_interpreter!r}
To match given interpreter {given_interpreter!r}.
Calculated platform: {calculated_platform!r}""".format(
given_platform=platform,
current_platform=cur_plat,
current_interpreter=PythonInterpreter.get(),
given_interpreter=interpreter,
calculated_platform=expanded_platform),
V=9)
return expanded_platform
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1 :]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set("PEX_ROOT", options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed("Building pex"):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log("Saving PEX file to %s" % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + "~"
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log("WARNING: attempting to run PEX with differing platform!")
pex_builder.freeze()
log("Running PEX file at %s with args %s" % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def test_platform_subsets(self):
# Pure platform subset
assert Platform.compatible('macosx-10.0-i386', 'macosx-10.0-intel')
# Version and platform subset
assert Platform.compatible('macosx-10.0-i386', 'macosx-10.1-intel')
assert Platform.compatible('macosx-10.0-x86_64', 'macosx-10.1-intel')
# Intersecting sets of platform but not pure subset
assert Platform.compatible('macosx-10.0-fat', 'macosx-10.1-intel')
# Non-intersecting sets of platform
assert not Platform.compatible('macosx-10.0-ppc', 'macosx-10.1-intel')
# Test our common case
assert Platform.compatible('macosx-10.4-x86_64', 'macosx-10.7-intel')
def test_platform_subsets(self):
# Pure platform subset
assert Platform.compatible("macosx-10.0-i386", "macosx-10.0-intel")
# Version and platform subset
assert Platform.compatible("macosx-10.0-i386", "macosx-10.1-intel")
assert Platform.compatible("macosx-10.0-x86_64", "macosx-10.1-intel")
# Intersecting sets of platform but not pure subset
assert Platform.compatible("macosx-10.0-fat", "macosx-10.1-intel")
# Non-intersecting sets of platform
assert not Platform.compatible("macosx-10.0-ppc", "macosx-10.1-intel")
# Test our common case
assert Platform.compatible("macosx-10.4-x86_64", "macosx-10.7-intel")
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def _build_pex(self, interpreter, path, req_libs):
builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
# Gather and de-dup all requirements.
reqs = OrderedSet()
for req_lib in req_libs:
for req in req_lib.requirements:
reqs.add(req)
# See which ones we need to build.
reqs_to_build = OrderedSet()
find_links = OrderedSet()
for req in reqs:
# TODO: should_build appears to be hardwired to always be True. Get rid of it?
if req.should_build(interpreter.python, Platform.current()):
reqs_to_build.add(req)
self.context.log.debug(' Dumping requirement: {}'.format(req))
builder.add_requirement(req.requirement)
if req.repository:
find_links.add(req.repository)
else:
self.context.log.debug(
'Skipping {} based on version filter'.format(req))
# Resolve the requirements into distributions.
distributions = self._resolve_multi(interpreter, reqs_to_build,
find_links)
locations = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
self.context.log.debug(
' Dumping distribution: .../{}'.format(
os.path.basename(dist.location)))
builder.add_distribution(dist)
locations.add(dist.location)
builder.freeze()
def _get_matching_wheel_dir(cls, wheel_dir, module_name):
wheels = os.listdir(wheel_dir)
names_and_platforms = {w:cls._name_and_platform(w) for w in wheels}
for whl_filename, (name, platform) in names_and_platforms.items():
if cls._current_platform_abbreviation in platform:
# TODO: this guards against packages which have names that are prefixes of other packages by
# checking if there is a version number beginning -- is there a more canonical way to do
# this?
if re.match(r'^{}\-[0-9]'.format(re.escape(module_name)), name):
return os.path.join(wheel_dir, whl_filename, module_name)
raise cls._NativeCodeExtractionSetupFailure(
"Could not find wheel in dir '{wheel_dir}' matching module name '{module_name}' "
"for current platform '{pex_current_platform}', when looking for platforms containing the "
"substring {cur_platform_abbrev}.\n"
"wheels: {wheels}"
.format(wheel_dir=wheel_dir,
module_name=module_name,
pex_current_platform=Platform.current().platform,
cur_platform_abbrev=cls._current_platform_abbreviation,
wheels=wheels))
def configure_clp_pex_environment(parser):
group = OptionGroup(
parser, "PEX environment options", "Tailor the interpreter and platform targets for the PEX environment."
)
group.add_option(
"--python",
dest="python",
default=None,
help="The Python interpreter to use to build the pex. Either specify an explicit "
"path to an interpreter, or specify a binary accessible on $PATH. "
"Default: Use current interpreter.",
)
group.add_option(
"--python-shebang",
dest="python_shebang",
default=None,
help="The exact shebang (#!...) line to add at the top of the PEX file minus the "
"#!. This overrides the default behavior, which picks an environment python "
"interpreter compatible with the one used to build the PEX file.",
)
group.add_option(
"--platform",
dest="platform",
default=Platform.current(),
help="The platform for which to build the PEX. Default: %default",
)
group.add_option(
"--interpreter-cache-dir",
dest="interpreter_cache_dir",
default="{pex_root}/interpreters",
help="The interpreter cache to use for keeping track of interpreter dependencies "
"for the pex tool. [Default: ~/.pex/interpreters]",
)
parser.add_option_group(group)
def configure_clp_pex_environment(parser):
group = OptionGroup(
parser, 'PEX environment options',
'Tailor the interpreter and platform targets for the PEX environment.')
group.add_option(
'--python',
dest='python',
default=None,
help=
'The Python interpreter to use to build the pex. Either specify an explicit '
'path to an interpreter, or specify a binary accessible on $PATH. '
'Default: Use current interpreter.')
group.add_option(
'--python-shebang',
dest='python_shebang',
default=None,
help=
'The exact shebang (#!...) line to add at the top of the PEX file minus the '
'#!. This overrides the default behavior, which picks an environment python '
'interpreter compatible with the one used to build the PEX file.')
group.add_option(
'--platform',
dest='platform',
default=Platform.current(),
help='The platform for which to build the PEX. Default: %default')
group.add_option(
'--interpreter-cache-dir',
dest='interpreter_cache_dir',
default=os.path.expanduser('~/.pex/interpreters'),
help=
'The interpreter cache to use for keeping track of interpreter dependencies '
'for the pex tool. [Default: %default]')
parser.add_option_group(group)
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def configure_clp_pex_environment(parser):
group = OptionGroup(
parser,
'PEX environment options',
'Tailor the interpreter and platform targets for the PEX environment.')
group.add_option(
'--python',
dest='python',
default=None,
help='The Python interpreter to use to build the pex. Either specify an explicit '
'path to an interpreter, or specify a binary accessible on $PATH. '
'Default: Use current interpreter.')
group.add_option(
'--python-shebang',
dest='python_shebang',
default=None,
help='The exact shebang (#!...) line to add at the top of the PEX file minus the '
'#!. This overrides the default behavior, which picks an environment python '
'interpreter compatible with the one used to build the PEX file.')
group.add_option(
'--platform',
dest='platform',
default=Platform.current(),
help='The platform for which to build the PEX. Default: %default')
group.add_option(
'--interpreter-cache-dir',
dest='interpreter_cache_dir',
default='{pex_root}/interpreters',
help='The interpreter cache to use for keeping track of interpreter dependencies '
'for the pex tool. [Default: ~/.pex/interpreters]')
parser.add_option_group(group)
def get_platforms(platform_list):
return tuple({Platform.current() if p == 'current' else p for p in platform_list})
def assert_tags(platform, expected_tags, manylinux=None):
tags = Platform.create(platform).supported_tags(force_manylinux=manylinux)
for expected_tag in expected_tags:
assert expected_tag in tags
def test_platform_current():
assert Platform.create('current') == Platform.current()
def test_platform_create_noop():
existing = Platform.create('linux-x86_64')
assert Platform.create(existing) == existing
def test_platform_create():
assert Platform.create('linux-x86_64') == ('linux_x86_64', None, None, None)
assert Platform.create('linux-x86_64-cp-27-cp27mu') == ('linux_x86_64', 'cp', '27', 'cp27mu')
assert Platform.create('linux-x86_64-cp-27-mu') == ('linux_x86_64', 'cp', '27', 'cp27mu')
assert Platform.create(
'macosx-10.4-x86_64-cp-27-m') == ('macosx_10_4_x86_64', 'cp', '27', 'cp27m')
def configure_clp():
usage = (
'%prog [options]\n\n'
'%prog builds a PEX (Python Executable) file based on the given specifications: '
'sources, requirements, their dependencies and other options')
parser = OptionParser(usage=usage, version='%prog {0}'.format(__version__))
parser.add_option(
'--pypi', '--no-pypi',
dest='pypi',
default=True,
action='callback',
callback=parse_bool,
help='Whether to use pypi to resolve dependencies; Default: use pypi')
parser.add_option(
'--wheel', '--no-wheel',
dest='use_wheel',
default=True,
action='callback',
callback=parse_bool,
help='Whether to allow wheel distributions; Default: allow wheels')
parser.add_option(
'--build', '--no-build',
dest='allow_builds',
default=True,
action='callback',
callback=parse_bool,
help='Whether to allow building of distributions from source; Default: allow builds')
parser.add_option(
'--python',
dest='python',
default=None,
help='The Python interpreter to use to build the pex. Either specify an explicit '
'path to an interpreter, or specify a binary accessible on $PATH. '
'Default: Use current interpreter.')
parser.add_option(
'--platform',
dest='platform',
default=Platform.current(),
help='The platform for which to build the PEX. Default: %%default')
parser.add_option(
'--zip-safe', '--not-zip-safe',
dest='zip_safe',
default=True,
action='callback',
callback=parse_bool,
help='Whether or not the sources in the pex file are zip safe. If they are '
'not zip safe, they will be written to disk prior to execution; '
'Default: zip safe.')
parser.add_option(
'--always-write-cache',
dest='always_write_cache',
default=False,
action='store_true',
help='Always write the internally cached distributions to disk prior to invoking '
'the pex source code. This can use less memory in RAM constrained '
'environments. [Default: %default]')
parser.add_option(
'--ignore-errors',
dest='ignore_errors',
default=False,
action='store_true',
help='Ignore run-time requirement resolution errors when invoking the pex. '
'[Default: %default]')
parser.add_option(
'--inherit-path',
dest='inherit_path',
default=False,
action='store_true',
help='Inherit the contents of sys.path (including site-packages) running the pex. '
'[Default: %default]')
parser.add_option(
'--cache-dir',
dest='cache_dir',
default=os.path.expanduser('~/.pex/build'),
help='The local cache directory to use for speeding up requirement '
'lookups; [Default: %default]')
parser.add_option(
'--cache-ttl',
dest='cache_ttl',
type=int,
default=None,
help='The cache TTL to use for inexact requirement specifications.')
parser.add_option(
'-o', '-p', '--output-file', '--pex-name',
dest='pex_name',
default=None,
help='The name of the generated .pex file: Omiting this will run PEX '
'immediately and not save it to a file.')
parser.add_option(
'-e', '--entry-point',
dest='entry_point',
default=None,
help='The entry point for this pex; Omiting this will enter the python '
'REPL with sources and requirements available for import. Can be '
'either a module or EntryPoint (module:function) format.')
parser.add_option(
'-r', '--requirement',
dest='requirements',
metavar='REQUIREMENT',
default=[],
action='append',
help='requirement to be included; may be specified multiple times.')
parser.add_option(
'--repo',
dest='repos',
metavar='PATH',
default=[],
action='append',
help='Additional repository path (directory or URL) to look for requirements.')
parser.add_option(
'-i', '--index',
dest='indices',
metavar='URL',
default=[],
action='append',
help='Additional cheeseshop indices to use to satisfy requirements.')
parser.add_option(
'-s', '--source-dir',
dest='source_dirs',
metavar='DIR',
default=[],
action='append',
help='Source to be packaged; This <DIR> should be a pip-installable project '
'with a setup.py.')
parser.add_option(
'-v',
dest='verbosity',
default=0,
action='callback',
callback=increment_verbosity,
help='Turn on logging verbosity, may be specified multiple times.')
return parser
def test_get_current_platform(self):
expected_platforms = [Platform.current(), 'linux-x86_64']
self.assertEqual(set(expected_platforms),
set(get_platforms(self.config.getlist('python-setup', 'platforms'))))
def translate(platform): return Platform.current() if platform == 'current' else platform
def test_get_current_platform(): expected_platforms = [Platform.current(), 'linux-x86_64'] assert set(expected_platforms) == set(PythonChroot.get_platforms(['current', 'linux-x86_64']))
