def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False,
zip_safe=True, **kwargs):
with make_installer(name=name,
version=version,
installer_impl=installer_impl,
zip_safe=zip_safe,
**kwargs) as installer:
dist_location = installer.bdist()
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td, os.path.basename(dist_location))
with open_zip(dist_location) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def test_entry_point_exit_code():
setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=[''],
entry_points={'console_scripts': ['my_app = my_app:do_something']},
)
""")
error_msg = 'setuptools expects this to exit non-zero'
my_app = dedent("""
def do_something():
return '%s'
""" % error_msg)
with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir:
installer = EggInstaller(project_dir)
dist = DistributionHelper.distribution_from_path(installer.bdist())
so, rc = run_simple_pex_test('', env={'PEX_SCRIPT': 'my_app'}, dists=[dist])
assert so.decode('utf-8').strip() == error_msg
assert rc == 1
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of
something from a wheel that doesn't come in importable form.
"""
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
dist = DistributionHelper.distribution_from_path(pyparsing_path)
pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def build(self, package, options):
# cache package locally
if package.remote:
package = Package.from_href(options.get_context().fetch(package, into=self.__cache))
os.utime(package.local_path, None)
# build into distribution
dist = super(CachingResolver, self).build(package, options)
# if distribution is not in cache, copy
target = os.path.join(self.__cache, os.path.basename(dist.location))
if not os.path.exists(target):
shutil.copyfile(dist.location, target + '~')
os.rename(target + '~', target)
os.utime(target, None)
return DistributionHelper.distribution_from_path(target)
def translate(self, package, into=None):
"""From a SourcePackage, translate to a binary distribution."""
if not isinstance(package, SourcePackage):
return None
if not package.local:
raise ValueError(
'SourceTranslator cannot translate remote packages.')
installer = None
version = self._interpreter.version
unpack_path = Archiver.unpack(package.local_path)
into = into or safe_mkdtemp()
try:
if self._use_2to3 and version >= (3, ):
with TRACER.timed('Translating 2->3 %s' % package.name):
self.run_2to3(unpack_path)
installer = self._installer_impl(unpack_path,
interpreter=self._interpreter)
with TRACER.timed('Packaging %s' % package.name):
try:
dist_path = installer.bdist()
except self._installer_impl.InstallFailure as e:
TRACER.log('Failed to install package at %s: %s' %
(unpack_path, e))
return None
target_path = os.path.join(into, os.path.basename(dist_path))
safe_copy(dist_path, target_path)
target_package = Package.from_href(target_path)
if not target_package:
TRACER.log('Target path %s does not look like a Package.' %
target_path)
return None
if not target_package.compatible(self._supported_tags):
TRACER.log('Target package %s is not compatible with %s' %
(target_package, self._supported_tags))
return None
return DistributionHelper.distribution_from_path(target_path)
except Exception as e:
TRACER.log('Failed to translate %s' % package)
TRACER.log(traceback.format_exc())
finally:
if installer:
installer.cleanup()
if unpack_path:
safe_rmtree(unpack_path)
def _load_internal_cache(self):
"""Possibly cache out the internal cache."""
internal_cache = os.path.join(self._pex, self._pex_info.internal_cache)
with TRACER.timed("Searching dependency cache: %s" % internal_cache,
V=2):
if len(self._pex_info.distributions) == 0:
# We have no .deps to load.
return
if os.path.isdir(self._pex):
for distribution_name in self._pex_info.distributions:
yield DistributionHelper.distribution_from_path(
os.path.join(internal_cache, distribution_name))
else:
with open_zip(self._pex) as zf:
for dist in self._write_zipped_internal_cache(zf):
yield dist
def test_get_script_from_distributions(tmpdir):
# type: (Any) -> None
whl_path = "./tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl"
install_dir = os.path.join(str(tmpdir), os.path.basename(whl_path))
get_pip().spawn_install_wheel(wheel=whl_path,
install_dir=install_dir).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert dist is not None
assert "aws-cfn-bootstrap" == dist.project_name
dist_script = get_script_from_distributions("cfn-signal", [dist])
assert dist_script.dist is dist
assert os.path.join(install_dir, "bin/cfn-signal") == dist_script.path
assert dist_script.read_contents().startswith(
"#!"), "Expected a `scripts`-style script w/shebang."
assert None is get_script_from_distributions("non_existent_script", [dist])
def add_dist_location(self, dist, name=None):
"""Add a distribution by its location on disk.
:param dist: The path to the distribution to add.
:keyword name: (optional) The name of the distribution, should the dist directory alone be
ambiguous. Packages contained within site-packages directories may require specifying
``name``.
:raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution.
PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution
supported by setuptools/pkg_resources.
"""
self._ensure_unfrozen('Adding a distribution')
bdist = DistributionHelper.distribution_from_path(dist)
if bdist is None:
raise self.InvalidDistribution('Could not find distribution at %s' % dist)
self.add_distribution(bdist)
self.add_requirement(bdist.as_requirement())
def test_pex_script(installer_impl, project_name):
with make_installer(name=project_name, installer_impl=installer_impl) as installer:
bdist = DistributionHelper.distribution_from_path(installer.bdist())
env_copy = os.environ.copy()
env_copy['PEX_SCRIPT'] = 'hello_world'
so, rc = run_simple_pex_test('', env=env_copy)
assert rc == 1, so.decode('utf-8')
assert b'Could not find' in so
so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
assert rc == 0, so.decode('utf-8')
assert b'hello world' in so
env_copy['PEX_SCRIPT'] = 'shell_script'
so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
assert rc == 1, so.decode('utf-8')
assert b'Unable to parse' in so
def make_bdist(name='my_project',
version='0.0.0',
zip_safe=True,
interpreter=None,
**kwargs):
with built_wheel(name=name,
version=version,
zip_safe=zip_safe,
interpreter=interpreter,
**kwargs) as dist_location:
install_dir = os.path.join(safe_mkdtemp(),
os.path.basename(dist_location))
get_pip().spawn_install_wheel(
wheel=dist_location,
install_dir=install_dir,
target=DistributionTarget.for_interpreter(interpreter)).wait()
yield DistributionHelper.distribution_from_path(install_dir)
def add_dist_location(self, dist, name=None):
"""Add a distribution by its location on disk.
:param dist: The path to the distribution to add.
:keyword name: (optional) The name of the distribution, should the dist directory alone be
ambiguous. Packages contained within site-packages directories may require specifying
``name``.
:raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution.
PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution
supported by setuptools/pkg_resources.
"""
self._ensure_unfrozen('Adding a distribution')
bdist = DistributionHelper.distribution_from_path(dist)
if bdist is None:
raise self.InvalidDistribution('Could not find distribution at %s' % dist)
self.add_distribution(bdist)
self.add_requirement(bdist.as_requirement())
def test_pex_script(installer_impl, project_name, zip_safe):
kw = dict(name=project_name, installer_impl=installer_impl, zip_safe=zip_safe)
with make_installer(**kw) as installer:
bdist = DistributionHelper.distribution_from_path(installer.bdist())
env_copy = os.environ.copy()
env_copy['PEX_SCRIPT'] = 'hello_world'
so, rc = run_simple_pex_test('', env=env_copy)
assert rc == 1, so.decode('utf-8')
assert b'Could not find script hello_world' in so
so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
assert rc == 0, so.decode('utf-8')
assert b'hello world' in so
env_copy['PEX_SCRIPT'] = 'shell_script'
so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
assert rc == 1, so.decode('utf-8')
assert b'Unable to parse' in so
def make_bdist(
name="my_project", # type: str
version="0.0.0", # type: str
zip_safe=True, # type: bool
interpreter=None, # type: Optional[PythonInterpreter]
**kwargs # type: Any
):
# type: (...) -> Iterator[Distribution]
with built_wheel(
name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs
) as dist_location:
install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location))
get_pip().spawn_install_wheel(
wheel=dist_location,
install_dir=install_dir,
target=DistributionTarget.for_interpreter(interpreter),
).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert dist is not None
yield dist
def _resolve(self, working_set, reqs):
environment = self._target_interpreter_env.copy()
environment["extra"] = list(set(itertools.chain(*(req.extras for req in reqs))))
reqs_by_key = OrderedDict()
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
"Skipping activation of `%s` due to environment marker de-selection" % req
)
continue
reqs_by_key.setdefault(req.key, []).append(req)
unresolved_reqs = OrderedDict()
resolveds = OrderedSet()
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for key, reqs in reqs_by_key.items():
with TRACER.timed("Resolving {} from {}".format(key, reqs), V=2):
# N.B.: We resolve the bare requirement with no version specifiers since the resolve process
# used to build this pex already did so. There may be multiple distributions satisfying any
# particular key (e.g.: a Python 2 specific version and a Python 3 specific version for a
# multi-python PEX) and we want the working set to pick the most appropriate one.
req = Requirement.parse(key)
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log("Failed to resolve a requirement: %s" % e)
requirers = unresolved_reqs.setdefault(e.req, OrderedSet())
if e.requirers:
for requirer in e.requirers:
requirers.update(reqs_by_key[requirer])
if unresolved_reqs:
TRACER.log("Unresolved requirements:")
for req in unresolved_reqs:
TRACER.log(" - %s" % req)
TRACER.log("Distributions contained within this pex:")
distributions_by_key = defaultdict(list)
if not self._pex_info.distributions:
TRACER.log(" None")
else:
for dist_name, dist_digest in self._pex_info.distributions.items():
TRACER.log(" - %s" % dist_name)
distribution = DistributionHelper.distribution_from_path(
path=os.path.join(self._pex_info.install_cache, dist_digest, dist_name)
)
distributions_by_key[distribution.as_requirement().key].append(distribution)
if not self._pex_info.ignore_errors:
items = []
for index, (requirement, requirers) in enumerate(unresolved_reqs.items()):
rendered_requirers = ""
if requirers:
rendered_requirers = ("\n Required by:" "\n {requirers}").format(
requirers="\n ".join(map(str, requirers))
)
items.append(
"{index: 2d}: {requirement}"
"{rendered_requirers}"
"\n But this pex only contains:"
"\n {distributions}".format(
index=index + 1,
requirement=requirement,
rendered_requirers=rendered_requirers,
distributions="\n ".join(
os.path.basename(d.location)
for d in distributions_by_key[requirement.key]
),
)
)
die(
"Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}".format(
platform=self._interpreter.platform, items="\n".join(items)
)
)
return resolveds
def add_dist_location(self, location):
self._builder.add_dist_location(location)
dist = DistributionHelper.distribution_from_path(location)
self._register_distribution(dist)
def test_distributionhelper_egg_assert():
d = DistributionHelper.distribution_from_path(
'./tests/example_packages/setuptools-18.0.1-py2.7.egg',
'setuptools'
)
assert len(d.resource_listdir('/')) > 3
def add_dist_location(self, location): self._builder.add_dist_location(location) dist = DistributionHelper.distribution_from_path(location) self._register_distribution(dist)
def test_distributionhelper_egg_assert():
d = DistributionHelper.distribution_from_path(
'./tests/example_packages/setuptools-18.0.1-py2.7.egg',
'setuptools'
)
assert len(d.resource_listdir('/')) > 3
def _resolve(self, working_set, reqs):
reqs_by_key = OrderedDict((req.key, req) for req in reqs)
unresolved_reqs = OrderedDict()
resolveds = OrderedSet()
environment = self._target_interpreter_env.copy()
environment['extra'] = list(
set(itertools.chain(*(req.extras for req in reqs))))
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs_by_key.values():
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
'Skipping activation of `%s` due to environment marker de-selection'
% req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
requirers = unresolved_reqs.setdefault(e.req, OrderedSet())
if e.requirers:
requirers.update(reqs_by_key[requirer]
for requirer in e.requirers)
if unresolved_reqs:
TRACER.log('Unresolved requirements:')
for req in unresolved_reqs:
TRACER.log(' - %s' % req)
TRACER.log('Distributions contained within this pex:')
distributions_by_key = defaultdict(list)
if not self._pex_info.distributions:
TRACER.log(' None')
else:
for dist_name, dist_digest in self._pex_info.distributions.items(
):
TRACER.log(' - %s' % dist_name)
distribution = DistributionHelper.distribution_from_path(
path=os.path.join(self._pex_info.install_cache,
dist_digest, dist_name))
distributions_by_key[
distribution.as_requirement().key].append(distribution)
if not self._pex_info.ignore_errors:
items = []
for index, (requirement,
requirers) in enumerate(unresolved_reqs.items()):
rendered_requirers = ''
if requirers:
rendered_requirers = (
'\n Required by:'
'\n {requirers}').format(
requirers='\n '.join(map(str, requirers)))
items.append('{index: 2d}: {requirement}'
'{rendered_requirers}'
'\n But this pex only contains:'
'\n {distributions}'.format(
index=index + 1,
requirement=requirement,
rendered_requirers=rendered_requirers,
distributions='\n '.join(
os.path.basename(d.location)
for d in distributions_by_key[
requirement.key])))
die('Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}'
.format(platform=Platform.of_interpreter(
self._interpreter),
items='\n'.join(items)))
return resolveds
