def write_zipped_internal_cache(cls, pex, pex_info):
prefix_length = len(pex_info.internal_cache) + 1
existing_cached_distributions = []
newly_cached_distributions = []
zip_safe_distributions = []
with open_zip(pex) as zf:
# Distribution names are the first element after ".deps/" and before the next "/"
distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0]
for filename in zf.namelist() if filename.startswith(pex_info.internal_cache))))
# Create Distribution objects from these, and possibly write to disk if necessary.
for distribution_name in distribution_names:
internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name])
# First check if this is already cached
dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash(
zf, internal_dist_path)
cached_location = os.path.join(pex_info.install_cache, '%s.%s' % (
distribution_name, dist_digest))
if os.path.exists(cached_location):
dist = DistributionHelper.distribution_from_path(cached_location)
if dist is not None:
existing_cached_distributions.append(dist)
continue
else:
dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path))
if dist is not None:
if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache:
zip_safe_distributions.append(dist)
continue
with TRACER.timed('Caching %s' % dist):
newly_cached_distributions.append(
CacheHelper.cache_distribution(zf, internal_dist_path, cached_location))
return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False,
zip_safe=True, **kwargs):
with make_installer(name=name,
version=version,
installer_impl=installer_impl,
zip_safe=zip_safe,
**kwargs) as installer:
dist_location = installer.bdist()
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td, os.path.basename(dist_location))
with open_zip(dist_location) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def test_access_zipped_assets():
try:
import __builtin__
builtin_path = __builtin__.__name__
except ImportError:
# Python3
import builtins
builtin_path = builtins.__name__
mock_open = mock.mock_open()
with nested(mock.patch('%s.open' % builtin_path, mock_open, create=True),
mock.patch('pex.util.resource_string', autospec=True, spec_set=True),
mock.patch('pex.util.resource_isdir', autospec=True, spec_set=True),
mock.patch('pex.util.resource_listdir', autospec=True, spec_set=True),
mock.patch('pex.util.safe_mkdtemp', autospec=True, spec_set=True)) as (
_, mock_resource_string, mock_resource_isdir, mock_resource_listdir, mock_safe_mkdtemp):
mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir'])
mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']])
mock_resource_isdir.side_effect = iter([False, True, False])
mock_resource_string.return_value = 'testing'
temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil')
assert mock_resource_listdir.call_count == 2
assert mock_open.call_count == 2
file_handle = mock_open.return_value.__enter__.return_value
assert file_handle.write.call_count == 2
assert mock_safe_mkdtemp.mock_calls == [mock.call()]
assert temp_dir == 'tmpJIMMEH'
def test_entry_point_exit_code():
setup_py = dedent("""
from setuptools import setup
setup(
name='my_app',
version='0.0.0',
zip_safe=True,
packages=[''],
entry_points={'console_scripts': ['my_app = my_app:do_something']},
)
""")
error_msg = 'setuptools expects this to exit non-zero'
my_app = dedent("""
def do_something():
return '%s'
""" % error_msg)
with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir:
installer = EggInstaller(project_dir)
dist = DistributionHelper.distribution_from_path(installer.bdist())
so, rc = run_simple_pex_test('', env={'PEX_SCRIPT': 'my_app'}, dists=[dist])
assert so.decode('utf-8').strip() == error_msg
assert rc == 1
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as (
td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_zipsafe():
make_egg = functools.partial(make_bdist, installer_impl=EggInstaller)
make_whl = functools.partial(make_bdist, installer_impl=WheelInstaller)
for zipped in (False, True):
for zip_safe in (False, True):
# Eggs can be zip safe
with make_egg(zipped=zipped, zip_safe=zip_safe) as dist:
assert DistributionHelper.zipsafe(dist) is zip_safe
# Wheels cannot be zip safe
with make_whl(zipped=zipped, zip_safe=zip_safe) as dist:
assert not DistributionHelper.zipsafe(dist)
for zipped in (False, True):
for zip_safe in (False, True):
with make_egg(zipped=zipped, zip_safe=zip_safe) as dist:
assert DistributionHelper.zipsafe(dist) is zip_safe
def test_pex_builder_wheeldep():
"""Repeat the pex_builder test, but this time include an import of
something from a wheel that doesn't come in importable form.
"""
with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1):
pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl"
dist = DistributionHelper.distribution_from_path(pyparsing_path)
pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist])
success_txt = os.path.join(td, 'success.txt')
PEX(td, interpreter=pb.interpreter).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def build(self, package, options):
# cache package locally
if package.remote:
package = Package.from_href(options.get_context().fetch(package, into=self.__cache))
os.utime(package.local_path, None)
# build into distribution
dist = super(CachingResolver, self).build(package, options)
# if distribution is not in cache, copy
target = os.path.join(self.__cache, os.path.basename(dist.location))
if not os.path.exists(target):
shutil.copyfile(dist.location, target + '~')
os.rename(target + '~', target)
os.utime(target, None)
return DistributionHelper.distribution_from_path(target)
def test_pex_script(installer_impl, project_name):
with make_installer(name=project_name, installer_impl=installer_impl) as installer:
bdist = DistributionHelper.distribution_from_path(installer.bdist())
env_copy = os.environ.copy()
env_copy['PEX_SCRIPT'] = 'hello_world'
so, rc = run_simple_pex_test('', env=env_copy)
assert rc == 1, so.decode('utf-8')
assert b'Could not find' in so
so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
assert rc == 0, so.decode('utf-8')
assert b'hello world' in so
env_copy['PEX_SCRIPT'] = 'shell_script'
so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist])
assert rc == 1, so.decode('utf-8')
assert b'Unable to parse' in so
def add_dist_location(self, dist, name=None):
"""Add a distribution by its location on disk.
:param dist: The path to the distribution to add.
:keyword name: (optional) The name of the distribution, should the dist directory alone be
ambiguous. Packages contained within site-packages directories may require specifying
``name``.
:raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution.
PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution
supported by setuptools/pkg_resources.
"""
self._ensure_unfrozen('Adding a distribution')
bdist = DistributionHelper.distribution_from_path(dist)
if bdist is None:
raise self.InvalidDistribution('Could not find distribution at %s' % dist)
self.add_distribution(bdist)
self.add_requirement(bdist.as_requirement())
def test_access_zipped_assets(
mock_resource_string,
mock_resource_isdir,
mock_resource_listdir,
mock_safe_mkdir,
mock_safe_mkdtemp):
mock_open = mock.mock_open()
mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir'])
mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']])
mock_resource_isdir.side_effect = iter([False, True, False])
mock_resource_string.return_value = 'testing'
with mock.patch('%s.open' % python_builtins.__name__, mock_open, create=True):
temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil')
assert mock_resource_listdir.call_count == 2
assert mock_open.call_count == 2
file_handle = mock_open.return_value.__enter__.return_value
assert file_handle.write.call_count == 2
assert mock_safe_mkdtemp.mock_calls == [mock.call()]
assert temp_dir == 'tmpJIMMEH'
assert mock_safe_mkdir.mock_calls == [mock.call(os.path.join('tmpJIMMEH', 'directory'))]
def make_bdist(
name="my_project", # type: str
version="0.0.0", # type: str
zip_safe=True, # type: bool
interpreter=None, # type: Optional[PythonInterpreter]
**kwargs # type: Any
):
# type: (...) -> Iterator[Distribution]
with built_wheel(name=name,
version=version,
zip_safe=zip_safe,
interpreter=interpreter,
**kwargs) as dist_location:
install_dir = os.path.join(safe_mkdtemp(),
os.path.basename(dist_location))
get_pip().spawn_install_wheel(
wheel=dist_location,
install_dir=install_dir,
target=DistributionTarget(interpreter=interpreter),
).wait()
dist = DistributionHelper.distribution_from_path(install_dir)
assert dist is not None
yield dist
def test_distributionhelper_egg_assert():
d = DistributionHelper.distribution_from_path(
'./tests/example_packages/setuptools-18.0.1-py2.7.egg',
'setuptools'
)
assert len(d.resource_listdir('/')) > 3
def _resolve(self, working_set, reqs):
environment = self._target_interpreter_env.copy()
environment["extra"] = list(set(itertools.chain(*(req.extras for req in reqs))))
reqs_by_key = OrderedDict()
for req in reqs:
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
"Skipping activation of `%s` due to environment marker de-selection" % req
)
continue
reqs_by_key.setdefault(req.key, []).append(req)
unresolved_reqs = OrderedDict()
resolveds = OrderedSet()
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for key, reqs in reqs_by_key.items():
with TRACER.timed("Resolving {} from {}".format(key, reqs), V=2):
# N.B.: We resolve the bare requirement with no version specifiers since the resolve process
# used to build this pex already did so. There may be multiple distributions satisfying any
# particular key (e.g.: a Python 2 specific version and a Python 3 specific version for a
# multi-python PEX) and we want the working set to pick the most appropriate one.
req = Requirement.parse(key)
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log("Failed to resolve a requirement: %s" % e)
requirers = unresolved_reqs.setdefault(e.req, OrderedSet())
if e.requirers:
for requirer in e.requirers:
requirers.update(reqs_by_key[requirer])
if unresolved_reqs:
TRACER.log("Unresolved requirements:")
for req in unresolved_reqs:
TRACER.log(" - %s" % req)
TRACER.log("Distributions contained within this pex:")
distributions_by_key = defaultdict(list)
if not self._pex_info.distributions:
TRACER.log(" None")
else:
for dist_name, dist_digest in self._pex_info.distributions.items():
TRACER.log(" - %s" % dist_name)
distribution = DistributionHelper.distribution_from_path(
path=os.path.join(self._pex_info.install_cache, dist_digest, dist_name)
)
distributions_by_key[distribution.as_requirement().key].append(distribution)
if not self._pex_info.ignore_errors:
items = []
for index, (requirement, requirers) in enumerate(unresolved_reqs.items()):
rendered_requirers = ""
if requirers:
rendered_requirers = ("\n Required by:" "\n {requirers}").format(
requirers="\n ".join(map(str, requirers))
)
items.append(
"{index: 2d}: {requirement}"
"{rendered_requirers}"
"\n But this pex only contains:"
"\n {distributions}".format(
index=index + 1,
requirement=requirement,
rendered_requirers=rendered_requirers,
distributions="\n ".join(
os.path.basename(d.location)
for d in distributions_by_key[requirement.key]
),
)
)
die(
"Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}".format(
platform=self._interpreter.platform, items="\n".join(items)
)
)
return resolveds
def add_dist_location(self, location):
self._builder.add_dist_location(location)
dist = DistributionHelper.distribution_from_path(location)
self._register_distribution(dist)
def _resolve(self, working_set, reqs):
reqs_by_key = OrderedDict((req.key, req) for req in reqs)
unresolved_reqs = OrderedDict()
resolveds = OrderedSet()
environment = self._target_interpreter_env.copy()
environment['extra'] = list(
set(itertools.chain(*(req.extras for req in reqs))))
# Resolve them one at a time so that we can figure out which ones we need to elide should
# there be an interpreter incompatibility.
for req in reqs_by_key.values():
if req.marker and not req.marker.evaluate(environment=environment):
TRACER.log(
'Skipping activation of `%s` due to environment marker de-selection'
% req)
continue
with TRACER.timed('Resolving %s' % req, V=2):
try:
resolveds.update(working_set.resolve([req], env=self))
except DistributionNotFound as e:
TRACER.log('Failed to resolve a requirement: %s' % e)
requirers = unresolved_reqs.setdefault(e.req, OrderedSet())
if e.requirers:
requirers.update(reqs_by_key[requirer]
for requirer in e.requirers)
if unresolved_reqs:
TRACER.log('Unresolved requirements:')
for req in unresolved_reqs:
TRACER.log(' - %s' % req)
TRACER.log('Distributions contained within this pex:')
distributions_by_key = defaultdict(list)
if not self._pex_info.distributions:
TRACER.log(' None')
else:
for dist_name, dist_digest in self._pex_info.distributions.items(
):
TRACER.log(' - %s' % dist_name)
distribution = DistributionHelper.distribution_from_path(
path=os.path.join(self._pex_info.install_cache,
dist_digest, dist_name))
distributions_by_key[
distribution.as_requirement().key].append(distribution)
if not self._pex_info.ignore_errors:
items = []
for index, (requirement,
requirers) in enumerate(unresolved_reqs.items()):
rendered_requirers = ''
if requirers:
rendered_requirers = (
'\n Required by:'
'\n {requirers}').format(
requirers='\n '.join(map(str, requirers)))
items.append('{index: 2d}: {requirement}'
'{rendered_requirers}'
'\n But this pex only contains:'
'\n {distributions}'.format(
index=index + 1,
requirement=requirement,
rendered_requirers=rendered_requirers,
distributions='\n '.join(
os.path.basename(d.location)
for d in distributions_by_key[
requirement.key])))
die('Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}'
.format(platform=Platform.of_interpreter(
self._interpreter),
items='\n'.join(items)))
return resolveds
def test_distributionhelper_egg_assert():
d = DistributionHelper.distribution_from_path(
'./tests/example_packages/setuptools-18.0.1-py2.7.egg',
'setuptools'
)
assert len(d.resource_listdir('/')) > 3
def add_dist_location(self, location): self._builder.add_dist_location(location) dist = DistributionHelper.distribution_from_path(location) self._register_distribution(dist)
