def make_distribution(name='my_project', zipped=False, zip_safe=True):
interp = {'project_name': name}
if zip_safe:
interp['content'] = dedent('''
def do_something():
print('hello world!')
''')
else:
interp['content'] = dedent('''
if __file__ == 'derp.py':
print('i am an idiot')
''')
with temporary_content(PROJECT_CONTENT, interp=interp) as td:
installer = Installer(td)
distribution = installer.distribution()
distiller = Distiller(distribution, debug=True)
dist_location = distiller.distill(into=safe_mkdtemp())
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td,
os.path.basename(dist_location))
with contextlib.closing(zipfile.ZipFile(dist_location)) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def make_distribution(name='my_project', zipped=False, zip_safe=True):
interp = {'project_name': name}
if zip_safe:
interp['content'] = dedent('''
def do_something():
print('hello world!')
''')
else:
interp['content'] = dedent('''
if __file__ == 'derp.py':
print('i am an idiot')
''')
with temporary_content(PROJECT_CONTENT, interp=interp) as td:
installer = Installer(td)
distribution = installer.distribution()
distiller = Distiller(distribution, debug=True)
dist_location = distiller.distill(into=safe_mkdtemp())
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td, os.path.basename(dist_location))
with contextlib.closing(zipfile.ZipFile(dist_location)) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def _prepare_bootstrap(self):
"""
Write enough of distribute and pip into the .pex .bootstrap directory so that
we can be fully self-contained.
"""
bare_env = pkg_resources.Environment()
pip_req = pkg_resources.Requirement.parse('pip>=1.1')
distribute_req = pkg_resources.Requirement.parse('distribute>=0.6.24')
pip_dist = distribute_dist = None
for dist in DistributionHelper.all_distributions(sys.path):
if dist in pip_req and bare_env.can_add(dist):
pip_dist = dist
if dist in distribute_req and bare_env.can_add(dist):
distribute_dist = dist
if pip_dist and distribute_dist:
break
if not pip_dist:
raise DistributionNotFound('Could not find pip!')
if not distribute_dist:
raise DistributionNotFound('Could not find distribute!')
PEX.debug('Writing .bootstrap library.')
for fn, content in DistributionHelper.walk_data(pip_dist):
if fn.startswith('pip/'):
# PEX.debug('BOOTSTRAP: Writing %s' % fn)
self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource')
for fn, content in DistributionHelper.walk_data(distribute_dist):
if fn.startswith('pkg_resources.py') or fn.startswith('setuptools'):
# PEX.debug('BOOTSTRAP: Writing %s' % fn)
self._chroot.write(content, os.path.join(self.BOOTSTRAP_DIR, fn), 'resource')
libraries = (
'twitter.common.dirutil',
'twitter.common.collections',
'twitter.common.contextutil',
'twitter.common.lang',
'twitter.common.python'
)
for name in libraries:
dirname = name.replace('.', '/')
provider = pkg_resources.get_provider(name)
if not isinstance(provider, pkg_resources.DefaultProvider):
mod = __import__(name, fromlist=['wutttt'])
provider = pkg_resources.ZipProvider(mod)
for fn in provider.resource_listdir(''):
if fn.endswith('.py'):
# PEX.debug('BOOTSTRAP: Writing %s' % os.path.join(dirname, fn))
self._chroot.write(provider.get_resource_string(name, fn),
os.path.join(self.BOOTSTRAP_DIR, dirname, fn), 'resource')
for initdir in ('twitter', 'twitter/common'):
self._chroot.write(
b"__import__('pkg_resources').declare_namespace(__name__)",
os.path.join(self.BOOTSTRAP_DIR, initdir, '__init__.py'),
'resource')
def make_distribution(name='my_project', zipped=False, zip_safe=True):
interp = {'project_name': name, 'zip_safe': zip_safe}
with temporary_content(PROJECT_CONTENT, interp=interp) as td:
installer = EggInstaller(td)
dist_location = installer.bdist()
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td, os.path.basename(dist_location))
with contextlib.closing(zipfile.ZipFile(dist_location)) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def make_distribution(name='my_project', zipped=False, zip_safe=True):
interp = {'project_name': name, 'zip_safe': zip_safe}
with temporary_content(PROJECT_CONTENT, interp=interp) as td:
installer = EggInstaller(td)
dist_location = installer.bdist()
if zipped:
yield DistributionHelper.distribution_from_path(dist_location)
else:
with temporary_dir() as td:
extract_path = os.path.join(td,
os.path.basename(dist_location))
with contextlib.closing(zipfile.ZipFile(dist_location)) as zf:
zf.extractall(extract_path)
yield DistributionHelper.distribution_from_path(extract_path)
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_distribution("p1", zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, "success.txt")
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == "success"
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(), make_distribution("p1", zipped=True)) as (td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, "r")) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, "success.txt")
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == "success"
def activate(self):
from pkg_resources import Requirement, WorkingSet, DistributionNotFound
if self._activated:
return
if self._pex_info.inherit_path:
self._ws = WorkingSet(sys.path)
# TODO(wickman) Implement dynamic fetchers if pex_info requirements specify dynamic=True
# or a non-empty repository.
all_reqs = [Requirement.parse(req) for req, _, _ in self._pex_info.requirements]
for req in all_reqs:
with PEX.timed('Resolved %s' % str(req)):
try:
resolved = self._ws.resolve([req], env=self)
except DistributionNotFound as e:
self._log('Failed to resolve %s: %s' % (req, e))
if not self._pex_info.ignore_errors:
raise
continue
for dist in resolved:
with PEX.timed(' Activated %s' % dist):
if self._really_zipsafe(dist):
self._ws.add(dist)
dist.activate()
else:
with PEX.timed(' Locally caching %s' % dist):
new_dist = DistributionHelper.locally_cache(dist, self._pex_info.install_cache)
new_dist.activate()
self._activated = True
def test_pex_builder():
# test w/ and w/o zipfile dists
with nested(temporary_dir(), make_distribution('p1',
zipped=True)) as (td, p1):
write_pex(td, exe_main, dists=[p1])
success_txt = os.path.join(td, 'success.txt')
PEX(td).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
# test w/ and w/o zipfile dists
with nested(temporary_dir(), temporary_dir(),
make_distribution('p1', zipped=True)) as (td1, td2, p1):
target_egg_dir = os.path.join(td2, os.path.basename(p1.location))
safe_mkdir(target_egg_dir)
with closing(zipfile.ZipFile(p1.location, 'r')) as zf:
zf.extractall(target_egg_dir)
p1 = DistributionHelper.distribution_from_path(target_egg_dir)
write_pex(td1, exe_main, dists=[p1])
success_txt = os.path.join(td1, 'success.txt')
PEX(td1).run(args=[success_txt])
assert os.path.exists(success_txt)
with open(success_txt) as fp:
assert fp.read() == 'success'
def test_zipsafe():
make_egg = functools.partial(make_distribution, installer_impl=EggInstaller)
for zipped in (False, True):
for zip_safe in (False, True):
with make_egg(zipped=zipped, zip_safe=zip_safe) as dist:
assert DistributionHelper.zipsafe(dist) is zip_safe
def test_zipsafe():
make_egg = functools.partial(make_distribution, installer_impl=EggInstaller)
make_whl = functools.partial(make_distribution, installer_impl=WheelInstaller)
for zipped in (False, True):
for zip_safe in (False, True):
# Eggs can be zip safe
with make_egg(zipped=zipped, zip_safe=zip_safe) as dist:
assert DistributionHelper.zipsafe(dist) is zip_safe
# Wheels cannot be zip safe
with make_whl(zipped=zipped, zip_safe=zip_safe) as dist:
assert not DistributionHelper.zipsafe(dist)
for zipped in (False, True):
for zip_safe in (False, True):
with make_egg(zipped=zipped, zip_safe=zip_safe) as dist:
assert DistributionHelper.zipsafe(dist) is zip_safe
def test_zipsafe():
for zipped in (False, True):
with make_distribution(zipped=zipped) as dist:
assert DistributionHelper.zipsafe(dist)
def test_zipsafe():
for zipped in (False, True):
with make_distribution(zipped=zipped) as dist:
assert DistributionHelper.zipsafe(dist)
