def write_zipped_internal_cache(cls, pex, pex_info): prefix_length = len(pex_info.internal_cache) + 1 existing_cached_distributions = [] newly_cached_distributions = [] zip_safe_distributions = [] with open_zip(pex) as zf: # Distribution names are the first element after ".deps/" and before the next "/" distribution_names = set(filter(None, (filename[prefix_length:].split('/')[0] for filename in zf.namelist() if filename.startswith(pex_info.internal_cache)))) # Create Distribution objects from these, and possibly write to disk if necessary. for distribution_name in distribution_names: internal_dist_path = '/'.join([pex_info.internal_cache, distribution_name]) # First check if this is already cached dist_digest = pex_info.distributions.get(distribution_name) or CacheHelper.zip_hash( zf, internal_dist_path) cached_location = os.path.join(pex_info.install_cache, '%s.%s' % ( distribution_name, dist_digest)) if os.path.exists(cached_location): dist = DistributionHelper.distribution_from_path(cached_location) if dist is not None: existing_cached_distributions.append(dist) continue else: dist = DistributionHelper.distribution_from_path(os.path.join(pex, internal_dist_path)) if dist is not None: if DistributionHelper.zipsafe(dist) and not pex_info.always_write_cache: zip_safe_distributions.append(dist) continue with TRACER.timed('Caching %s' % dist): newly_cached_distributions.append( CacheHelper.cache_distribution(zf, internal_dist_path, cached_location)) return existing_cached_distributions, newly_cached_distributions, zip_safe_distributions
def make_bdist(name='my_project', version='0.0.0', installer_impl=EggInstaller, zipped=False, zip_safe=True, **kwargs): with make_installer(name=name, version=version, installer_impl=installer_impl, zip_safe=zip_safe, **kwargs) as installer: dist_location = installer.bdist() if zipped: yield DistributionHelper.distribution_from_path(dist_location) else: with temporary_dir() as td: extract_path = os.path.join(td, os.path.basename(dist_location)) with open_zip(dist_location) as zf: zf.extractall(extract_path) yield DistributionHelper.distribution_from_path(extract_path)
def test_access_zipped_assets(): try: import __builtin__ builtin_path = __builtin__.__name__ except ImportError: # Python3 import builtins builtin_path = builtins.__name__ mock_open = mock.mock_open() with nested(mock.patch('%s.open' % builtin_path, mock_open, create=True), mock.patch('pex.util.resource_string', autospec=True, spec_set=True), mock.patch('pex.util.resource_isdir', autospec=True, spec_set=True), mock.patch('pex.util.resource_listdir', autospec=True, spec_set=True), mock.patch('pex.util.safe_mkdtemp', autospec=True, spec_set=True)) as ( _, mock_resource_string, mock_resource_isdir, mock_resource_listdir, mock_safe_mkdtemp): mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir']) mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']]) mock_resource_isdir.side_effect = iter([False, True, False]) mock_resource_string.return_value = 'testing' temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil') assert mock_resource_listdir.call_count == 2 assert mock_open.call_count == 2 file_handle = mock_open.return_value.__enter__.return_value assert file_handle.write.call_count == 2 assert mock_safe_mkdtemp.mock_calls == [mock.call()] assert temp_dir == 'tmpJIMMEH'
def test_entry_point_exit_code(): setup_py = dedent(""" from setuptools import setup setup( name='my_app', version='0.0.0', zip_safe=True, packages=[''], entry_points={'console_scripts': ['my_app = my_app:do_something']}, ) """) error_msg = 'setuptools expects this to exit non-zero' my_app = dedent(""" def do_something(): return '%s' """ % error_msg) with temporary_content({'setup.py': setup_py, 'my_app.py': my_app}) as project_dir: installer = EggInstaller(project_dir) dist = DistributionHelper.distribution_from_path(installer.bdist()) so, rc = run_simple_pex_test('', env={'PEX_SCRIPT': 'my_app'}, dists=[dist]) assert so.decode('utf-8').strip() == error_msg assert rc == 1
def test_pex_builder(): # test w/ and w/o zipfile dists with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1): write_pex(td, exe_main, dists=[p1]) success_txt = os.path.join(td, 'success.txt') PEX(td).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success' # test w/ and w/o zipfile dists with nested(temporary_dir(), temporary_dir(), make_bdist('p1', zipped=True)) as ( td1, td2, p1): target_egg_dir = os.path.join(td2, os.path.basename(p1.location)) safe_mkdir(target_egg_dir) with closing(zipfile.ZipFile(p1.location, 'r')) as zf: zf.extractall(target_egg_dir) p1 = DistributionHelper.distribution_from_path(target_egg_dir) write_pex(td1, exe_main, dists=[p1]) success_txt = os.path.join(td1, 'success.txt') PEX(td1).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success'
def test_zipsafe(): make_egg = functools.partial(make_bdist, installer_impl=EggInstaller) make_whl = functools.partial(make_bdist, installer_impl=WheelInstaller) for zipped in (False, True): for zip_safe in (False, True): # Eggs can be zip safe with make_egg(zipped=zipped, zip_safe=zip_safe) as dist: assert DistributionHelper.zipsafe(dist) is zip_safe # Wheels cannot be zip safe with make_whl(zipped=zipped, zip_safe=zip_safe) as dist: assert not DistributionHelper.zipsafe(dist) for zipped in (False, True): for zip_safe in (False, True): with make_egg(zipped=zipped, zip_safe=zip_safe) as dist: assert DistributionHelper.zipsafe(dist) is zip_safe
def test_pex_builder_wheeldep(): """Repeat the pex_builder test, but this time include an import of something from a wheel that doesn't come in importable form. """ with nested(temporary_dir(), make_bdist('p1', zipped=True)) as (td, p1): pyparsing_path = "./tests/example_packages/pyparsing-2.1.10-py2.py3-none-any.whl" dist = DistributionHelper.distribution_from_path(pyparsing_path) pb = write_pex(td, wheeldeps_exe_main, dists=[p1, dist]) success_txt = os.path.join(td, 'success.txt') PEX(td, interpreter=pb.interpreter).run(args=[success_txt]) assert os.path.exists(success_txt) with open(success_txt) as fp: assert fp.read() == 'success'
def build(self, package, options): # cache package locally if package.remote: package = Package.from_href(options.get_context().fetch(package, into=self.__cache)) os.utime(package.local_path, None) # build into distribution dist = super(CachingResolver, self).build(package, options) # if distribution is not in cache, copy target = os.path.join(self.__cache, os.path.basename(dist.location)) if not os.path.exists(target): shutil.copyfile(dist.location, target + '~') os.rename(target + '~', target) os.utime(target, None) return DistributionHelper.distribution_from_path(target)
def test_pex_script(installer_impl, project_name): with make_installer(name=project_name, installer_impl=installer_impl) as installer: bdist = DistributionHelper.distribution_from_path(installer.bdist()) env_copy = os.environ.copy() env_copy['PEX_SCRIPT'] = 'hello_world' so, rc = run_simple_pex_test('', env=env_copy) assert rc == 1, so.decode('utf-8') assert b'Could not find' in so so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist]) assert rc == 0, so.decode('utf-8') assert b'hello world' in so env_copy['PEX_SCRIPT'] = 'shell_script' so, rc = run_simple_pex_test('', env=env_copy, dists=[bdist]) assert rc == 1, so.decode('utf-8') assert b'Unable to parse' in so
def add_dist_location(self, dist, name=None): """Add a distribution by its location on disk. :param dist: The path to the distribution to add. :keyword name: (optional) The name of the distribution, should the dist directory alone be ambiguous. Packages contained within site-packages directories may require specifying ``name``. :raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution. PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution supported by setuptools/pkg_resources. """ self._ensure_unfrozen('Adding a distribution') bdist = DistributionHelper.distribution_from_path(dist) if bdist is None: raise self.InvalidDistribution('Could not find distribution at %s' % dist) self.add_distribution(bdist) self.add_requirement(bdist.as_requirement())
def test_access_zipped_assets( mock_resource_string, mock_resource_isdir, mock_resource_listdir, mock_safe_mkdir, mock_safe_mkdtemp): mock_open = mock.mock_open() mock_safe_mkdtemp.side_effect = iter(['tmpJIMMEH', 'faketmpDir']) mock_resource_listdir.side_effect = iter([['./__init__.py', './directory/'], ['file.py']]) mock_resource_isdir.side_effect = iter([False, True, False]) mock_resource_string.return_value = 'testing' with mock.patch('%s.open' % python_builtins.__name__, mock_open, create=True): temp_dir = DistributionHelper.access_zipped_assets('twitter.common', 'dirutil') assert mock_resource_listdir.call_count == 2 assert mock_open.call_count == 2 file_handle = mock_open.return_value.__enter__.return_value assert file_handle.write.call_count == 2 assert mock_safe_mkdtemp.mock_calls == [mock.call()] assert temp_dir == 'tmpJIMMEH' assert mock_safe_mkdir.mock_calls == [mock.call(os.path.join('tmpJIMMEH', 'directory'))]
def make_bdist( name="my_project", # type: str version="0.0.0", # type: str zip_safe=True, # type: bool interpreter=None, # type: Optional[PythonInterpreter] **kwargs # type: Any ): # type: (...) -> Iterator[Distribution] with built_wheel(name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs) as dist_location: install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location)) get_pip().spawn_install_wheel( wheel=dist_location, install_dir=install_dir, target=DistributionTarget(interpreter=interpreter), ).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert dist is not None yield dist
def test_distributionhelper_egg_assert(): d = DistributionHelper.distribution_from_path( './tests/example_packages/setuptools-18.0.1-py2.7.egg', 'setuptools' ) assert len(d.resource_listdir('/')) > 3
def _resolve(self, working_set, reqs): environment = self._target_interpreter_env.copy() environment["extra"] = list(set(itertools.chain(*(req.extras for req in reqs)))) reqs_by_key = OrderedDict() for req in reqs: if req.marker and not req.marker.evaluate(environment=environment): TRACER.log( "Skipping activation of `%s` due to environment marker de-selection" % req ) continue reqs_by_key.setdefault(req.key, []).append(req) unresolved_reqs = OrderedDict() resolveds = OrderedSet() # Resolve them one at a time so that we can figure out which ones we need to elide should # there be an interpreter incompatibility. for key, reqs in reqs_by_key.items(): with TRACER.timed("Resolving {} from {}".format(key, reqs), V=2): # N.B.: We resolve the bare requirement with no version specifiers since the resolve process # used to build this pex already did so. There may be multiple distributions satisfying any # particular key (e.g.: a Python 2 specific version and a Python 3 specific version for a # multi-python PEX) and we want the working set to pick the most appropriate one. req = Requirement.parse(key) try: resolveds.update(working_set.resolve([req], env=self)) except DistributionNotFound as e: TRACER.log("Failed to resolve a requirement: %s" % e) requirers = unresolved_reqs.setdefault(e.req, OrderedSet()) if e.requirers: for requirer in e.requirers: requirers.update(reqs_by_key[requirer]) if unresolved_reqs: TRACER.log("Unresolved requirements:") for req in unresolved_reqs: TRACER.log(" - %s" % req) TRACER.log("Distributions contained within this pex:") distributions_by_key = defaultdict(list) if not self._pex_info.distributions: TRACER.log(" None") else: for dist_name, dist_digest in self._pex_info.distributions.items(): TRACER.log(" - %s" % dist_name) distribution = DistributionHelper.distribution_from_path( path=os.path.join(self._pex_info.install_cache, dist_digest, dist_name) ) distributions_by_key[distribution.as_requirement().key].append(distribution) if not self._pex_info.ignore_errors: items = [] for index, (requirement, requirers) in enumerate(unresolved_reqs.items()): rendered_requirers = "" if requirers: rendered_requirers = ("\n Required by:" "\n {requirers}").format( requirers="\n ".join(map(str, requirers)) ) items.append( "{index: 2d}: {requirement}" "{rendered_requirers}" "\n But this pex only contains:" "\n {distributions}".format( index=index + 1, requirement=requirement, rendered_requirers=rendered_requirers, distributions="\n ".join( os.path.basename(d.location) for d in distributions_by_key[requirement.key] ), ) ) die( "Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}".format( platform=self._interpreter.platform, items="\n".join(items) ) ) return resolveds
def add_dist_location(self, location): self._builder.add_dist_location(location) dist = DistributionHelper.distribution_from_path(location) self._register_distribution(dist)
def _resolve(self, working_set, reqs): reqs_by_key = OrderedDict((req.key, req) for req in reqs) unresolved_reqs = OrderedDict() resolveds = OrderedSet() environment = self._target_interpreter_env.copy() environment['extra'] = list( set(itertools.chain(*(req.extras for req in reqs)))) # Resolve them one at a time so that we can figure out which ones we need to elide should # there be an interpreter incompatibility. for req in reqs_by_key.values(): if req.marker and not req.marker.evaluate(environment=environment): TRACER.log( 'Skipping activation of `%s` due to environment marker de-selection' % req) continue with TRACER.timed('Resolving %s' % req, V=2): try: resolveds.update(working_set.resolve([req], env=self)) except DistributionNotFound as e: TRACER.log('Failed to resolve a requirement: %s' % e) requirers = unresolved_reqs.setdefault(e.req, OrderedSet()) if e.requirers: requirers.update(reqs_by_key[requirer] for requirer in e.requirers) if unresolved_reqs: TRACER.log('Unresolved requirements:') for req in unresolved_reqs: TRACER.log(' - %s' % req) TRACER.log('Distributions contained within this pex:') distributions_by_key = defaultdict(list) if not self._pex_info.distributions: TRACER.log(' None') else: for dist_name, dist_digest in self._pex_info.distributions.items( ): TRACER.log(' - %s' % dist_name) distribution = DistributionHelper.distribution_from_path( path=os.path.join(self._pex_info.install_cache, dist_digest, dist_name)) distributions_by_key[ distribution.as_requirement().key].append(distribution) if not self._pex_info.ignore_errors: items = [] for index, (requirement, requirers) in enumerate(unresolved_reqs.items()): rendered_requirers = '' if requirers: rendered_requirers = ( '\n Required by:' '\n {requirers}').format( requirers='\n '.join(map(str, requirers))) items.append('{index: 2d}: {requirement}' '{rendered_requirers}' '\n But this pex only contains:' '\n {distributions}'.format( index=index + 1, requirement=requirement, rendered_requirers=rendered_requirers, distributions='\n '.join( os.path.basename(d.location) for d in distributions_by_key[ requirement.key]))) die('Failed to execute PEX file. Needed {platform} compatible dependencies for:\n{items}' .format(platform=Platform.of_interpreter( self._interpreter), items='\n'.join(items))) return resolveds