def installed_wheel(wheel_path): # type: (str) -> Iterator[Distribution] with temporary_dir() as install_dir: get_pip().spawn_install_wheel(wheel=wheel_path, install_dir=install_dir).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert dist is not None, "Could not load a distribution from {}.".format(install_dir) yield dist
def add_dist_location(self, dist, name=None): """Add a distribution by its location on disk. :param dist: The path to the distribution to add. :keyword name: (optional) The name of the distribution, should the dist directory alone be ambiguous. Packages contained within site-packages directories may require specifying ``name``. :raises PEXBuilder.InvalidDistribution: When the path does not contain a matching distribution. PEX supports packed and unpacked .whl and .egg distributions, as well as any distribution supported by setuptools/pkg_resources. """ self._ensure_unfrozen("Adding a distribution") dist_path = dist if os.path.isfile(dist_path) and dist_path.endswith(".whl"): dist_path = os.path.join(safe_mkdtemp(), os.path.basename(dist)) get_pip().spawn_install_wheel( wheel=dist, install_dir=dist_path, target=DistributionTarget.for_interpreter(self.interpreter), ).wait() dist = DistributionHelper.distribution_from_path(dist_path) self.add_distribution(dist, dist_name=name) self.add_requirement(dist.as_requirement())
def _add_dist_wheel_file(self, path, dist_name): with temporary_dir() as install_dir: get_pip().spawn_install_wheel( wheel=path, install_dir=install_dir, target=DistributionTarget.for_interpreter(self.interpreter), ).wait() return self._add_dist_dir(install_dir, dist_name)
def pip_wheel(pip_tgz_sdist): # type: (str) -> Iterator[str] with temporary_dir() as wheel_dir: get_pip().spawn_build_wheels([pip_tgz_sdist], wheel_dir=wheel_dir).wait() wheels = os.listdir(wheel_dir) assert len(wheels) == 1, "Expected 1 wheel to be built for {}.".format(pip_tgz_sdist) wheel = os.path.join(wheel_dir, wheels[0]) assert wheel.endswith(".whl") yield wheel
def install_wheel( wheel_path, # type: str install_dir, # type: str ): # type: (...) -> Distribution get_pip().spawn_install_wheel(wheel=wheel_path, install_dir=install_dir).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert dist is not None, "Could not load a distribution from {}".format(install_dir) return dist
def resolved_distribution(requirement): # type: (str) -> Distribution with temporary_dir() as td: download_dir = os.path.join(td, "download") get_pip().spawn_download_distributions( download_dir=download_dir, requirements=[requirement], transitive=False ).wait() wheels = os.listdir(download_dir) assert len(wheels) == 1, "Expected 1 wheel to be downloaded for {}".format(requirement) wheel_path = os.path.join(download_dir, wheels[0]) install_dir = os.path.join(td, "install") yield install_wheel(wheel_path, install_dir=install_dir)
def bdist(self): get_pip().spawn_build_wheels(distributions=[self._source_dir], wheel_dir=self._wheel_dir, interpreter=self._interpreter).wait() dists = os.listdir(self._wheel_dir) if len(dists) == 0: raise self.BuildFailure('No distributions were produced!') elif len(dists) > 1: raise self.BuildFailure( 'Ambiguous source distributions found: %s' % (' '.join(dists))) else: return os.path.join(self._wheel_dir, dists[0])
def bdist(self): # type: () -> str get_pip().spawn_build_wheels( distributions=[self._source_dir], wheel_dir=self._wheel_dir, interpreter=self._interpreter, ).wait() dists = os.listdir(self._wheel_dir) if len(dists) == 0: raise self.BuildFailure("No distributions were produced!") if len(dists) > 1: raise self.BuildFailure("Ambiguous source distributions found: %s" % (" ".join(dists))) return os.path.join(self._wheel_dir, dists[0])
def make_bdist(name='my_project', version='0.0.0', zip_safe=True, interpreter=None, **kwargs): with built_wheel(name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs) as dist_location: install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location)) get_pip().spawn_install_wheel( wheel=dist_location, install_dir=install_dir, target=DistributionTarget.for_interpreter(interpreter) ).wait() yield DistributionHelper.distribution_from_path(install_dir)
def downloaded_sdist(requirement): # type: (str) -> Iterator[str] with temporary_dir() as td: download_dir = os.path.join(td, "download") get_pip().spawn_download_distributions( download_dir=download_dir, requirements=[requirement], transitive=False, use_wheel=False, ).wait() dists = os.listdir(download_dir) assert len(dists) == 1, "Expected 1 dist to be downloaded for {}.".format(requirement) sdist = os.path.join(download_dir, dists[0]) assert sdist.endswith((".sdist", ".tar.gz", ".zip")) yield sdist
def test_get_script_from_distributions(tmpdir): whl_path = './tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl' install_dir = os.path.join(str(tmpdir), os.path.basename(whl_path)) get_pip().spawn_install_wheel(wheel=whl_path, install_dir=install_dir).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert 'aws-cfn-bootstrap' == dist.project_name dist_script = get_script_from_distributions('cfn-signal', [dist]) assert dist_script.dist is dist assert os.path.join(install_dir, 'bin/cfn-signal') == dist_script.path assert dist_script.read_contents().startswith('#!'), ( 'Expected a `scripts`-style script w/shebang.') assert None is get_script_from_distributions('non_existent_script', [dist])
def test_get_script_from_distributions(tmpdir): # type: (Any) -> None whl_path = "./tests/example_packages/aws_cfn_bootstrap-1.4-py2-none-any.whl" install_dir = os.path.join(str(tmpdir), os.path.basename(whl_path)) get_pip().spawn_install_wheel(wheel=whl_path, install_dir=install_dir).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert "aws-cfn-bootstrap" == dist.project_name dist_script = get_script_from_distributions("cfn-signal", [dist]) assert dist_script.dist is dist assert os.path.join(install_dir, "bin/cfn-signal") == dist_script.path assert dist_script.read_contents().startswith( "#!"), "Expected a `scripts`-style script w/shebang." assert None is get_script_from_distributions("non_existent_script", [dist])
def _spawn_wheel_build(self, built_wheels_dir, build_request): build_result = build_request.result(built_wheels_dir) build_job = get_pip().spawn_build_wheels( distributions=[build_request.source_path], wheel_dir=build_result.build_dir, cache=self._cache, interpreter=build_request.target.get_interpreter()) return SpawnedJob.wait(job=build_job, result=build_result)
def _spawn_install(self, installed_wheels_dir, install_request): install_result = install_request.result(installed_wheels_dir) install_job = get_pip().spawn_install_wheel( wheel=install_request.wheel_path, install_dir=install_result.build_chroot, compile=self._compile, cache=self._cache, target=install_request.target) return SpawnedJob.wait(job=install_job, result=install_result)
def _spawn_wheel_build(self, built_wheels_dir, build_request): build_result = build_request.result(built_wheels_dir) build_job = get_pip().spawn_build_wheels( distributions=[build_request.source_path], wheel_dir=build_result.build_dir, cache=self._cache, indexes=self._indexes, find_links=self._find_links, network_configuration=self._network_configuration, interpreter=build_request.target.get_interpreter()) return SpawnedJob.wait(job=build_job, result=build_result)
def make_bdist( name="my_project", # type: str version="0.0.0", # type: str zip_safe=True, # type: bool interpreter=None, # type: Optional[PythonInterpreter] **kwargs # type: Any ): # type: (...) -> Iterator[Distribution] with built_wheel( name=name, version=version, zip_safe=zip_safe, interpreter=interpreter, **kwargs ) as dist_location: install_dir = os.path.join(safe_mkdtemp(), os.path.basename(dist_location)) get_pip().spawn_install_wheel( wheel=dist_location, install_dir=install_dir, target=DistributionTarget.for_interpreter(interpreter), ).wait() dist = DistributionHelper.distribution_from_path(install_dir) assert dist is not None yield dist
def _spawn_resolve(self, resolved_dists_dir, target): download_dir = os.path.join(resolved_dists_dir, target.id) download_job = get_pip().spawn_download_distributions( download_dir=download_dir, requirements=self._requirements, requirement_files=self._requirement_files, constraint_files=self._constraint_files, allow_prereleases=self._allow_prereleases, transitive=self._transitive, target=target, indexes=self._indexes, find_links=self._find_links, cache=self._cache, build=self._build, manylinux=self._manylinux, use_wheel=self._use_wheel) return SpawnedJob.wait(job=download_job, result=ResolveResult(target, download_dir))
def _spawn_download(self, resolved_dists_dir, target): download_dir = os.path.join(resolved_dists_dir, target.id) download_job = get_pip().spawn_download_distributions( download_dir=download_dir, requirements=self.requirements, requirement_files=self.requirement_files, constraint_files=self.constraint_files, allow_prereleases=self.allow_prereleases, transitive=self.transitive, target=target, indexes=self.indexes, find_links=self.find_links, network_configuration=self.network_configuration, cache=self.cache, build=self.build, manylinux=self.manylinux, use_wheel=self.use_wheel) return SpawnedJob.wait(job=download_job, result=DownloadResult(target, download_dir))
def _calculate_tags( self, manylinux=None, # type: Optional[str] ): # type: (...) -> Iterator[tags.Tag] from pex.jobs import SpawnedJob from pex.pip import get_pip def parse_tags(output): # type: (bytes) -> Iterator[tags.Tag] count = None # type: Optional[int] try: for line in output.decode("utf-8").splitlines(): if count is None: match = re.match( r"^Compatible tags: (?P<count>\d+)\s+", line) if match: count = int(match.group("count")) continue count -= 1 if count < 0: raise AssertionError( "Expected {} tags but got more.".format(count)) for tag in tags.parse_tag(line.strip()): yield tag finally: if count != 0: raise AssertionError( "Finished with count {}.".format(count)) job = SpawnedJob.stdout( job=get_pip().spawn_debug( platform=self.platform, impl=self.impl, version=self.version, abi=self.abi, manylinux=manylinux, ), result_func=parse_tags, ) return job.await_result()
def test_issues_789_demo(): # type: () -> None tmpdir = safe_mkdtemp() pex_project_dir = (subprocess.check_output( ["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()) # 1. Imagine we've pre-resolved the requirements needed in our wheel house. requirements = [ "ansicolors", "isort", "setuptools", # N.B.: isort doesn't declare its setuptools dependency. ] wheelhouse = os.path.join(tmpdir, "wheelhouse") get_pip().spawn_download_distributions(download_dir=wheelhouse, requirements=requirements).wait() # 2. Also imagine this configuration is passed to a tool (PEX or a wrapper as in this test # example) via the CLI or other configuration data sources. For example, Pants has a `PythonSetup` # that combines with BUILD target data to get you this sort of configuration info outside pex. resolver_settings = dict( indexes=[], # Turn off pypi. find_links=[wheelhouse], # Use our wheel house. build=False, # Use only pre-built wheels. ) # type: Dict[str, Any] # 3. That same configuration was used to build a standard pex: resolver_args = [] if len(resolver_settings["find_links"]) == 0: resolver_args.append("--no-index") else: for index in resolver_settings["indexes"]: resolver_args.extend(["--index", index]) for repo in resolver_settings["find_links"]: resolver_args.extend(["--find-links", repo]) resolver_args.append( "--build" if resolver_settings["build"] else "--no-build") project_code_dir = os.path.join(tmpdir, "project_code_dir") with safe_open(os.path.join(project_code_dir, "colorized_isort.py"), "w") as fp: fp.write( dedent("""\ import colors import os import subprocess import sys def run(): env = os.environ.copy() env.update(PEX_MODULE='isort') isort_process = subprocess.Popen( sys.argv, env=env, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout, stderr = isort_process.communicate() print(colors.green(stdout.decode('utf-8'))) print(colors.red(stderr.decode('utf-8'))) sys.exit(isort_process.returncode) """)) colorized_isort_pex = os.path.join(tmpdir, "colorized_isort.pex") args = [ "--sources-directory", project_code_dir, "--entry-point", "colorized_isort:run", "--output-file", colorized_isort_pex, ] result = run_pex_command(args + resolver_args + requirements) result.assert_success() # 4. Now the tool builds a "dehydrated" PEX using the standard pex + resolve settings as the # template. ptex_cache = os.path.join(tmpdir, ".ptex") colorized_isort_pex_info = PexInfo.from_pex(colorized_isort_pex) colorized_isort_pex_info.pex_root = ptex_cache # Force the standard pex to extract its code. An external tool like Pants would already know the # orignal source code file paths, but we need to discover here. colorized_isort_pex_code_dir = os.path.join( colorized_isort_pex_info.zip_unsafe_cache, colorized_isort_pex_info.code_hash) env = os.environ.copy() env.update(PEX_ROOT=ptex_cache, PEX_INTERPRETER="1", PEX_FORCE_LOCAL="1") subprocess.check_call([colorized_isort_pex, "-c", ""], env=env) colorized_isort_ptex_code_dir = os.path.join( tmpdir, "colorized_isort_ptex_code_dir") safe_mkdir(colorized_isort_ptex_code_dir) code = [] for root, dirs, files in os.walk(colorized_isort_pex_code_dir): rel_root = os.path.relpath(root, colorized_isort_pex_code_dir) for f in files: # Don't ship compiled python from the code extract above, the target interpreter will not # match ours in general. if f.endswith(".pyc"): continue rel_path = os.path.normpath(os.path.join(rel_root, f)) # The root __main__.py is special for any zipapp including pex, let it write its own # __main__.py bootstrap. Similarly. PEX-INFO is special to pex and we want the PEX-INFO for # The ptex pex, not the pex being ptexed. if rel_path in ("__main__.py", PexInfo.PATH): continue os.symlink(os.path.join(root, f), os.path.join(colorized_isort_ptex_code_dir, rel_path)) code.append(rel_path) ptex_code_dir = os.path.join(tmpdir, "ptex_code_dir") ptex_info = dict(code=code, resolver_settings=resolver_settings) with safe_open(os.path.join(ptex_code_dir, "PTEX-INFO"), "w") as fp: json.dump(ptex_info, fp) with safe_open(os.path.join(ptex_code_dir, "IPEX-INFO"), "w") as fp: fp.write(colorized_isort_pex_info.dump()) with safe_open(os.path.join(ptex_code_dir, "ptex.py"), "w") as fp: fp.write( dedent("""\ import json import os import sys from pex import resolver from pex.common import open_zip from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.util import CacheHelper from pex.variables import ENV self = sys.argv[0] ipex_file = '{}.ipex'.format(os.path.splitext(self)[0]) if not os.path.isfile(ipex_file): print('Hydrating {} to {}'.format(self, ipex_file)) ptex_pex_info = PexInfo.from_pex(self) code_root = os.path.join(ptex_pex_info.zip_unsafe_cache, ptex_pex_info.code_hash) with open_zip(self) as zf: # Populate the pex with the pinned requirements and distribution names & hashes. ipex_info = PexInfo.from_json(zf.read('IPEX-INFO')) ipex_builder = PEXBuilder(pex_info=ipex_info) # Populate the pex with the needed code. ptex_info = json.loads(zf.read('PTEX-INFO').decode('utf-8')) for path in ptex_info['code']: ipex_builder.add_source(os.path.join(code_root, path), path) # Perform a fully pinned intransitive resolve to hydrate the install cache (not the # pex!). resolver_settings = ptex_info['resolver_settings'] resolved_distributions = resolver.resolve( requirements=[str(req) for req in ipex_info.requirements], cache=ipex_info.pex_root, transitive=False, **resolver_settings ) ipex_builder.build(ipex_file) os.execv(ipex_file, [ipex_file] + sys.argv[1:]) """)) colorized_isort_ptex = os.path.join(tmpdir, "colorized_isort.ptex") result = run_pex_command([ "--not-zip-safe", "--always-write-cache", "--pex-root", ptex_cache, pex_project_dir, # type: ignore[list-item] # This is unicode in Py2, whereas everthing else is bytes. That's fine. "--sources-directory", ptex_code_dir, "--sources-directory", colorized_isort_ptex_code_dir, "--entry-point", "ptex", "--output-file", colorized_isort_ptex, ]) result.assert_success() subprocess.check_call([colorized_isort_ptex, "--version"]) with pytest.raises(CalledProcessError): subprocess.check_call([colorized_isort_ptex, "--not-a-flag"]) safe_rmtree(ptex_cache) # The dehydrated pex now fails since it lost its hydration from the cache. with pytest.raises(CalledProcessError): subprocess.check_call([colorized_isort_ptex, "--version"])