def vendorize(root_dir, vendor_specs, prefix): for vendor_spec in vendor_specs: cmd = [ 'pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir, vendor_spec.requirement ] result = subprocess.call(cmd) if result != 0: raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec)) # We know we can get these as a by-product of a pip install but never need them. safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin')) safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py')) # The RECORD contains file hashes of all installed files and is unfortunately unstable in the # case of scripts which get a shebang added with a system-specific path to the python # interpreter to execute. for record_file in glob.glob( os.path.join(vendor_spec.target_dir, '*-*.dist-info/RECORD')): safe_delete(record_file) vendor_spec.create_packages() vendored_path = [ vendor_spec.target_dir for vendor_spec in vendor_specs if vendor_spec.rewrite ] import_rewriter = ImportRewriter.for_path_items(prefix=prefix, path_items=vendored_path) rewrite_paths = [os.path.join(root_dir, c) for c in ('pex', 'tests')] + vendored_path for rewrite_path in rewrite_paths: for root, dirs, files in os.walk(rewrite_path): if root == os.path.join(root_dir, 'pex', 'vendor'): dirs[:] = [d for d in dirs if d != '_vendored'] for f in files: if f.endswith('.py'): python_file = os.path.join(root, f) print( green('Examining {python_file}...'.format( python_file=python_file))) modifications = import_rewriter.rewrite(python_file) if modifications: num_mods = len(modifications) print( bold( green( ' Vendorized {count} import{plural} in {python_file}' .format(count=num_mods, plural='s' if num_mods > 1 else '', python_file=python_file)))) for _from, _to in modifications.items(): print(' {} -> {}'.format(_from, _to))
def bootstrap_python_installer(dest): # type: (str) -> None safe_rmtree(dest) for _ in range(3): try: subprocess.check_call(["git", "clone", "https://github.com/pyenv/pyenv.git", dest]) except subprocess.CalledProcessError as e: print("caught exception: %r" % e) continue else: break else: raise RuntimeError("Helper method could not clone pyenv from git after 3 tries") # Create an empty file indicating the fingerprint of the correct set of test interpreters. touch(os.path.join(dest, _INTERPRETER_SET_FINGERPRINT))
def bootstrap_python_installer(dest): safe_rmtree(dest) for _ in range(3): try: subprocess.check_call( ['git', 'clone', 'https://github.com/pyenv/pyenv.git', dest] ) except subprocess.CalledProcessError as e: print('caught exception: %r' % e) continue else: break else: raise RuntimeError("Helper method could not clone pyenv from git after 3 tries") # Create an empty file indicating the fingerprint of the correct set of test interpreters. touch(os.path.join(dest, _INTERPRETER_SET_FINGERPRINT))
def translate(self, package, into=None): """From a SourcePackage, translate to a binary distribution.""" if not isinstance(package, SourcePackage): return None if not package.local: raise ValueError( 'SourceTranslator cannot translate remote packages.') installer = None version = self._interpreter.version unpack_path = Archiver.unpack(package.local_path) into = into or safe_mkdtemp() try: if self._use_2to3 and version >= (3, ): with TRACER.timed('Translating 2->3 %s' % package.name): self.run_2to3(unpack_path) installer = self._installer_impl(unpack_path, interpreter=self._interpreter) with TRACER.timed('Packaging %s' % package.name): try: dist_path = installer.bdist() except self._installer_impl.InstallFailure as e: TRACER.log('Failed to install package at %s: %s' % (unpack_path, e)) return None target_path = os.path.join(into, os.path.basename(dist_path)) safe_copy(dist_path, target_path) target_package = Package.from_href(target_path) if not target_package: TRACER.log('Target path %s does not look like a Package.' % target_path) return None if not target_package.compatible(self._supported_tags): TRACER.log('Target package %s is not compatible with %s' % (target_package, self._supported_tags)) return None return DistributionHelper.distribution_from_path(target_path) except Exception as e: TRACER.log('Failed to translate %s' % package) TRACER.log(traceback.format_exc()) finally: if installer: installer.cleanup() if unpack_path: safe_rmtree(unpack_path)
def force_local(cls, pex_file, pex_info): if pex_info.code_hash is None: # Do not support force_local if code_hash is not set. (It should always be set.) return pex_file explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash) TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir) if not os.path.exists(explode_dir): explode_tmp = explode_dir + '.' + uuid.uuid4().hex with TRACER.timed('Unzipping %s' % pex_file): try: safe_mkdir(explode_tmp) with open_zip(pex_file) as pex_zip: pex_files = (x for x in pex_zip.namelist() if not x.startswith(pex_builder.BOOTSTRAP_DIR) and not x.startswith(PexInfo.INTERNAL_CACHE)) pex_zip.extractall(explode_tmp, pex_files) except: # noqa: T803 safe_rmtree(explode_tmp) raise TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir)) rename_if_empty(explode_tmp, explode_dir) return explode_dir
def vendorize(root_dir, vendor_specs, prefix): for vendor_spec in vendor_specs: cmd = [ 'pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir, vendor_spec.requirement ] result = subprocess.call(cmd) if result != 0: raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec)) # We know we can get these as a by-product of a pip install but never need them. safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin')) safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py')) vendor_spec.create_packages() vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs] import_rewriter = ImportRewriter.for_path_items(prefix=prefix, path_items=vendored_path) for root, dirs, files in os.walk(root_dir): if root == root_dir: dirs[:] = ['pex', 'tests'] for f in files: if f.endswith('.py'): python_file = os.path.join(root, f) print( green('Examining {python_file}...'.format( python_file=python_file))) modifications = import_rewriter.rewrite(python_file) if modifications: num_mods = len(modifications) print( bold( green( ' Vendorized {count} import{plural} in {python_file}' .format(count=num_mods, plural='s' if num_mods > 1 else '', python_file=python_file)))) for _from, _to in modifications.items(): print(' {} -> {}'.format(_from, _to))
def vendorize(root_dir, vendor_specs, prefix): for vendor_spec in vendor_specs: cmd = ['pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir, vendor_spec.requirement] result = subprocess.call(cmd) if result != 0: raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec)) # We know we can get these as a by-product of a pip install but never need them. safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin')) safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py')) # The RECORD contains file hashes of all installed files and is unfortunately unstable in the # case of scripts which get a shebang added with a system-specific path to the python # interpreter to execute. safe_delete(os.path.join(vendor_spec.target_dir, '{}-{}.dist-info/RECORD'.format(vendor_spec.key, vendor_spec.version))) vendor_spec.create_packages() vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs] import_rewriter = ImportRewriter.for_path_items(prefix=prefix, path_items=vendored_path) for root, dirs, files in os.walk(root_dir): if root == root_dir: dirs[:] = ['pex', 'tests'] for f in files: if f.endswith('.py'): python_file = os.path.join(root, f) print(green('Examining {python_file}...'.format(python_file=python_file))) modifications = import_rewriter.rewrite(python_file) if modifications: num_mods = len(modifications) print(bold(green(' Vendorized {count} import{plural} in {python_file}' .format(count=num_mods, plural='s' if num_mods > 1 else '', python_file=python_file)))) for _from, _to in modifications.items(): print(' {} -> {}'.format(_from, _to))
def cleanup(self): safe_rmtree(self._install_tmp)
def vendorize(root_dir, vendor_specs, prefix): for vendor_spec in vendor_specs: # NB: We set --no-build-isolation to prevent pip from installing the requirements listed in # its [build-system] config in its pyproject.toml. # # Those requirements are (currently) the unpinned ["setuptools", "wheel"], which will cause pip # to use the latest version of those packages. This is a hermeticity problem: re-vendoring a # package at a later time may yield a different result. At the very least the result will # differ in the version embedded in the WHEEL metadata file, which causes issues with our # tests. # # Setting --no-build-isolation means that versions of setuptools and wheel must be provided # in the environment in which we run the pip command, which is the environment in which we run # pex.vendor. Since we document that pex.vendor should be run via tox, that environment will # contain pinned versions of setuptools and wheel. As a result, vendoring (at least via tox) # is hermetic. cmd = [ 'pip', 'install', '--upgrade', '--no-build-isolation', '--no-compile', '--target', vendor_spec.target_dir, vendor_spec.requirement ] constraints_file = os.path.join(vendor_spec.target_dir, 'constraints.txt') if vendor_spec.constrain and os.path.isfile(constraints_file): cmd.extend(['--constraint', constraints_file]) result = subprocess.call(cmd) if result != 0: raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec)) if vendor_spec.constrain: cmd = ['pip', 'freeze', '--all', '--path', vendor_spec.target_dir] process = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout, _ = process.communicate() if process.returncode != 0: raise VendorizeError( 'Failed to freeze vendoring of {!r}'.format(vendor_spec)) with open(constraints_file, 'wb') as fp: fp.write(stdout) # We know we can get these as a by-product of a pip install but never need them. safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin')) safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py')) # The RECORD contains file hashes of all installed files and is unfortunately unstable in the # case of scripts which get a shebang added with a system-specific path to the python # interpreter to execute. for record_file in glob.glob( os.path.join(vendor_spec.target_dir, '*-*.dist-info/RECORD')): safe_delete(record_file) vendor_spec.create_packages() vendored_path = [ vendor_spec.target_dir for vendor_spec in vendor_specs if vendor_spec.rewrite ] import_rewriter = ImportRewriter.for_path_items(prefix=prefix, path_items=vendored_path) rewrite_paths = [os.path.join(root_dir, c) for c in ('pex', 'tests')] + vendored_path for rewrite_path in rewrite_paths: for root, dirs, files in os.walk(rewrite_path): if root == os.path.join(root_dir, 'pex', 'vendor'): dirs[:] = [d for d in dirs if d != '_vendored'] for f in files: if f.endswith('.py'): python_file = os.path.join(root, f) print( green('Examining {python_file}...'.format( python_file=python_file))) modifications = import_rewriter.rewrite(python_file) if modifications: num_mods = len(modifications) print( bold( green( ' Vendorized {count} import{plural} in {python_file}' .format(count=num_mods, plural='s' if num_mods > 1 else '', python_file=python_file)))) for _from, _to in modifications.items(): print(' {} -> {}'.format(_from, _to))
def test_issues_789_demo(): # type: () -> None tmpdir = safe_mkdtemp() pex_project_dir = (subprocess.check_output( ["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()) # 1. Imagine we've pre-resolved the requirements needed in our wheel house. requirements = [ "ansicolors", "isort", "setuptools", # N.B.: isort doesn't declare its setuptools dependency. ] wheelhouse = os.path.join(tmpdir, "wheelhouse") get_pip().spawn_download_distributions(download_dir=wheelhouse, requirements=requirements).wait() # 2. Also imagine this configuration is passed to a tool (PEX or a wrapper as in this test # example) via the CLI or other configuration data sources. For example, Pants has a `PythonSetup` # that combines with BUILD target data to get you this sort of configuration info outside pex. resolver_settings = dict( indexes=[], # Turn off pypi. find_links=[wheelhouse], # Use our wheel house. build=False, # Use only pre-built wheels. ) # type: Dict[str, Any] # 3. That same configuration was used to build a standard pex: resolver_args = [] if len(resolver_settings["find_links"]) == 0: resolver_args.append("--no-index") else: for index in resolver_settings["indexes"]: resolver_args.extend(["--index", index]) for repo in resolver_settings["find_links"]: resolver_args.extend(["--find-links", repo]) resolver_args.append( "--build" if resolver_settings["build"] else "--no-build") project_code_dir = os.path.join(tmpdir, "project_code_dir") with safe_open(os.path.join(project_code_dir, "colorized_isort.py"), "w") as fp: fp.write( dedent("""\ import colors import os import subprocess import sys def run(): env = os.environ.copy() env.update(PEX_MODULE='isort') isort_process = subprocess.Popen( sys.argv, env=env, stdout = subprocess.PIPE, stderr = subprocess.PIPE ) stdout, stderr = isort_process.communicate() print(colors.green(stdout.decode('utf-8'))) print(colors.red(stderr.decode('utf-8'))) sys.exit(isort_process.returncode) """)) colorized_isort_pex = os.path.join(tmpdir, "colorized_isort.pex") args = [ "--sources-directory", project_code_dir, "--entry-point", "colorized_isort:run", "--output-file", colorized_isort_pex, ] result = run_pex_command(args + resolver_args + requirements) result.assert_success() # 4. Now the tool builds a "dehydrated" PEX using the standard pex + resolve settings as the # template. ptex_cache = os.path.join(tmpdir, ".ptex") colorized_isort_pex_info = PexInfo.from_pex(colorized_isort_pex) colorized_isort_pex_info.pex_root = ptex_cache # Force the standard pex to extract its code. An external tool like Pants would already know the # orignal source code file paths, but we need to discover here. colorized_isort_pex_code_dir = os.path.join( colorized_isort_pex_info.zip_unsafe_cache, colorized_isort_pex_info.code_hash) env = os.environ.copy() env.update(PEX_ROOT=ptex_cache, PEX_INTERPRETER="1", PEX_FORCE_LOCAL="1") subprocess.check_call([colorized_isort_pex, "-c", ""], env=env) colorized_isort_ptex_code_dir = os.path.join( tmpdir, "colorized_isort_ptex_code_dir") safe_mkdir(colorized_isort_ptex_code_dir) code = [] for root, dirs, files in os.walk(colorized_isort_pex_code_dir): rel_root = os.path.relpath(root, colorized_isort_pex_code_dir) for f in files: # Don't ship compiled python from the code extract above, the target interpreter will not # match ours in general. if f.endswith(".pyc"): continue rel_path = os.path.normpath(os.path.join(rel_root, f)) # The root __main__.py is special for any zipapp including pex, let it write its own # __main__.py bootstrap. Similarly. PEX-INFO is special to pex and we want the PEX-INFO for # The ptex pex, not the pex being ptexed. if rel_path in ("__main__.py", PexInfo.PATH): continue os.symlink(os.path.join(root, f), os.path.join(colorized_isort_ptex_code_dir, rel_path)) code.append(rel_path) ptex_code_dir = os.path.join(tmpdir, "ptex_code_dir") ptex_info = dict(code=code, resolver_settings=resolver_settings) with safe_open(os.path.join(ptex_code_dir, "PTEX-INFO"), "w") as fp: json.dump(ptex_info, fp) with safe_open(os.path.join(ptex_code_dir, "IPEX-INFO"), "w") as fp: fp.write(colorized_isort_pex_info.dump()) with safe_open(os.path.join(ptex_code_dir, "ptex.py"), "w") as fp: fp.write( dedent("""\ import json import os import sys from pex import resolver from pex.common import open_zip from pex.pex_builder import PEXBuilder from pex.pex_info import PexInfo from pex.util import CacheHelper from pex.variables import ENV self = sys.argv[0] ipex_file = '{}.ipex'.format(os.path.splitext(self)[0]) if not os.path.isfile(ipex_file): print('Hydrating {} to {}'.format(self, ipex_file)) ptex_pex_info = PexInfo.from_pex(self) code_root = os.path.join(ptex_pex_info.zip_unsafe_cache, ptex_pex_info.code_hash) with open_zip(self) as zf: # Populate the pex with the pinned requirements and distribution names & hashes. ipex_info = PexInfo.from_json(zf.read('IPEX-INFO')) ipex_builder = PEXBuilder(pex_info=ipex_info) # Populate the pex with the needed code. ptex_info = json.loads(zf.read('PTEX-INFO').decode('utf-8')) for path in ptex_info['code']: ipex_builder.add_source(os.path.join(code_root, path), path) # Perform a fully pinned intransitive resolve to hydrate the install cache (not the # pex!). resolver_settings = ptex_info['resolver_settings'] resolved_distributions = resolver.resolve( requirements=[str(req) for req in ipex_info.requirements], cache=ipex_info.pex_root, transitive=False, **resolver_settings ) ipex_builder.build(ipex_file) os.execv(ipex_file, [ipex_file] + sys.argv[1:]) """)) colorized_isort_ptex = os.path.join(tmpdir, "colorized_isort.ptex") result = run_pex_command([ "--not-zip-safe", "--always-write-cache", "--pex-root", ptex_cache, pex_project_dir, # type: ignore[list-item] # This is unicode in Py2, whereas everthing else is bytes. That's fine. "--sources-directory", ptex_code_dir, "--sources-directory", colorized_isort_ptex_code_dir, "--entry-point", "ptex", "--output-file", colorized_isort_ptex, ]) result.assert_success() subprocess.check_call([colorized_isort_ptex, "--version"]) with pytest.raises(CalledProcessError): subprocess.check_call([colorized_isort_ptex, "--not-a-flag"]) safe_rmtree(ptex_cache) # The dehydrated pex now fails since it lost its hydration from the cache. with pytest.raises(CalledProcessError): subprocess.check_call([colorized_isort_ptex, "--version"])
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode('utf-8').strip() if not identity: raise cls.IdentificationError( 'Could not establish identity of %s' % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode('utf-8')) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters') os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed('GCing interpreter cache from prior OS version'): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) cache_dir = os.path.join(os_cache_dir, interpreter_hash) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, 'rb') as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(['pex']) cmd, env = cls._create_isolated_cmd(binary, args=[ '-c', dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)) ], pythonpath=pythonpath) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of %s" % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. path_id = binary.replace(os.sep, ".").lstrip(".") cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def supported_tags(self, manylinux=None): # type: (Optional[str]) -> Tuple[tags.Tag, ...] # We use a 2 level cache, probing memory first and then a json file on disk in order to # avoid calculating tags when possible since it's an O(500ms) operation that involves # spawning Pip. # Read level 1. memory_cache_key = (self, manylinux) supported_tags = self._SUPPORTED_TAGS_BY_PLATFORM.get(memory_cache_key) if supported_tags is not None: return supported_tags # Read level 2. components = list(attr.astuple(self)) if manylinux: components.append(manylinux) disk_cache_key = os.path.join(ENV.PEX_ROOT, "platforms", self.SEP.join(components)) with atomic_directory(target_dir=disk_cache_key, exclusive=False) as cache_dir: if not cache_dir.is_finalized: # Missed both caches - spawn calculation. plat_info = attr.asdict(self) plat_info.update( supported_tags=[ (tag.interpreter, tag.abi, tag.platform) for tag in self._calculate_tags(manylinux=manylinux) ], ) # Write level 2. with safe_open(os.path.join(cache_dir.work_dir, self.PLAT_INFO_FILE), "w") as fp: json.dump(plat_info, fp) with open(os.path.join(disk_cache_key, self.PLAT_INFO_FILE)) as fp: try: data = json.load(fp) except ValueError as e: TRACER.log( "Regenerating the platform info file at {} since it did not contain parsable " "JSON data: {}".format(fp.name, e) ) safe_rmtree(disk_cache_key) return self.supported_tags(manylinux=manylinux) if not isinstance(data, dict): TRACER.log( "Regenerating the platform info file at {} since it did not contain a " "configuration object. Found: {!r}".format(fp.name, data) ) safe_rmtree(disk_cache_key) return self.supported_tags(manylinux=manylinux) sup_tags = data.get("supported_tags") if not isinstance(sup_tags, list): TRACER.log( "Regenerating the platform info file at {} since it was missing a valid " "`supported_tags` list. Found: {!r}".format(fp.name, sup_tags) ) safe_rmtree(disk_cache_key) return self.supported_tags(manylinux=manylinux) count = len(sup_tags) def parse_tag( index, # type: int tag, # type: List[Any] ): # type: (...) -> tags.Tag if len(tag) != 3 or not all( isinstance(component, compatibility.string) for component in tag ): raise ValueError( "Serialized platform tags should be lists of three strings. Tag {index} of " "{count} was: {tag!r}.".format(index=index, count=count, tag=tag) ) interpreter, abi, platform = tag return tags.Tag(interpreter=interpreter, abi=abi, platform=platform) try: supported_tags = tuple(parse_tag(index, tag) for index, tag in enumerate(sup_tags)) # Write level 1. self._SUPPORTED_TAGS_BY_PLATFORM[memory_cache_key] = supported_tags return supported_tags except ValueError as e: TRACER.log( "Regenerating the platform info file at {} since it did not contain parsable " "tag data: {}".format(fp.name, e) ) safe_rmtree(disk_cache_key) return self.supported_tags(manylinux=manylinux)
modifications = import_rewriter.rewrite(python_file) if modifications: num_mods = len(modifications) print( bold( green( " Vendorized {count} import{plural} in {python_file}" .format( count=num_mods, plural="s" if num_mods > 1 else "", python_file=python_file, )))) for _from, _to in modifications.items(): print(" {} -> {}".format(_from, _to)) if __name__ == "__main__": if len(sys.argv) != 1: print("Usage: {}".format(sys.argv[0]), file=sys.stderr) sys.exit(1) root_directory = VendorSpec.ROOT import_prefix = third_party.import_prefix() try: safe_rmtree(VendorSpec.vendor_root()) vendorize(root_directory, list(iter_vendor_specs()), import_prefix) sys.exit(0) except VendorizeError as e: print("Problem encountered vendorizing: {}".format(e), file=sys.stderr) sys.exit(1)
def temporary_dir(): td = tempfile.mkdtemp() try: yield td finally: safe_rmtree(td)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout, check_binary=False): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of {}.".format(binary)) interpreter = cls(PythonIdentity.decode(identity)) # We should not need to check this since binary == interpreter.binary should always be # true, but historically this could be untrue as noted in `PythonIdentity.get`. if check_binary and not os.path.exists(interpreter.binary): raise cls.InterpreterNotFound( "Cached interpreter for {} reports a binary of {}, which could not be found" .format(binary, interpreter.binary)) return interpreter # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. # # N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed # length hash of the interpreter binary path to ensure uniqueness and not run afoul of file # name length limits. path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest() cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed( create_interpreter(fp.read(), check_binary=True)) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get(binary={binary!r}).encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(binary=binary, cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)