def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
cmd = [
'pip', 'install', '--upgrade', '--no-compile', '--target',
vendor_spec.target_dir, vendor_spec.requirement
]
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
# The RECORD contains file hashes of all installed files and is unfortunately unstable in the
# case of scripts which get a shebang added with a system-specific path to the python
# interpreter to execute.
for record_file in glob.glob(
os.path.join(vendor_spec.target_dir, '*-*.dist-info/RECORD')):
safe_delete(record_file)
vendor_spec.create_packages()
vendored_path = [
vendor_spec.target_dir for vendor_spec in vendor_specs
if vendor_spec.rewrite
]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix,
path_items=vendored_path)
rewrite_paths = [os.path.join(root_dir, c)
for c in ('pex', 'tests')] + vendored_path
for rewrite_path in rewrite_paths:
for root, dirs, files in os.walk(rewrite_path):
if root == os.path.join(root_dir, 'pex', 'vendor'):
dirs[:] = [d for d in dirs if d != '_vendored']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(
green('Examining {python_file}...'.format(
python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def bootstrap_python_installer(dest):
# type: (str) -> None
safe_rmtree(dest)
for _ in range(3):
try:
subprocess.check_call(["git", "clone", "https://github.com/pyenv/pyenv.git", dest])
except subprocess.CalledProcessError as e:
print("caught exception: %r" % e)
continue
else:
break
else:
raise RuntimeError("Helper method could not clone pyenv from git after 3 tries")
# Create an empty file indicating the fingerprint of the correct set of test interpreters.
touch(os.path.join(dest, _INTERPRETER_SET_FINGERPRINT))
def bootstrap_python_installer(dest):
safe_rmtree(dest)
for _ in range(3):
try:
subprocess.check_call(
['git', 'clone', 'https://github.com/pyenv/pyenv.git', dest]
)
except subprocess.CalledProcessError as e:
print('caught exception: %r' % e)
continue
else:
break
else:
raise RuntimeError("Helper method could not clone pyenv from git after 3 tries")
# Create an empty file indicating the fingerprint of the correct set of test interpreters.
touch(os.path.join(dest, _INTERPRETER_SET_FINGERPRINT))
def translate(self, package, into=None):
"""From a SourcePackage, translate to a binary distribution."""
if not isinstance(package, SourcePackage):
return None
if not package.local:
raise ValueError(
'SourceTranslator cannot translate remote packages.')
installer = None
version = self._interpreter.version
unpack_path = Archiver.unpack(package.local_path)
into = into or safe_mkdtemp()
try:
if self._use_2to3 and version >= (3, ):
with TRACER.timed('Translating 2->3 %s' % package.name):
self.run_2to3(unpack_path)
installer = self._installer_impl(unpack_path,
interpreter=self._interpreter)
with TRACER.timed('Packaging %s' % package.name):
try:
dist_path = installer.bdist()
except self._installer_impl.InstallFailure as e:
TRACER.log('Failed to install package at %s: %s' %
(unpack_path, e))
return None
target_path = os.path.join(into, os.path.basename(dist_path))
safe_copy(dist_path, target_path)
target_package = Package.from_href(target_path)
if not target_package:
TRACER.log('Target path %s does not look like a Package.' %
target_path)
return None
if not target_package.compatible(self._supported_tags):
TRACER.log('Target package %s is not compatible with %s' %
(target_package, self._supported_tags))
return None
return DistributionHelper.distribution_from_path(target_path)
except Exception as e:
TRACER.log('Failed to translate %s' % package)
TRACER.log(traceback.format_exc())
finally:
if installer:
installer.cleanup()
if unpack_path:
safe_rmtree(unpack_path)
def force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
if not os.path.exists(explode_dir):
explode_tmp = explode_dir + '.' + uuid.uuid4().hex
with TRACER.timed('Unzipping %s' % pex_file):
try:
safe_mkdir(explode_tmp)
with open_zip(pex_file) as pex_zip:
pex_files = (x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
not x.startswith(PexInfo.INTERNAL_CACHE))
pex_zip.extractall(explode_tmp, pex_files)
except: # noqa: T803
safe_rmtree(explode_tmp)
raise
TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir))
rename_if_empty(explode_tmp, explode_dir)
return explode_dir
def force_local(cls, pex_file, pex_info):
if pex_info.code_hash is None:
# Do not support force_local if code_hash is not set. (It should always be set.)
return pex_file
explode_dir = os.path.join(pex_info.zip_unsafe_cache, pex_info.code_hash)
TRACER.log('PEX is not zip safe, exploding to %s' % explode_dir)
if not os.path.exists(explode_dir):
explode_tmp = explode_dir + '.' + uuid.uuid4().hex
with TRACER.timed('Unzipping %s' % pex_file):
try:
safe_mkdir(explode_tmp)
with open_zip(pex_file) as pex_zip:
pex_files = (x for x in pex_zip.namelist()
if not x.startswith(pex_builder.BOOTSTRAP_DIR) and
not x.startswith(PexInfo.INTERNAL_CACHE))
pex_zip.extractall(explode_tmp, pex_files)
except: # noqa: T803
safe_rmtree(explode_tmp)
raise
TRACER.log('Renaming %s to %s' % (explode_tmp, explode_dir))
rename_if_empty(explode_tmp, explode_dir)
return explode_dir
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
cmd = [
'pip', 'install', '--upgrade', '--no-compile', '--target',
vendor_spec.target_dir, vendor_spec.requirement
]
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
vendor_spec.create_packages()
vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix,
path_items=vendored_path)
for root, dirs, files in os.walk(root_dir):
if root == root_dir:
dirs[:] = ['pex', 'tests']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(
green('Examining {python_file}...'.format(
python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
cmd = ['pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir,
vendor_spec.requirement]
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
# The RECORD contains file hashes of all installed files and is unfortunately unstable in the
# case of scripts which get a shebang added with a system-specific path to the python
# interpreter to execute.
safe_delete(os.path.join(vendor_spec.target_dir,
'{}-{}.dist-info/RECORD'.format(vendor_spec.key, vendor_spec.version)))
vendor_spec.create_packages()
vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix, path_items=vendored_path)
for root, dirs, files in os.walk(root_dir):
if root == root_dir:
dirs[:] = ['pex', 'tests']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(green('Examining {python_file}...'.format(python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(bold(green(' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def cleanup(self): safe_rmtree(self._install_tmp)
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
# NB: We set --no-build-isolation to prevent pip from installing the requirements listed in
# its [build-system] config in its pyproject.toml.
#
# Those requirements are (currently) the unpinned ["setuptools", "wheel"], which will cause pip
# to use the latest version of those packages. This is a hermeticity problem: re-vendoring a
# package at a later time may yield a different result. At the very least the result will
# differ in the version embedded in the WHEEL metadata file, which causes issues with our
# tests.
#
# Setting --no-build-isolation means that versions of setuptools and wheel must be provided
# in the environment in which we run the pip command, which is the environment in which we run
# pex.vendor. Since we document that pex.vendor should be run via tox, that environment will
# contain pinned versions of setuptools and wheel. As a result, vendoring (at least via tox)
# is hermetic.
cmd = [
'pip', 'install', '--upgrade', '--no-build-isolation',
'--no-compile', '--target', vendor_spec.target_dir,
vendor_spec.requirement
]
constraints_file = os.path.join(vendor_spec.target_dir,
'constraints.txt')
if vendor_spec.constrain and os.path.isfile(constraints_file):
cmd.extend(['--constraint', constraints_file])
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
if vendor_spec.constrain:
cmd = ['pip', 'freeze', '--all', '--path', vendor_spec.target_dir]
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
stdout, _ = process.communicate()
if process.returncode != 0:
raise VendorizeError(
'Failed to freeze vendoring of {!r}'.format(vendor_spec))
with open(constraints_file, 'wb') as fp:
fp.write(stdout)
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
# The RECORD contains file hashes of all installed files and is unfortunately unstable in the
# case of scripts which get a shebang added with a system-specific path to the python
# interpreter to execute.
for record_file in glob.glob(
os.path.join(vendor_spec.target_dir, '*-*.dist-info/RECORD')):
safe_delete(record_file)
vendor_spec.create_packages()
vendored_path = [
vendor_spec.target_dir for vendor_spec in vendor_specs
if vendor_spec.rewrite
]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix,
path_items=vendored_path)
rewrite_paths = [os.path.join(root_dir, c)
for c in ('pex', 'tests')] + vendored_path
for rewrite_path in rewrite_paths:
for root, dirs, files in os.walk(rewrite_path):
if root == os.path.join(root_dir, 'pex', 'vendor'):
dirs[:] = [d for d in dirs if d != '_vendored']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(
green('Examining {python_file}...'.format(
python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def test_issues_789_demo():
# type: () -> None
tmpdir = safe_mkdtemp()
pex_project_dir = (subprocess.check_output(
["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip())
# 1. Imagine we've pre-resolved the requirements needed in our wheel house.
requirements = [
"ansicolors",
"isort",
"setuptools", # N.B.: isort doesn't declare its setuptools dependency.
]
wheelhouse = os.path.join(tmpdir, "wheelhouse")
get_pip().spawn_download_distributions(download_dir=wheelhouse,
requirements=requirements).wait()
# 2. Also imagine this configuration is passed to a tool (PEX or a wrapper as in this test
# example) via the CLI or other configuration data sources. For example, Pants has a `PythonSetup`
# that combines with BUILD target data to get you this sort of configuration info outside pex.
resolver_settings = dict(
indexes=[], # Turn off pypi.
find_links=[wheelhouse], # Use our wheel house.
build=False, # Use only pre-built wheels.
) # type: Dict[str, Any]
# 3. That same configuration was used to build a standard pex:
resolver_args = []
if len(resolver_settings["find_links"]) == 0:
resolver_args.append("--no-index")
else:
for index in resolver_settings["indexes"]:
resolver_args.extend(["--index", index])
for repo in resolver_settings["find_links"]:
resolver_args.extend(["--find-links", repo])
resolver_args.append(
"--build" if resolver_settings["build"] else "--no-build")
project_code_dir = os.path.join(tmpdir, "project_code_dir")
with safe_open(os.path.join(project_code_dir, "colorized_isort.py"),
"w") as fp:
fp.write(
dedent("""\
import colors
import os
import subprocess
import sys
def run():
env = os.environ.copy()
env.update(PEX_MODULE='isort')
isort_process = subprocess.Popen(
sys.argv,
env=env,
stdout = subprocess.PIPE,
stderr = subprocess.PIPE
)
stdout, stderr = isort_process.communicate()
print(colors.green(stdout.decode('utf-8')))
print(colors.red(stderr.decode('utf-8')))
sys.exit(isort_process.returncode)
"""))
colorized_isort_pex = os.path.join(tmpdir, "colorized_isort.pex")
args = [
"--sources-directory",
project_code_dir,
"--entry-point",
"colorized_isort:run",
"--output-file",
colorized_isort_pex,
]
result = run_pex_command(args + resolver_args + requirements)
result.assert_success()
# 4. Now the tool builds a "dehydrated" PEX using the standard pex + resolve settings as the
# template.
ptex_cache = os.path.join(tmpdir, ".ptex")
colorized_isort_pex_info = PexInfo.from_pex(colorized_isort_pex)
colorized_isort_pex_info.pex_root = ptex_cache
# Force the standard pex to extract its code. An external tool like Pants would already know the
# orignal source code file paths, but we need to discover here.
colorized_isort_pex_code_dir = os.path.join(
colorized_isort_pex_info.zip_unsafe_cache,
colorized_isort_pex_info.code_hash)
env = os.environ.copy()
env.update(PEX_ROOT=ptex_cache, PEX_INTERPRETER="1", PEX_FORCE_LOCAL="1")
subprocess.check_call([colorized_isort_pex, "-c", ""], env=env)
colorized_isort_ptex_code_dir = os.path.join(
tmpdir, "colorized_isort_ptex_code_dir")
safe_mkdir(colorized_isort_ptex_code_dir)
code = []
for root, dirs, files in os.walk(colorized_isort_pex_code_dir):
rel_root = os.path.relpath(root, colorized_isort_pex_code_dir)
for f in files:
# Don't ship compiled python from the code extract above, the target interpreter will not
# match ours in general.
if f.endswith(".pyc"):
continue
rel_path = os.path.normpath(os.path.join(rel_root, f))
# The root __main__.py is special for any zipapp including pex, let it write its own
# __main__.py bootstrap. Similarly. PEX-INFO is special to pex and we want the PEX-INFO for
# The ptex pex, not the pex being ptexed.
if rel_path in ("__main__.py", PexInfo.PATH):
continue
os.symlink(os.path.join(root, f),
os.path.join(colorized_isort_ptex_code_dir, rel_path))
code.append(rel_path)
ptex_code_dir = os.path.join(tmpdir, "ptex_code_dir")
ptex_info = dict(code=code, resolver_settings=resolver_settings)
with safe_open(os.path.join(ptex_code_dir, "PTEX-INFO"), "w") as fp:
json.dump(ptex_info, fp)
with safe_open(os.path.join(ptex_code_dir, "IPEX-INFO"), "w") as fp:
fp.write(colorized_isort_pex_info.dump())
with safe_open(os.path.join(ptex_code_dir, "ptex.py"), "w") as fp:
fp.write(
dedent("""\
import json
import os
import sys
from pex import resolver
from pex.common import open_zip
from pex.pex_builder import PEXBuilder
from pex.pex_info import PexInfo
from pex.util import CacheHelper
from pex.variables import ENV
self = sys.argv[0]
ipex_file = '{}.ipex'.format(os.path.splitext(self)[0])
if not os.path.isfile(ipex_file):
print('Hydrating {} to {}'.format(self, ipex_file))
ptex_pex_info = PexInfo.from_pex(self)
code_root = os.path.join(ptex_pex_info.zip_unsafe_cache, ptex_pex_info.code_hash)
with open_zip(self) as zf:
# Populate the pex with the pinned requirements and distribution names & hashes.
ipex_info = PexInfo.from_json(zf.read('IPEX-INFO'))
ipex_builder = PEXBuilder(pex_info=ipex_info)
# Populate the pex with the needed code.
ptex_info = json.loads(zf.read('PTEX-INFO').decode('utf-8'))
for path in ptex_info['code']:
ipex_builder.add_source(os.path.join(code_root, path), path)
# Perform a fully pinned intransitive resolve to hydrate the install cache (not the
# pex!).
resolver_settings = ptex_info['resolver_settings']
resolved_distributions = resolver.resolve(
requirements=[str(req) for req in ipex_info.requirements],
cache=ipex_info.pex_root,
transitive=False,
**resolver_settings
)
ipex_builder.build(ipex_file)
os.execv(ipex_file, [ipex_file] + sys.argv[1:])
"""))
colorized_isort_ptex = os.path.join(tmpdir, "colorized_isort.ptex")
result = run_pex_command([
"--not-zip-safe",
"--always-write-cache",
"--pex-root",
ptex_cache,
pex_project_dir, # type: ignore[list-item] # This is unicode in Py2, whereas everthing else is bytes. That's fine.
"--sources-directory",
ptex_code_dir,
"--sources-directory",
colorized_isort_ptex_code_dir,
"--entry-point",
"ptex",
"--output-file",
colorized_isort_ptex,
])
result.assert_success()
subprocess.check_call([colorized_isort_ptex, "--version"])
with pytest.raises(CalledProcessError):
subprocess.check_call([colorized_isort_ptex, "--not-a-flag"])
safe_rmtree(ptex_cache)
# The dehydrated pex now fails since it lost its hydration from the cache.
with pytest.raises(CalledProcessError):
subprocess.check_call([colorized_isort_ptex, "--version"])
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode('utf-8').strip()
if not identity:
raise cls.IdentificationError(
'Could not establish identity of %s' % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode('utf-8'))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters')
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed('GCing interpreter cache from prior OS version'):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
cache_dir = os.path.join(os_cache_dir, interpreter_hash)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, 'rb') as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(['pex'])
cmd, env = cls._create_isolated_cmd(binary,
args=[
'-c',
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE))
],
pythonpath=pythonpath)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of %s" % binary)
return cls(PythonIdentity.decode(identity))
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
path_id = binary.replace(os.sep, ".").lstrip(".")
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(create_interpreter(fp.read()))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get().encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
def supported_tags(self, manylinux=None):
# type: (Optional[str]) -> Tuple[tags.Tag, ...]
# We use a 2 level cache, probing memory first and then a json file on disk in order to
# avoid calculating tags when possible since it's an O(500ms) operation that involves
# spawning Pip.
# Read level 1.
memory_cache_key = (self, manylinux)
supported_tags = self._SUPPORTED_TAGS_BY_PLATFORM.get(memory_cache_key)
if supported_tags is not None:
return supported_tags
# Read level 2.
components = list(attr.astuple(self))
if manylinux:
components.append(manylinux)
disk_cache_key = os.path.join(ENV.PEX_ROOT, "platforms", self.SEP.join(components))
with atomic_directory(target_dir=disk_cache_key, exclusive=False) as cache_dir:
if not cache_dir.is_finalized:
# Missed both caches - spawn calculation.
plat_info = attr.asdict(self)
plat_info.update(
supported_tags=[
(tag.interpreter, tag.abi, tag.platform)
for tag in self._calculate_tags(manylinux=manylinux)
],
)
# Write level 2.
with safe_open(os.path.join(cache_dir.work_dir, self.PLAT_INFO_FILE), "w") as fp:
json.dump(plat_info, fp)
with open(os.path.join(disk_cache_key, self.PLAT_INFO_FILE)) as fp:
try:
data = json.load(fp)
except ValueError as e:
TRACER.log(
"Regenerating the platform info file at {} since it did not contain parsable "
"JSON data: {}".format(fp.name, e)
)
safe_rmtree(disk_cache_key)
return self.supported_tags(manylinux=manylinux)
if not isinstance(data, dict):
TRACER.log(
"Regenerating the platform info file at {} since it did not contain a "
"configuration object. Found: {!r}".format(fp.name, data)
)
safe_rmtree(disk_cache_key)
return self.supported_tags(manylinux=manylinux)
sup_tags = data.get("supported_tags")
if not isinstance(sup_tags, list):
TRACER.log(
"Regenerating the platform info file at {} since it was missing a valid "
"`supported_tags` list. Found: {!r}".format(fp.name, sup_tags)
)
safe_rmtree(disk_cache_key)
return self.supported_tags(manylinux=manylinux)
count = len(sup_tags)
def parse_tag(
index, # type: int
tag, # type: List[Any]
):
# type: (...) -> tags.Tag
if len(tag) != 3 or not all(
isinstance(component, compatibility.string) for component in tag
):
raise ValueError(
"Serialized platform tags should be lists of three strings. Tag {index} of "
"{count} was: {tag!r}.".format(index=index, count=count, tag=tag)
)
interpreter, abi, platform = tag
return tags.Tag(interpreter=interpreter, abi=abi, platform=platform)
try:
supported_tags = tuple(parse_tag(index, tag) for index, tag in enumerate(sup_tags))
# Write level 1.
self._SUPPORTED_TAGS_BY_PLATFORM[memory_cache_key] = supported_tags
return supported_tags
except ValueError as e:
TRACER.log(
"Regenerating the platform info file at {} since it did not contain parsable "
"tag data: {}".format(fp.name, e)
)
safe_rmtree(disk_cache_key)
return self.supported_tags(manylinux=manylinux)
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
" Vendorized {count} import{plural} in {python_file}"
.format(
count=num_mods,
plural="s" if num_mods > 1 else "",
python_file=python_file,
))))
for _from, _to in modifications.items():
print(" {} -> {}".format(_from, _to))
if __name__ == "__main__":
if len(sys.argv) != 1:
print("Usage: {}".format(sys.argv[0]), file=sys.stderr)
sys.exit(1)
root_directory = VendorSpec.ROOT
import_prefix = third_party.import_prefix()
try:
safe_rmtree(VendorSpec.vendor_root())
vendorize(root_directory, list(iter_vendor_specs()), import_prefix)
sys.exit(0)
except VendorizeError as e:
print("Problem encountered vendorizing: {}".format(e), file=sys.stderr)
sys.exit(1)
def temporary_dir():
td = tempfile.mkdtemp()
try:
yield td
finally:
safe_rmtree(td)
def temporary_dir():
td = tempfile.mkdtemp()
try:
yield td
finally:
safe_rmtree(td)
def _spawn_from_binary_external(cls, binary):
def create_interpreter(stdout, check_binary=False):
identity = stdout.decode("utf-8").strip()
if not identity:
raise cls.IdentificationError(
"Could not establish identity of {}.".format(binary))
interpreter = cls(PythonIdentity.decode(identity))
# We should not need to check this since binary == interpreter.binary should always be
# true, but historically this could be untrue as noted in `PythonIdentity.get`.
if check_binary and not os.path.exists(interpreter.binary):
raise cls.InterpreterNotFound(
"Cached interpreter for {} reports a binary of {}, which could not be found"
.format(binary, interpreter.binary))
return interpreter
# Part of the PythonInterpreter data are environment markers that depend on the current OS
# release. That data can change when the OS is upgraded but (some of) the installed interpreters
# remain the same. As such, include the OS in the hash structure for cached interpreters.
os_digest = hashlib.sha1()
for os_identifier in platform.release(), platform.version():
os_digest.update(os_identifier.encode("utf-8"))
os_hash = os_digest.hexdigest()
interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters")
os_cache_dir = os.path.join(interpreter_cache_dir, os_hash)
if os.path.isdir(
interpreter_cache_dir) and not os.path.isdir(os_cache_dir):
with TRACER.timed("GCing interpreter cache from prior OS version"):
safe_rmtree(interpreter_cache_dir)
interpreter_hash = CacheHelper.hash(binary)
# Some distributions include more than one copy of the same interpreter via a hard link (e.g.:
# python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must
# emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and
# would otherwise be unstable.
#
# See cls._REGEXEN for a related affordance.
#
# N.B.: The path for --venv mode interpreters can be quite long; so we just used a fixed
# length hash of the interpreter binary path to ensure uniqueness and not run afoul of file
# name length limits.
path_id = hashlib.sha1(binary.encode("utf-8")).hexdigest()
cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id)
cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE)
if os.path.isfile(cache_file):
try:
with open(cache_file, "rb") as fp:
return SpawnedJob.completed(
create_interpreter(fp.read(), check_binary=True))
except (IOError, OSError, cls.Error, PythonIdentity.Error):
safe_rmtree(cache_dir)
return cls._spawn_from_binary_external(binary)
else:
pythonpath = third_party.expose(["pex"])
cmd, env = cls._create_isolated_cmd(
binary,
args=[
"-c",
dedent("""\
import os
import sys
from pex.common import atomic_directory, safe_open
from pex.interpreter import PythonIdentity
encoded_identity = PythonIdentity.get(binary={binary!r}).encode()
sys.stdout.write(encoded_identity)
with atomic_directory({cache_dir!r}, exclusive=False) as cache_dir:
if cache_dir:
with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp:
fp.write(encoded_identity)
""".format(binary=binary,
cache_dir=cache_dir,
info_file=cls.INTERP_INFO_FILE)),
],
pythonpath=pythonpath,
)
process = Executor.open_process(cmd,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
job = Job(command=cmd, process=process)
return SpawnedJob.stdout(job, result_func=create_interpreter)
