def test_pyenv_shims(self):
# type: () -> None
py35, _, run_pyenv = ensure_python_distribution(PY35)
py36 = ensure_python_interpreter(PY36)
pyenv_root = str(run_pyenv(["root"]).strip())
pyenv_shims = os.path.join(pyenv_root, "shims")
def pyenv_global(*versions):
run_pyenv(["global"] + list(versions))
def assert_shim(shim_name, expected_binary_path):
python = PythonInterpreter.from_binary(
os.path.join(pyenv_shims, shim_name))
assert expected_binary_path == python.binary
with temporary_dir() as pex_root:
with ENV.patch(PEX_ROOT=pex_root) as pex_env:
with environment_as(PYENV_ROOT=pyenv_root, **pex_env):
pyenv_global(PY35, PY36)
assert_shim("python3", py35)
pyenv_global(PY36, PY35)
# The python3 shim is now pointing at python3.6 but the Pex cache has a valid
# entry for the old python3.5 association (the interpreter still exists.)
assert_shim("python3", py35)
# The shim pointer is now invalid since python3.5 was uninstalled and so should
# be re-read.
py35_deleted = "{}.uninstalled".format(py35)
os.rename(py35, py35_deleted)
try:
assert_shim("python3", py36)
finally:
os.rename(py35_deleted, py35)
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def test_make_relative():
with ENV.patch(PEX_ROOT='/pex_root'):
assert '/pex_root/interpreters' == make_relative_to_root(
'{pex_root}/interpreters')
#Verify the user can specify arbitrary absolute paths.
assert '/tmp/interpreters' == make_relative_to_root(
'/tmp/interpreters')
def temporary_pex_root():
with temporary_dir() as pex_root, ENV.patch(PEX_ROOT=os.path.realpath(pex_root)) as env:
original_isolated = third_party._ISOLATED
try:
third_party._ISOLATED = None
yield os.path.realpath(pex_root), env
finally:
third_party._ISOLATED = original_isolated
def main():
pparser = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
V=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
shutil.move(tmp_name, poptions.pex_name)
def temporary_pex_root():
# type: () -> Iterator[Tuple[str, Dict[str, str]]]
with temporary_dir() as pex_root, ENV.patch(
PEX_ROOT=os.path.realpath(pex_root)) as env:
original_isolated = third_party._ISOLATED
try:
third_party._ISOLATED = None
yield os.path.realpath(pex_root), env
finally:
third_party._ISOLATED = original_isolated
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=options.pex_root) as patched_env:
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!',
V=1)
log('Running on platform {} but built for {}'.format(
Platform.current(), ', '.join(map(str,
options.platforms))),
V=1)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze()
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!'
)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def test_project_name_and_version_fallback(tmpdir):
# type: (Any) -> None
def tmp_path(relpath):
# type: (str) -> str
return os.path.join(str(tmpdir), relpath)
expected_metadata_project_name_and_version = ProjectNameAndVersion("foo", "1.2.3")
pkg_info_src = tmp_path("PKG-INFO")
with open(pkg_info_src, "w") as fp:
fp.write("Name: {}\n".format(expected_metadata_project_name_and_version.project_name))
fp.write("Version: {}\n".format(expected_metadata_project_name_and_version.version))
sdist_path = tmp_path("bar-baz-4.5.6.tar.gz")
with tarfile.open(sdist_path, mode="w:gz") as tf:
# N.B.: Valid PKG-INFO at an invalid location.
tf.add(pkg_info_src, arcname="PKG-INFO")
with ENV.patch(PEX_EMIT_WARNINGS="True"), warnings.catch_warnings(record=True) as events:
assert project_name_and_version(sdist_path, fallback_to_filename=False) is None
assert 1 == len(events)
warning = events[0]
assert PEXWarning == warning.category
assert "bar-baz-4.5.6/PKG-INFO" in str(warning.message)
assert ProjectNameAndVersion("bar-baz", "4.5.6") == project_name_and_version(
sdist_path, fallback_to_filename=True
)
name_and_version = "eggs-7.8.9"
pkf_info_path = "{}/PKG-INFO".format(name_and_version)
def write_sdist_tgz(extension):
sdist_path = tmp_path("{}.{}".format(name_and_version, extension))
with tarfile.open(sdist_path, mode="w:gz") as tf:
tf.add(pkg_info_src, arcname=pkf_info_path)
return sdist_path
assert expected_metadata_project_name_and_version == project_name_and_version(
write_sdist_tgz("tar.gz"), fallback_to_filename=False
)
assert expected_metadata_project_name_and_version == project_name_and_version(
write_sdist_tgz("sdist"), fallback_to_filename=False
)
zip_sdist_path = tmp_path("{}.zip".format(name_and_version))
with open_zip(zip_sdist_path, mode="w") as zf:
zf.write(pkg_info_src, arcname=pkf_info_path)
assert expected_metadata_project_name_and_version == project_name_and_version(
zip_sdist_path, fallback_to_filename=False
)
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!')
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def main(args=None):
parser = create_parser()
args = parser.parse_args(args)
if args.build_distro:
if not args.spark_home:
die("No spark home given but building a distribution")
spark_home = os.path.realpath(os.path.abspath(args.spark_home))
if not os.path.exists(spark_home):
die("No spark home given but building a distribution")
spark_name = os.path.basename(spark_home)
args.spark_home = spark_home
args.spark_name = spark_name
else:
spark_home = None
spark_name = None
spex_name = args.spex_name
spex_file = spex_name + '.spex'
with ENV.patch(PEX_VERBOSE=str(args.verbosity)):
with TRACER.timed('Building spex'):
with TRACER.timed('Building pex'):
pex_builder = build_pex(args)
with dump_args_as_config(args) as cfg:
pex_builder.add_resource(cfg, 'SPEX-INFO')
log('Saving PEX file to %s' % spex_file, verbose=args.verbosity)
tmp_name = args.spex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, spex_file)
if args.build_distro:
with TRACER.timed('Building spark package'):
spark_distro = uber_distro_location(spark_name)
establish_spark_distro(spark_distro, spark_home, spark_name, spex_file, spex_name)
log('Spark package built')
with TRACER.timed('Building full distribution'):
create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args)
log('Saved full distribution to %s' % spark_distro)
return 0
def run(self):
name = self.distribution.get_name()
version = self.distribution.get_version()
parser, options_builder = configure_clp()
package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
options, reqs = parser.parse_args(self.pex_args)
if options.entry_point or options.script:
die('Must not specify entry_point or script to --pex-args')
reqs = [package_dir] + reqs
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
pex_builder = build_pex(reqs, options, options_builder)
def split_and_strip(entry_point):
console_script, entry_point = entry_point.split('=', 2)
return console_script.strip(), entry_point.strip()
try:
console_scripts = dict(split_and_strip(script)
for script in self.distribution.entry_points.get('console_scripts', []))
except ValueError:
console_scripts = {}
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
if self.bdist_all:
# Write all entry points into unversioned pex files.
for script_name in console_scripts:
target = os.path.join(self.bdist_dir, script_name)
log.info('Writing %s to %s' % (script_name, target))
self._write(pex_builder, target, script=script_name)
elif name in console_scripts:
# The package has a namesake entry point, so use it.
log.info('Writing %s to %s' % (name, target))
self._write(pex_builder, target, script=name)
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
self._write(pex_builder, target, script=None)
def run(self):
name = self.distribution.get_name()
version = self.distribution.get_version()
parser, options_builder = configure_clp()
package_dir = os.path.dirname(
os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
options, reqs = parser.parse_args(self.pex_args)
# Update cache_dir with pex_root in case this is being called directly.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
if options.entry_point or options.script:
die('Must not specify entry_point or script to --pex-args')
reqs = [package_dir] + reqs
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=options.pex_root):
pex_builder = build_pex(reqs, options, options_builder)
console_scripts = self.parse_entry_points()
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
if self.bdist_all:
# Write all entry points into unversioned pex files.
for script_name in console_scripts:
target = os.path.join(self.bdist_dir, script_name)
log.info('Writing %s to %s' % (script_name, target))
self._write(pex_builder, target, script=script_name)
elif name in console_scripts:
# The package has a namesake entry point, so use it.
log.info('Writing %s to %s' % (name, target))
self._write(pex_builder, target, script=name)
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
self._write(pex_builder, target, script=None)
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform and Platform.current() not in options.platform:
log('WARNING: attempting to run PEX with incompatible platforms!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def run(self):
name = self.distribution.get_name()
version = self.distribution.get_version()
parser, options_builder = configure_clp()
package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
options, reqs = parser.parse_args(self.pex_args)
# Update cache_dir with pex_root in case this is being called directly.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
if options.entry_point or options.script:
die('Must not specify entry_point or script to --pex-args')
reqs = [package_dir] + reqs
with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root):
pex_builder = build_pex(reqs, options, options_builder)
console_scripts = self.parse_entry_points()
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
if self.bdist_all:
# Write all entry points into unversioned pex files.
for script_name in console_scripts:
target = os.path.join(self.bdist_dir, script_name)
log.info('Writing %s to %s' % (script_name, target))
self._write(pex_builder, target, script=script_name)
elif name in console_scripts:
# The package has a namesake entry point, so use it.
log.info('Writing %s to %s' % (name, target))
self._write(pex_builder, target, script=name)
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
self._write(pex_builder, target, script=None)
def test_identify_cwd_isolation_issues_1231(tmpdir):
# type: (Any) -> None
python36, pip = ensure_python_venv(PY36)
polluted_cwd = os.path.join(str(tmpdir), "dir")
subprocess.check_call(
args=[pip, "install", "--target", polluted_cwd, "pex==2.1.16"])
pex_root = os.path.join(str(tmpdir), "pex_root")
with pushd(polluted_cwd), ENV.patch(PEX_ROOT=pex_root):
interp = PythonInterpreter.from_binary(python36)
interp_info_files = {
os.path.join(root, f)
for root, _, files in os.walk(pex_root) for f in files
if f == PythonInterpreter.INTERP_INFO_FILE
}
assert 1 == len(interp_info_files)
with open(interp_info_files.pop()) as fp:
assert interp.binary == json.load(fp)["binary"]
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def bootstrap_pex(entry_point):
# type: (str) -> None
pex_info = _bootstrap(entry_point)
# ENV.PEX_ROOT is consulted by PythonInterpreter and Platform so set that up as early as
# possible in the run.
with ENV.patch(PEX_ROOT=pex_info.pex_root):
if not ENV.PEX_TOOLS and pex_info.venv:
try:
target = find_compatible_interpreter(
interpreter_constraints=pex_info.interpreter_constraints, )
except UnsatisfiableInterpreterConstraintsError as e:
die(str(e))
from . import pex
venv_pex = ensure_venv(pex.PEX(entry_point, interpreter=target))
os.execv(venv_pex, [venv_pex] + sys.argv[1:])
else:
maybe_reexec_pex(pex_info.interpreter_constraints)
from . import pex
pex.PEX(entry_point).execute()
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main():
pparser, resolver_options_builder = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
if poptions.pex_root:
ENV.set('PEX_ROOT', poptions.pex_root)
else:
poptions.pex_root = ENV.PEX_ROOT
if poptions.cache_dir:
poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir)
poptions.interpreter_cache_dir = pexbin.make_relative_to_root(
poptions.interpreter_cache_dir)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions,
resolver_options_builder)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
v=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, poptions.pex_name)
def test_make_relative():
with ENV.patch(PEX_ROOT='/pex_root'):
assert '/pex_root/interpreters' == make_relative_to_root('{pex_root}/interpreters')
#Verify the user can specify arbitrary absolute paths.
assert '/tmp/interpreters' == make_relative_to_root('/tmp/interpreters')
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options = parser.parse_args(args=args)
# Ensure the TMPDIR is an absolute path (So subprocesses that change CWD can find it) and
# that it exists.
tmpdir = os.path.realpath(options.tmpdir)
if not os.path.exists(tmpdir):
die("The specified --tmpdir does not exist: {}".format(tmpdir))
if not os.path.isdir(tmpdir):
die("The specified --tmpdir is not a directory: {}".format(tmpdir))
tempfile.tempdir = os.environ["TMPDIR"] = tmpdir
if options.cache_dir:
pex_warnings.warn(
"The --cache-dir option is deprecated, use --pex-root instead.")
if options.pex_root and options.cache_dir != options.pex_root:
die("Both --cache-dir and --pex-root were passed with conflicting values. "
"Just set --pex-root.")
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn(
"The pex root has been set via {via} but --disable-cache is also set. "
"Ignoring {via} and disabling caches.".format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root("--cache-dir")
elif options.pex_root:
warn_ignore_pex_root("--pex-root")
elif os.environ.get("PEX_ROOT"):
warn_ignore_pex_root("PEX_ROOT")
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_repository and (options.indexes or options.find_links):
die('The "--pex-repository" option cannot be used together with the "--index" or '
'"--find-links" options.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=pex_root,
TMPDIR=tmpdir) as patched_env:
with TRACER.timed("Building pex"):
pex_builder = build_pex(options.requirements,
options,
cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
interpreter = pex_builder.interpreter
pex = PEX(pex_builder.path(),
interpreter=interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log("Saving PEX file to %s" % options.pex_name,
V=options.verbosity)
pex_builder.build(
options.pex_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time,
)
if options.seed != Seed.NONE:
seed_info = seed_cache(options,
pex,
verbose=options.seed == Seed.VERBOSE)
print(seed_info)
else:
if not _compatible_with_current_platform(interpreter,
options.platforms):
log("WARNING: attempting to run PEX with incompatible platforms!",
V=1)
log(
"Running on platform {} but built for {}".format(
interpreter.platform,
", ".join(map(str, options.platforms))),
V=1,
)
log(
"Running PEX file at %s with args %s" %
(pex_builder.path(), cmdline),
V=options.verbosity,
)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def test_pyenv_shims(self, tmpdir):
# type: (Any) -> None
py35, _, run_pyenv = ensure_python_distribution(PY35)
py36 = ensure_python_interpreter(PY36)
pyenv_root = str(run_pyenv(["root"]).strip())
pyenv_shims = os.path.join(pyenv_root, "shims")
def pyenv_global(*versions):
# type: (*str) -> None
run_pyenv(["global"] + list(versions))
def pyenv_local(*versions):
# type: (*str) -> None
run_pyenv(["local"] + list(versions))
@contextmanager
def pyenv_shell(*versions):
# type: (*str) -> Iterator[None]
with environment_as(PYENV_VERSION=":".join(versions)):
yield
pex_root = os.path.join(str(tmpdir), "pex_root")
cwd = safe_mkdir(os.path.join(str(tmpdir), "home", "jake", "project"))
with ENV.patch(PEX_ROOT=pex_root) as pex_env, environment_as(
PYENV_ROOT=pyenv_root, PEX_PYTHON_PATH=pyenv_shims, **pex_env
), pyenv_shell(), pushd(cwd):
pyenv = Pyenv.find()
assert pyenv is not None
assert pyenv_root == pyenv.root
def interpreter_for_shim(shim_name):
# type: (str) -> PythonInterpreter
binary = os.path.join(pyenv_shims, shim_name)
return PythonInterpreter.from_binary(binary, pyenv=pyenv)
def assert_shim(
shim_name, # type: str
expected_binary_path, # type: str
):
# type: (...) -> None
python = interpreter_for_shim(shim_name)
assert expected_binary_path == python.binary
def assert_shim_inactive(shim_name):
# type: (str) -> None
with pytest.raises(PythonInterpreter.IdentificationError):
interpreter_for_shim(shim_name)
pyenv_global(PY35, PY36)
assert_shim("python", py35)
assert_shim("python3", py35)
assert_shim("python3.5", py35)
assert_shim("python3.6", py36)
pyenv_global(PY36, PY35)
assert_shim("python", py36)
assert_shim("python3", py36)
assert_shim("python3.6", py36)
assert_shim("python3.5", py35)
pyenv_local(PY35)
assert_shim("python", py35)
assert_shim("python3", py35)
assert_shim("python3.5", py35)
assert_shim_inactive("python3.6")
with pyenv_shell(PY36):
assert_shim("python", py36)
assert_shim("python3", py36)
assert_shim("python3.6", py36)
assert_shim_inactive("python3.5")
with pyenv_shell(PY35, PY36):
assert_shim("python", py35)
assert_shim("python3", py35)
assert_shim("python3.5", py35)
assert_shim("python3.6", py36)
# The shim pointer is now invalid since python3.5 was uninstalled and so
# should be re-read and found invalid.
py35_version_dir = os.path.dirname(os.path.dirname(py35))
py35_deleted = "{}.uninstalled".format(py35_version_dir)
os.rename(py35_version_dir, py35_deleted)
try:
assert_shim_inactive("python")
assert_shim_inactive("python3")
assert_shim_inactive("python3.5")
finally:
os.rename(py35_deleted, py35_version_dir)
assert_shim("python", py35)
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.cache_dir:
pex_warnings.warn('The --cache-dir option is deprecated, use --pex-root instead.')
if options.pex_root and options.cache_dir != options.pex_root:
die('Both --cache-dir and --pex-root were passed with conflicting values. '
'Just set --pex-root.')
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn('The pex root has been set via {via} but --disable-cache is also set. '
'Ignoring {via} and disabling caches.'.format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root('--cache-dir')
elif options.pex_root:
warn_ignore_pex_root('--pex-root')
elif os.environ.get('PEX_ROOT'):
warn_ignore_pex_root('PEX_ROOT')
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root) as patched_env:
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!', V=1)
log('Running on platform {} but built for {}'
.format(Platform.current(), ', '.join(map(str, options.platforms))), V=1)
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [pexbin.transform_legacy_arg(arg) for arg in args]
pparser = pexbin.configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
pparser.add_argument(
"--manifest-file",
dest="manifest_file",
default=None,
metavar="FILE",
type=str,
help="pex_wrapper manifest file.",
)
poptions = pparser.parse_args(args=args)
manifest_file = poptions.manifest_file
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
reqs = manifest.get('requirements', [])
requirement_configuration = RequirementConfiguration(requirements=reqs)
try:
resolver_configuration = resolver_options.configure(poptions)
except resolver_options.InvalidConfigurationError as e:
die(str(e))
try:
target_configuration = target_options.configure(poptions)
except target_options.InterpreterNotFound as e:
die(str(e))
except target_options.InterpreterConstraintsNotSatisfied as e:
die(str(e), exit_code=pexbin.CANNOT_SETUP_INTERPRETER)
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(
requirement_configuration=requirement_configuration,
resolver_configuration=resolver_configuration,
target_configuration=target_configuration,
options=poptions)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_source(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
V=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
shutil.move(tmp_name, poptions.pex_name)
