def main():
parser = configure_clp()
options, args = parser.parse_args()
with TraceLogger.env_override(PEX_VERBOSE=options.verbosity):
pex_builder = build_pex(args, options)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
return pex.run(args=list(args))
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main():
parser = configure_clp()
options, args = parser.parse_args()
verbosity = 5 if options.verbosity else -1
with Tracer.env_override(PEX_VERBOSE=verbosity, PEX_HTTP=verbosity):
pex_builder = build_pex(args, options)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), args), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
return pex.run(args=list(args))
def main():
pparser = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
V=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
shutil.move(tmp_name, poptions.pex_name)
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=options.pex_root) as patched_env:
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!',
V=1)
log('Running on platform {} but built for {}'.format(
Platform.current(), ', '.join(map(str,
options.platforms))),
V=1)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
cmd = [
'pip', 'install', '--upgrade', '--no-compile', '--target',
vendor_spec.target_dir, vendor_spec.requirement
]
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
# The RECORD contains file hashes of all installed files and is unfortunately unstable in the
# case of scripts which get a shebang added with a system-specific path to the python
# interpreter to execute.
for record_file in glob.glob(
os.path.join(vendor_spec.target_dir, '*-*.dist-info/RECORD')):
safe_delete(record_file)
vendor_spec.create_packages()
vendored_path = [
vendor_spec.target_dir for vendor_spec in vendor_specs
if vendor_spec.rewrite
]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix,
path_items=vendored_path)
rewrite_paths = [os.path.join(root_dir, c)
for c in ('pex', 'tests')] + vendored_path
for rewrite_path in rewrite_paths:
for root, dirs, files in os.walk(rewrite_path):
if root == os.path.join(root_dir, 'pex', 'vendor'):
dirs[:] = [d for d in dirs if d != '_vendored']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(
green('Examining {python_file}...'.format(
python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze()
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!'
)
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!')
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline)))
def main(args=None):
parser = create_parser()
args = parser.parse_args(args)
if args.build_distro:
if not args.spark_home:
die("No spark home given but building a distribution")
spark_home = os.path.realpath(os.path.abspath(args.spark_home))
if not os.path.exists(spark_home):
die("No spark home given but building a distribution")
spark_name = os.path.basename(spark_home)
args.spark_home = spark_home
args.spark_name = spark_name
else:
spark_home = None
spark_name = None
spex_name = args.spex_name
spex_file = spex_name + '.spex'
with ENV.patch(PEX_VERBOSE=str(args.verbosity)):
with TRACER.timed('Building spex'):
with TRACER.timed('Building pex'):
pex_builder = build_pex(args)
with dump_args_as_config(args) as cfg:
pex_builder.add_resource(cfg, 'SPEX-INFO')
log('Saving PEX file to %s' % spex_file, verbose=args.verbosity)
tmp_name = args.spex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, spex_file)
if args.build_distro:
with TRACER.timed('Building spark package'):
spark_distro = uber_distro_location(spark_name)
establish_spark_distro(spark_distro, spark_home, spark_name, spex_file, spex_name)
log('Spark package built')
with TRACER.timed('Building full distribution'):
create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args)
log('Saved full distribution to %s' % spark_distro)
return 0
def expose(dists):
"""Exposes vendored code in isolated chroots.
Any vendored distributions listed in ``dists`` will be unpacked to individual chroots for addition
to the ``sys.path``; ie: ``expose(['setuptools', 'wheel'])`` will unpack these vendored
distributions and yield the two chroot paths they were unpacked to.
:param dists: A list of vendored distribution names to expose.
:type dists: list of str
:raise: :class:`ValueError` if any distributions to expose cannot be found.
:returns: An iterator of exposed vendored distribution chroot paths.
"""
from pex.common import safe_delete
for path in VendorImporter.expose(dists, root=isolated()):
safe_delete(os.path.join(path, '__init__.py'))
yield path
def expose(dists):
"""Exposes vendored code in isolated chroots.
Any vendored distributions listed in ``dists`` will be unpacked to individual chroots for addition
to the ``sys.path``; ie: ``expose(['setuptools', 'wheel'])`` will unpack these vendored
distributions and yield the two chroot paths they were unpacked to.
:param dists: A list of vendored distribution names to expose.
:type dists: list of str
:raise: :class:`ValueError` if any distributions to expose cannot be found.
:returns: An iterator of exposed vendored distribution chroot paths.
"""
from pex.common import safe_delete
for path in VendorImporter.expose(dists, root=isolated()):
safe_delete(os.path.join(path, '__init__.py'))
yield path
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform and Platform.current() not in options.platform:
log('WARNING: attempting to run PEX with incompatible platforms!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def main(args=None):
args = args or sys.argv[1:]
parser, resolver_options_builder = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.pex_root:
ENV.set('PEX_ROOT', options.pex_root)
else:
options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable.
# Don't alter cache if it is disabled.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
cmd = [
'pip', 'install', '--upgrade', '--no-compile', '--target',
vendor_spec.target_dir, vendor_spec.requirement
]
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
vendor_spec.create_packages()
vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix,
path_items=vendored_path)
for root, dirs, files in os.walk(root_dir):
if root == root_dir:
dirs[:] = ['pex', 'tests']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(
green('Examining {python_file}...'.format(
python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
cmd = ['pip', 'install', '--upgrade', '--no-compile', '--target', vendor_spec.target_dir,
vendor_spec.requirement]
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
# The RECORD contains file hashes of all installed files and is unfortunately unstable in the
# case of scripts which get a shebang added with a system-specific path to the python
# interpreter to execute.
safe_delete(os.path.join(vendor_spec.target_dir,
'{}-{}.dist-info/RECORD'.format(vendor_spec.key, vendor_spec.version)))
vendor_spec.create_packages()
vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix, path_items=vendored_path)
for root, dirs, files in os.walk(root_dir):
if root == root_dir:
dirs[:] = ['pex', 'tests']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(green('Examining {python_file}...'.format(python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(bold(green(' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def main():
parser, resolver_options_builder = configure_clp()
# split arguments early because optparse is dumb
args = sys.argv[1:]
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, resolver_options_builder)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name,
v=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, options.pex_name)
return 0
if options.platform != Platform.current():
log('WARNING: attempting to run PEX with differing platform!')
pex_builder.freeze()
log('Running PEX file at %s with args %s' %
(pex_builder.path(), cmdline),
v=options.verbosity)
pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter)
sys.exit(pex.run(args=list(cmdline)))
def vendorize(root_dir, vendor_specs, prefix):
for vendor_spec in vendor_specs:
# NB: We set --no-build-isolation to prevent pip from installing the requirements listed in
# its [build-system] config in its pyproject.toml.
#
# Those requirements are (currently) the unpinned ["setuptools", "wheel"], which will cause pip
# to use the latest version of those packages. This is a hermeticity problem: re-vendoring a
# package at a later time may yield a different result. At the very least the result will
# differ in the version embedded in the WHEEL metadata file, which causes issues with our
# tests.
#
# Setting --no-build-isolation means that versions of setuptools and wheel must be provided
# in the environment in which we run the pip command, which is the environment in which we run
# pex.vendor. Since we document that pex.vendor should be run via tox, that environment will
# contain pinned versions of setuptools and wheel. As a result, vendoring (at least via tox)
# is hermetic.
cmd = [
'pip', 'install', '--upgrade', '--no-build-isolation',
'--no-compile', '--target', vendor_spec.target_dir,
vendor_spec.requirement
]
constraints_file = os.path.join(vendor_spec.target_dir,
'constraints.txt')
if vendor_spec.constrain and os.path.isfile(constraints_file):
cmd.extend(['--constraint', constraints_file])
result = subprocess.call(cmd)
if result != 0:
raise VendorizeError('Failed to vendor {!r}'.format(vendor_spec))
if vendor_spec.constrain:
cmd = ['pip', 'freeze', '--all', '--path', vendor_spec.target_dir]
process = subprocess.Popen(cmd, stdout=subprocess.PIPE)
stdout, _ = process.communicate()
if process.returncode != 0:
raise VendorizeError(
'Failed to freeze vendoring of {!r}'.format(vendor_spec))
with open(constraints_file, 'wb') as fp:
fp.write(stdout)
# We know we can get these as a by-product of a pip install but never need them.
safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
# The RECORD contains file hashes of all installed files and is unfortunately unstable in the
# case of scripts which get a shebang added with a system-specific path to the python
# interpreter to execute.
for record_file in glob.glob(
os.path.join(vendor_spec.target_dir, '*-*.dist-info/RECORD')):
safe_delete(record_file)
vendor_spec.create_packages()
vendored_path = [
vendor_spec.target_dir for vendor_spec in vendor_specs
if vendor_spec.rewrite
]
import_rewriter = ImportRewriter.for_path_items(prefix=prefix,
path_items=vendored_path)
rewrite_paths = [os.path.join(root_dir, c)
for c in ('pex', 'tests')] + vendored_path
for rewrite_path in rewrite_paths:
for root, dirs, files in os.walk(rewrite_path):
if root == os.path.join(root_dir, 'pex', 'vendor'):
dirs[:] = [d for d in dirs if d != '_vendored']
for f in files:
if f.endswith('.py'):
python_file = os.path.join(root, f)
print(
green('Examining {python_file}...'.format(
python_file=python_file)))
modifications = import_rewriter.rewrite(python_file)
if modifications:
num_mods = len(modifications)
print(
bold(
green(
' Vendorized {count} import{plural} in {python_file}'
.format(count=num_mods,
plural='s' if num_mods > 1 else '',
python_file=python_file))))
for _from, _to in modifications.items():
print(' {} -> {}'.format(_from, _to))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
# Ensure the TMPDIR is an absolute path (So subprocesses that change CWD can find it) and
# that it exists.
tmpdir = os.path.realpath(options.tmpdir)
if not os.path.exists(tmpdir):
die("The specified --tmpdir does not exist: {}".format(tmpdir))
if not os.path.isdir(tmpdir):
die("The specified --tmpdir is not a directory: {}".format(tmpdir))
tempfile.tempdir = os.environ["TMPDIR"] = tmpdir
if options.cache_dir:
pex_warnings.warn(
"The --cache-dir option is deprecated, use --pex-root instead.")
if options.pex_root and options.cache_dir != options.pex_root:
die("Both --cache-dir and --pex-root were passed with conflicting values. "
"Just set --pex-root.")
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn(
"The pex root has been set via {via} but --disable-cache is also set. "
"Ignoring {via} and disabling caches.".format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root("--cache-dir")
elif options.pex_root:
warn_ignore_pex_root("--pex-root")
elif os.environ.get("PEX_ROOT"):
warn_ignore_pex_root("PEX_ROOT")
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.'
)
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=pex_root,
TMPDIR=tmpdir) as patched_env:
with TRACER.timed("Building pex"):
pex_builder = build_pex(reqs, options, cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
interpreter = pex_builder.interpreter
pex = PEX(pex_builder.path(),
interpreter=interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log("Saving PEX file to %s" % options.pex_name,
V=options.verbosity)
tmp_name = options.pex_name + "~"
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time,
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(interpreter,
options.platforms):
log("WARNING: attempting to run PEX with incompatible platforms!",
V=1)
log(
"Running on platform {} but built for {}".format(
interpreter.platform,
", ".join(map(str, options.platforms))),
V=1,
)
log(
"Running PEX file at %s with args %s" %
(pex_builder.path(), cmdline),
V=options.verbosity,
)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [transform_legacy_arg(arg) for arg in args]
parser = configure_clp()
try:
separator = args.index('--')
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
options, reqs = parser.parse_args(args=args)
if options.cache_dir:
pex_warnings.warn('The --cache-dir option is deprecated, use --pex-root instead.')
if options.pex_root and options.cache_dir != options.pex_root:
die('Both --cache-dir and --pex-root were passed with conflicting values. '
'Just set --pex-root.')
if options.disable_cache:
def warn_ignore_pex_root(set_via):
pex_warnings.warn('The pex root has been set via {via} but --disable-cache is also set. '
'Ignoring {via} and disabling caches.'.format(via=set_via))
if options.cache_dir:
warn_ignore_pex_root('--cache-dir')
elif options.pex_root:
warn_ignore_pex_root('--pex-root')
elif os.environ.get('PEX_ROOT'):
warn_ignore_pex_root('PEX_ROOT')
pex_root = safe_mkdtemp()
else:
pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT
if options.python and options.interpreter_constraint:
die('The "--python" and "--interpreter-constraint" options cannot be used together.')
with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root) as patched_env:
with TRACER.timed('Building pex'):
pex_builder = build_pex(reqs, options, cache=ENV.PEX_ROOT)
pex_builder.freeze(bytecode_compile=options.compile)
pex = PEX(pex_builder.path(),
interpreter=pex_builder.interpreter,
verify_entry_point=options.validate_ep)
if options.pex_name is not None:
log('Saving PEX file to %s' % options.pex_name, V=options.verbosity)
tmp_name = options.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(
tmp_name,
bytecode_compile=options.compile,
deterministic_timestamp=not options.use_system_time
)
os.rename(tmp_name, options.pex_name)
else:
if not _compatible_with_current_platform(options.platforms):
log('WARNING: attempting to run PEX with incompatible platforms!', V=1)
log('Running on platform {} but built for {}'
.format(Platform.current(), ', '.join(map(str, options.platforms))), V=1)
log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline),
V=options.verbosity)
sys.exit(pex.run(args=list(cmdline), env=patched_env))
def main():
pparser, resolver_options_builder = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
if poptions.pex_root:
ENV.set('PEX_ROOT', poptions.pex_root)
else:
poptions.pex_root = ENV.PEX_ROOT
if poptions.cache_dir:
poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir)
poptions.interpreter_cache_dir = pexbin.make_relative_to_root(
poptions.interpreter_cache_dir)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions,
resolver_options_builder)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
v=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, poptions.pex_name)
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [pexbin.transform_legacy_arg(arg) for arg in args]
pparser = pexbin.configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
pparser.add_argument(
"--manifest-file",
dest="manifest_file",
default=None,
metavar="FILE",
type=str,
help="pex_wrapper manifest file.",
)
poptions = pparser.parse_args(args=args)
manifest_file = poptions.manifest_file
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
reqs = manifest.get('requirements', [])
requirement_configuration = RequirementConfiguration(requirements=reqs)
try:
resolver_configuration = resolver_options.configure(poptions)
except resolver_options.InvalidConfigurationError as e:
die(str(e))
try:
target_configuration = target_options.configure(poptions)
except target_options.InterpreterNotFound as e:
die(str(e))
except target_options.InterpreterConstraintsNotSatisfied as e:
die(str(e), exit_code=pexbin.CANNOT_SETUP_INTERPRETER)
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(
requirement_configuration=requirement_configuration,
resolver_configuration=resolver_configuration,
target_configuration=target_configuration,
options=poptions)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_source(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
V=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
shutil.move(tmp_name, poptions.pex_name)
