def test_clp_requirements_txt():
parser = configure_clp()
options, _ = parser.parse_args(
args='-r requirements1.txt -r requirements2.txt'.split())
assert options.requirement_files == [
'requirements1.txt', 'requirements2.txt'
]
def run(self):
parser, options_builder = configure_clp()
options, reqs = parser.parse_args(self.pex_args)
if options.entry_point or options.script or options.pex_name:
die('Must not specify entry point, script or output file to --pex-args, given: {}'
.format(' '.join(self.pex_args)))
name = self.distribution.get_name()
version = self.distribution.get_version()
package_dir = os.path.dirname(
os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
console_scripts = self.parse_entry_points()
pex_specs = []
if self.bdist_all:
# Write all entry points into unversioned pex files.
pex_specs.extend(
(script_name, os.path.join(self.bdist_dir, script_name))
for script_name in console_scripts)
else:
target = os.path.join(self.bdist_dir,
name + '-' + version + '.pex')
pex_specs.append(
(name if name in console_scripts else None, target))
# In order for code to run to here, pex is on the sys.path - make sure to propagate the
# sys.path so the subprocess can find us.
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir
] + reqs + self.pex_args
if self.get_log_level() < log.INFO and options.verbosity == 0:
args.append('-v')
for script_name, target in pex_specs:
cmd = args + ['--output-file', target]
if script_name:
log.info('Writing %s to %s' % (script_name, target))
cmd += ['--script', script_name]
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
log.debug('Building pex via: {}'.format(' '.join(cmd)))
process = Executor.open_process(cmd,
stderr=subprocess.PIPE,
env=env)
_, stderr = process.communicate()
result = process.returncode
if result != 0:
die(
'Failed to create pex via {}:\n{}'.format(
' '.join(cmd), stderr.decode('utf-8')), result)
def run(self):
parser = configure_clp()
options, reqs = parser.parse_args(self.pex_args)
if options.entry_point or options.script or options.pex_name:
die("Must not specify entry point, script or output file to --pex-args, given: {}"
.format(" ".join(self.pex_args)))
name = self.distribution.get_name()
version = self.distribution.get_version()
package_dir = os.path.dirname(
os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, "dist")
console_scripts = self.parse_entry_points()
pex_specs = []
if self.bdist_all:
# Write all entry points into unversioned pex files.
pex_specs.extend(
(script_name, os.path.join(self.bdist_dir, script_name))
for script_name in console_scripts)
else:
target = os.path.join(self.bdist_dir,
name + "-" + version + ".pex")
pex_specs.append(
(name if name in console_scripts else None, target))
args = ["-m", "pex", package_dir] + reqs + self.pex_args
if self.get_log_level() < log.INFO and options.verbosity == 0:
args.append("-v")
for script_name, target in pex_specs:
pex_cmd = args + ["--output-file", target]
if script_name:
log.info("Writing %s to %s" % (script_name, target))
pex_cmd += ["--script", script_name]
else:
# The package has no namesake entry point, so build an environment pex.
log.info("Writing environment pex into %s" % target)
cmd, process = PythonInterpreter.get().open_process(
args=pex_cmd,
stderr=subprocess.PIPE,
# In order for code to run to here, pex is on the sys.path - make sure to propagate the
# sys.path so the subprocess can find us.
pythonpath=sys.path,
)
_, stderr = process.communicate()
result = process.returncode
if result != 0:
die(
"Failed to create pex via {}:\n{}".format(
" ".join(cmd), stderr.decode("utf-8")),
result,
)
def test_clp_requirements_txt():
# type: () -> None
parser = configure_clp()
options, _ = parser.parse_args(
args="-r requirements1.txt -r requirements2.txt".split())
assert options.requirement_files == [
"requirements1.txt", "requirements2.txt"
]
def test_clp_prereleases_resolver():
# type: () -> None
with nested(
built_wheel(name="prerelease-dep", version="1.2.3b1"),
built_wheel(name="transitive-dep",
install_reqs=["prerelease-dep"]),
built_wheel(name="dep",
install_reqs=["prerelease-dep>=1.2",
"transitive-dep"]),
temporary_dir(),
temporary_dir(),
) as (prerelease_dep, transitive_dep, dep, dist_dir, cache_dir):
for dist in (prerelease_dep, transitive_dep, dep):
safe_copy(dist, os.path.join(dist_dir, os.path.basename(dist)))
parser = configure_clp()
options, reqs = parser.parse_args(args=[
"--no-index",
"--find-links",
dist_dir,
"--cache-dir",
cache_dir, # Avoid dangling {pex_root}.
"--no-pre",
"dep",
])
assert not options.allow_prereleases
with pytest.raises(
SystemExit,
message="Should have failed to resolve prerelease dep"):
build_pex(reqs, options)
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(args=[
"--no-index",
"--find-links",
dist_dir,
"--cache-dir",
cache_dir, # Avoid dangling {pex_root}.
"--pre",
"dep",
])
assert options.allow_prereleases
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
pex_builder = build_pex(reqs, options)
assert pex_builder is not None
assert len(
pex_builder.info.distributions) == 3, "Should have resolved deps"
def test_clp_prereleases_resolver():
prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
with temporary_dir() as td:
safe_copy(prerelease_dep, os.path.join(td, os.path.basename(prerelease_dep)))
fetcher = Fetcher([td])
# When no specific options are specified, allow_prereleases is None
parser, resolver_options_builder = configure_clp()
assert resolver_options_builder._allow_prereleases is None
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(args=['--pre', 'dep==1.2.3b1', 'dep'])
assert resolver_options_builder._allow_prereleases
# We need to use our own fetcher instead of PyPI
resolver_options_builder._fetchers.insert(0, fetcher)
#####
# The resolver created during processing of command line options (configure_clp)
# is not actually passed into the API call (resolve_multi) from build_pex().
# Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
# is created. The only way to supply our own fetcher to that new instance is to patch it
# here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
# below and the change in the `pex.resolver` module where the patched object resides.
#
import pex.resolver
class BuilderWithFetcher(ResolverOptionsBuilder):
def __init__(self,
fetchers=None,
allow_all_external=False,
allow_external=None,
allow_unverified=None,
allow_prereleases=None,
precedence=None,
context=None
):
super(BuilderWithFetcher, self).__init__(fetchers=fetchers,
allow_all_external=allow_all_external,
allow_external=allow_external,
allow_unverified=allow_unverified,
allow_prereleases=allow_prereleases,
precedence=precedence,
context=context)
self._fetchers.insert(0, fetcher)
# end stub
#####
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder', BuilderWithFetcher):
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder is not None
def main():
pparser = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
V=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
shutil.move(tmp_name, poptions.pex_name)
def test_clp_preamble_file():
with NamedTemporaryFile() as tmpfile:
tmpfile.write(to_bytes('print "foo!"'))
tmpfile.flush()
parser = configure_clp()
options, reqs = parser.parse_args(args=["--preamble-file", tmpfile.name])
assert options.preamble_file == tmpfile.name
pex_builder = build_pex(reqs, options)
assert pex_builder._preamble == 'print "foo!"'
def test_clp_arg_file():
# type: () -> None
with NamedTemporaryFile() as tmpfile:
tmpfile.write(to_bytes("-r\nrequirements1.txt\r-r\nrequirements2.txt"))
tmpfile.flush()
parser = configure_clp()
options = parser.parse_args(args=["@" + tmpfile.name])
assert options.requirement_files == [
"requirements1.txt", "requirements2.txt"
]
def test_clp_preamble_file():
with NamedTemporaryFile() as tmpfile:
tmpfile.write(to_bytes('print "foo!"'))
tmpfile.flush()
parser, resolver_options_builder = configure_clp()
options, reqs = parser.parse_args(args=['--preamble-file', tmpfile.name])
assert options.preamble_file == tmpfile.name
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder._preamble == 'print "foo!"'
def _build(self):
target_path = os.path.join(self.install_directory,
self.target.get("executable_name"))
parser, resolver_options_builder = configure_clp()
pex_builder = PEXBuilder(interpreter=self._get_interpreter(parser))
pex_builder.set_entry_point(self.target.get("entry_point"))
for dist in self._get_distributions(resolver_options_builder):
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.build(target_path)
self.directory.symlink_to_bin(self.target.get("executable_name"),
target_path)
def run(self):
parser, options_builder = configure_clp()
options, reqs = parser.parse_args(self.pex_args)
if options.entry_point or options.script or options.pex_name:
die('Must not specify entry point, script or output file to --pex-args, given: {}'
.format(' '.join(self.pex_args)))
name = self.distribution.get_name()
version = self.distribution.get_version()
package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
console_scripts = self.parse_entry_points()
pex_specs = []
if self.bdist_all:
# Write all entry points into unversioned pex files.
pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name))
for script_name in console_scripts)
else:
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
pex_specs.append((name if name in console_scripts else None, target))
# In order for code to run to here, pex is on the sys.path - make sure to propagate the
# sys.path so the subprocess can find us.
env = os.environ.copy()
env['PYTHONPATH'] = os.pathsep.join(sys.path)
args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args
if self.get_log_level() < log.INFO and options.verbosity == 0:
args.append('-v')
for script_name, target in pex_specs:
cmd = args + ['--output-file', target]
if script_name:
log.info('Writing %s to %s' % (script_name, target))
cmd += ['--script', script_name]
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
log.debug('Building pex via: {}'.format(' '.join(cmd)))
process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env)
_, stderr = process.communicate()
result = process.returncode
if result != 0:
die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')),
result)
def _build(self):
target_path = os.path.join(self.install_directory,
self.target.get("executable_name"))
parser, resolver_options_builder = configure_clp()
pex_builder = PEXBuilder(
interpreter=self._get_interpreter(parser)
)
pex_builder.set_entry_point(self.target.get("entry_point"))
for dist in self._get_distributions(resolver_options_builder):
pex_builder.add_distribution(dist)
pex_builder.add_requirement(dist.as_requirement())
pex_builder.build(target_path)
self.directory.symlink_to_bin(self.target.get("executable_name"),
target_path)
def run(self):
name = self.distribution.get_name()
version = self.distribution.get_version()
parser, options_builder = configure_clp()
package_dir = os.path.dirname(
os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
options, reqs = parser.parse_args(self.pex_args)
# Update cache_dir with pex_root in case this is being called directly.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(
options.interpreter_cache_dir)
if options.entry_point or options.script:
die('Must not specify entry_point or script to --pex-args')
reqs = [package_dir] + reqs
with ENV.patch(PEX_VERBOSE=str(options.verbosity),
PEX_ROOT=options.pex_root):
pex_builder = build_pex(reqs, options, options_builder)
console_scripts = self.parse_entry_points()
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
if self.bdist_all:
# Write all entry points into unversioned pex files.
for script_name in console_scripts:
target = os.path.join(self.bdist_dir, script_name)
log.info('Writing %s to %s' % (script_name, target))
self._write(pex_builder, target, script=script_name)
elif name in console_scripts:
# The package has a namesake entry point, so use it.
log.info('Writing %s to %s' % (name, target))
self._write(pex_builder, target, script=name)
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
self._write(pex_builder, target, script=None)
def run(self):
name = self.distribution.get_name()
version = self.distribution.get_version()
parser, options_builder = configure_clp()
package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
options, reqs = parser.parse_args(self.pex_args)
if options.entry_point or options.script:
die('Must not specify entry_point or script to --pex-args')
reqs = [package_dir] + reqs
with ENV.patch(PEX_VERBOSE=str(options.verbosity)):
pex_builder = build_pex(reqs, options, options_builder)
def split_and_strip(entry_point):
console_script, entry_point = entry_point.split('=', 2)
return console_script.strip(), entry_point.strip()
try:
console_scripts = dict(split_and_strip(script)
for script in self.distribution.entry_points.get('console_scripts', []))
except ValueError:
console_scripts = {}
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
if self.bdist_all:
# Write all entry points into unversioned pex files.
for script_name in console_scripts:
target = os.path.join(self.bdist_dir, script_name)
log.info('Writing %s to %s' % (script_name, target))
self._write(pex_builder, target, script=script_name)
elif name in console_scripts:
# The package has a namesake entry point, so use it.
log.info('Writing %s to %s' % (name, target))
self._write(pex_builder, target, script=name)
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
self._write(pex_builder, target, script=None)
def run(self):
name = self.distribution.get_name()
version = self.distribution.get_version()
parser, options_builder = configure_clp()
package_dir = os.path.dirname(os.path.realpath(os.path.expanduser(
self.distribution.script_name)))
if self.bdist_dir is None:
self.bdist_dir = os.path.join(package_dir, 'dist')
options, reqs = parser.parse_args(self.pex_args)
# Update cache_dir with pex_root in case this is being called directly.
if options.cache_dir:
options.cache_dir = make_relative_to_root(options.cache_dir)
options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir)
if options.entry_point or options.script:
die('Must not specify entry_point or script to --pex-args')
reqs = [package_dir] + reqs
with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root):
pex_builder = build_pex(reqs, options, options_builder)
console_scripts = self.parse_entry_points()
target = os.path.join(self.bdist_dir, name + '-' + version + '.pex')
if self.bdist_all:
# Write all entry points into unversioned pex files.
for script_name in console_scripts:
target = os.path.join(self.bdist_dir, script_name)
log.info('Writing %s to %s' % (script_name, target))
self._write(pex_builder, target, script=script_name)
elif name in console_scripts:
# The package has a namesake entry point, so use it.
log.info('Writing %s to %s' % (name, target))
self._write(pex_builder, target, script=name)
else:
# The package has no namesake entry point, so build an environment pex.
log.info('Writing environment pex into %s' % target)
self._write(pex_builder, target, script=None)
def main():
# These are the options that this class will accept from the rule
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True)
parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default="/usr/bin/python2.7")
parser.add_option('--find-links', dest='find_links', default='')
options, args = parser.parse_args()
if len(args) == 2:
output = args[0]
manifest_text = open(args[1], 'r').read()
elif len(args) == 1:
output = args[0]
manifest_text = sys.stdin.read()
else:
parser.error("'output' positional argument is required")
return 1
if manifest_text.startswith('"') and manifest_text.endswith('"'):
manifest_text = manifest_text[1:len(manifest_text) - 1]
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
x.write(manifest_text)
manifest = parse_manifest(manifest_text)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# These are the options that pex will use
pparser, resolver_options_builder = configure_clp()
# Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
# Enabling Trace logging in pex/tracer.py shows this upon failure:
# pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
# is not compatible with CPython-2.7.3 / linux-x86_64
poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
poptions.entry_point = options.entry_point
poptions.find_links = options.find_links
poptions.pypi = options.pypi
poptions.python = options.python
poptions.zip_safe = options.zip_safe
print("pex options: %s" % poptions)
os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python
# The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
# we keep a recent version in and force it into the process by constructing a custom
# PythonInterpreter instance using it.
interpreter = PythonInterpreter(
poptions.python,
PythonInterpreter.from_binary(options.python).identity,
extras={
# TODO: Fix this to resolve automatically
('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg'
})
# resolve setuptools
interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions)
# possibly resolve wheel
if interpreter and poptions.use_wheel:
interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions)
# Add prebuilt libraries listed in the manifest.
reqs = manifest.get('requirements', {}).keys()
if len(reqs) > 0:
print("pex requirements: %s" % reqs)
pex_builder = build_pex(reqs, poptions,
resolver_options_builder, interpreter=interpreter)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
pex_builder.add_source(
dereference_symlinks(pkg_resources_py),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def main():
""" Main """
# Options that this wrapper will accept from the bazel rule
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-pypi', action='store_false',
dest='pypi', default=True)
parser.add_option('--not-zip-safe', action='store_false',
dest='zip_safe', default=True)
parser.add_option('--python', default="python2.7")
parser.add_option('--find-links', dest='find_links', default='')
parser.add_option('--no-use-wheel', action='store_false',
dest='use_wheel', default=True)
parser.add_option('--pex-root', dest='pex_root', default=".pex")
options, args = parser.parse_args()
# The manifest is passed via stdin or a file, as it can sometimes get too
# large to be passed as a CLA.
if len(args) == 2:
output = args[0]
manifest_text = open(args[1], 'r').read()
elif len(args) == 1:
output = args[0]
manifest_text = sys.stdin.read()
else:
parser.error("'output' positional argument is required")
return 1
if manifest_text.startswith('"') and manifest_text.endswith('"'):
manifest_text = manifest_text[1:len(manifest_text) - 1]
manifest = parse_manifest(manifest_text)
# These are the options that pex will use
pparser, resolver_options_builder = configure_clp()
poptions, preqs = pparser.parse_args(sys.argv)
poptions.entry_point = options.entry_point
poptions.find_links = options.find_links
poptions.pypi = options.pypi
poptions.python = options.python
poptions.use_wheel = options.use_wheel
poptions.zip_safe = options.zip_safe
poptions.pex_root = options.pex_root
poptions.cache_dir = options.pex_root + "/build"
poptions.interpreter_cache_dir = options.pex_root + "/interpreters"
# sys.stderr.write("pex options: %s\n" % poptions)
os.environ["PATH"] = os.getenv("PATH",
"%s:/bin:/usr/bin" % poptions.python)
if os.path.exists(options.python):
pybin = poptions.python
else:
pybin = distutils.spawn.find_executable(options.python)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in and force it into the
# process by constructing a custom PythonInterpreter instance using it.
# interpreter = PythonInterpreter.from_binary(pybin,
# [SETUPTOOLS_PATH,
# WHEEL_PATH])
interpreter = PythonInterpreter(
pybin,
PythonInterpreter.from_binary(pybin).identity,
extras={
# TODO: Fix this to resolve automatically
('setuptools', '18.0.1'): SETUPTOOLS_PATH,
# FIXME: I don't think this accomplishes anything at all.
('wheel', '0.23.0'): WHEEL_PATH,
})
# resolve setuptools
interpreter = resolve_or_die(interpreter,
SETUPTOOLS_REQUIREMENT,
poptions)
# possibly resolve wheel
if interpreter and poptions.use_wheel:
interpreter = resolve_or_die(interpreter,
WHEEL_REQUIREMENT,
poptions)
# Add prebuilt libraries listed in the manifest.
reqs = manifest.get('requirements', {}).keys()
# if len(reqs) > 0:
# sys.stderr.write("pex requirements: %s" % reqs)
pex_builder = build_pex(reqs, poptions,
resolver_options_builder,
interpreter=interpreter)
# Set whether this PEX is zip-safe, meaning everything will stay zipped
# up and we'll rely on python's zip-import mechanism to load modules
# from the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
pex_builder.add_source(
dereference_symlinks(PKG_RESOURCES_PATH),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
raise Exception("Failed to add {}: {}".format(src, err))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as err:
raise Exception("Failed to add {}: {}".format(req, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
def test_clp_requirements_txt(): parser, builder = configure_clp() options, _ = parser.parse_args(args='-r requirements1.txt -r requirements2.txt'.split()) assert options.requirement_files == ['requirements1.txt', 'requirements2.txt']
def convert_to_pex(unified_model, output_file_path: str) -> str:
"""
Convert the given unified model into an executable PEX file.
# Arguments
unified_model (UnifiedModel or str): Unified model instance or path to model file
output_file_path (string): Path to save the model.
# Returns
Full path to the converted model
"""
# https://gist.github.com/simeonf/062af826e79259bc7686
log.info("Start conversion to PEX")
if isinstance(unified_model, UnifiedModel):
pass
elif os.path.exists(str(unified_model)):
# load model instance
unified_model = UnifiedModel.load(str(unified_model))
else:
raise Exception("Could not find model for: " + str(unified_model))
from git import Repo
from pex.bin import pex
model_temp_folder = tempfile.mkdtemp()
model_instance_name = "model_instance"
model_instance_folder = os.path.join(model_temp_folder,
model_instance_name)
os.makedirs(model_instance_folder)
with open(os.path.join(model_instance_folder, "setup.py"),
"w") as text_file:
text_file.write("from distutils.core import setup " +
"\nsetup(name='" + model_instance_name + "'," +
"\n\tpackages=['" + model_instance_name + "']," +
"\n\tversion='1.0'," + "\n\tpackage_data={'" +
model_instance_name + "': ['" + model_instance_name +
"']})")
model_instance_package = os.path.join(model_instance_folder,
model_instance_name)
os.makedirs(model_instance_package)
with open(os.path.join(model_instance_package, "__init__.py"),
"w") as text_file:
text_file.write(
"import os, pkg_resources" +
"\nfrom unified_model import cli_handler" +
"\nos.environ[cli_handler.DEFAULT_MODEL_PATH_ENV] = pkg_resources.resource_filename(__name__, '"
+ model_instance_name + "')" + "\ncli_handler.cli()")
with open(os.path.join(model_instance_package, "__main__.py"),
"w") as text_file:
text_file.write("")
unified_model.save(
os.path.join(model_instance_package, model_instance_name))
lib_repo_folder = os.path.join(model_temp_folder, "unified-model")
Repo.clone_from(UNIFIED_MODEL_REPO_URL, lib_repo_folder)
parser, resolver_options_builder = pex.configure_clp()
args = [lib_repo_folder, model_instance_folder, "--disable-cache"]
for req in unified_model._requirements:
if isinstance(req, six.string_types):
args.append(req)
options, reqs = parser.parse_args(args=args)
pex_builder = pex.build_pex(reqs, options, resolver_options_builder)
pex_builder.set_entry_point(model_instance_name)
pex_builder.build(output_file_path)
shutil.rmtree(model_temp_folder)
log.info("Conversion to PEX successful: " + output_file_path)
return output_file_path
def test_clp_prereleases_resolver():
prerelease_dep = make_sdist(name='dep', version='1.2.3b1')
with temporary_dir() as td:
safe_copy(prerelease_dep,
os.path.join(td, os.path.basename(prerelease_dep)))
fetcher = Fetcher([td])
# When no specific options are specified, allow_prereleases is None
parser, resolver_options_builder = configure_clp()
assert resolver_options_builder._allow_prereleases is None
# When we specify `--pre`, allow_prereleases is True
options, reqs = parser.parse_args(
args=['--pre', 'dep==1.2.3b1', 'dep'])
assert resolver_options_builder._allow_prereleases
# We need to use our own fetcher instead of PyPI
resolver_options_builder._fetchers.insert(0, fetcher)
#####
# The resolver created during processing of command line options (configure_clp)
# is not actually passed into the API call (resolve_multi) from build_pex().
# Instead, resolve_multi() calls resolve() where a new ResolverOptionsBuilder instance
# is created. The only way to supply our own fetcher to that new instance is to patch it
# here in the test so that it can fetch our test package (dep-1.2.3b1). Hence, this class
# below and the change in the `pex.resolver` module where the patched object resides.
#
import pex.resolver
class BuilderWithFetcher(ResolverOptionsBuilder):
def __init__(self,
fetchers=None,
allow_all_external=False,
allow_external=None,
allow_unverified=None,
allow_prereleases=None,
use_manylinux=None,
precedence=None,
context=None):
super(BuilderWithFetcher,
self).__init__(fetchers=fetchers,
allow_all_external=allow_all_external,
allow_external=allow_external,
allow_unverified=allow_unverified,
allow_prereleases=allow_prereleases,
use_manylinux=None,
precedence=precedence,
context=context)
self._fetchers.insert(0, fetcher)
# end stub
#####
# Without a corresponding fix in pex.py, this test failed for a dependency requirement of
# dep==1.2.3b1 from one package and just dep (any version accepted) from another package.
# The failure was an exit from build_pex() with the message:
#
# Could not satisfy all requirements for dep==1.2.3b1:
# dep==1.2.3b1, dep
#
# With a correct behavior the assert line is reached and pex_builder object created.
with mock.patch.object(pex.resolver, 'ResolverOptionsBuilder',
BuilderWithFetcher):
pex_builder = build_pex(reqs, options, resolver_options_builder)
assert pex_builder is not None
def test_clp_constraints_txt(): parser, builder = configure_clp() options, _ = parser.parse_args(args='--constraint requirements1.txt'.split()) assert options.constraint_files == ['requirements1.txt']
def main(args=None):
args = args[:] if args else sys.argv[1:]
args = [pexbin.transform_legacy_arg(arg) for arg in args]
pparser = pexbin.configure_clp()
try:
separator = args.index("--")
args, cmdline = args[:separator], args[separator + 1:]
except ValueError:
args, cmdline = args, []
pparser.add_argument(
"--manifest-file",
dest="manifest_file",
default=None,
metavar="FILE",
type=str,
help="pex_wrapper manifest file.",
)
poptions = pparser.parse_args(args=args)
manifest_file = poptions.manifest_file
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
reqs = manifest.get('requirements', [])
requirement_configuration = RequirementConfiguration(requirements=reqs)
try:
resolver_configuration = resolver_options.configure(poptions)
except resolver_options.InvalidConfigurationError as e:
die(str(e))
try:
target_configuration = target_options.configure(poptions)
except target_options.InterpreterNotFound as e:
die(str(e))
except target_options.InterpreterConstraintsNotSatisfied as e:
die(str(e), exit_code=pexbin.CANNOT_SETUP_INTERPRETER)
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity),
PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(
requirement_configuration=requirement_configuration,
resolver_configuration=resolver_configuration,
target_configuration=target_configuration,
options=poptions)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_source(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
V=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
shutil.move(tmp_name, poptions.pex_name)
def main():
# These are the options that this class will accept from the rule
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-pypi',
action='store_false',
dest='pypi',
default=True)
parser.add_option('--disable-cache',
action='store_true',
dest='disable_cache',
default=False)
parser.add_option('--not-zip-safe',
action='store_false',
dest='zip_safe',
default=True)
parser.add_option('--python', default="/usr/bin/python2.7")
parser.add_option('--find-links', dest='find_links', default='')
options, args = parser.parse_args()
if len(args) == 2:
output = args[0]
manifest_text = open(args[1], 'r').read()
elif len(args) == 1:
output = args[0]
manifest_text = sys.stdin.read()
else:
parser.error("'output' positional argument is required")
return 1
if manifest_text.startswith('"') and manifest_text.endswith('"'):
manifest_text = manifest_text[1:len(manifest_text) - 1]
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x:
x.write(manifest_text)
manifest = parse_manifest(manifest_text)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# These are the options that pex will use
pparser, resolver_options_builder = configure_clp()
# Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host.
# Enabling Trace logging in pex/tracer.py shows this upon failure:
# pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl')
# is not compatible with CPython-2.7.3 / linux-x86_64
poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv)
poptions.entry_point = options.entry_point
poptions.find_links = options.find_links
poptions.pypi = options.pypi
poptions.python = options.python
poptions.zip_safe = options.zip_safe
poptions.disable_cache = options.disable_cache
print("pex options: %s" % poptions)
os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python
# The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So
# we keep a recent version in and force it into the process by constructing a custom
# PythonInterpreter instance using it.
interpreter = PythonInterpreter(
poptions.python,
PythonInterpreter.from_binary(options.python).identity,
extras={
# TODO: Fix this to resolve automatically
('setuptools', '18.0.1'):
'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl',
('wheel', '0.23.0'):
'third_party/pex/wheel-0.23.0-py2.7.egg'
})
# resolve setuptools
interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT,
poptions)
# possibly resolve wheel
if interpreter and poptions.use_wheel:
interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT,
poptions)
# Add prebuilt libraries listed in the manifest.
reqs = manifest.get('requirements', {}).keys()
if len(reqs) > 0:
print("pex requirements: %s" % reqs)
pex_builder = build_pex(reqs,
poptions,
resolver_options_builder,
interpreter=interpreter)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
pex_builder.add_source(
dereference_symlinks(pkg_resources_py),
os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py'))
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def test_clp_constraints_txt():
parser = configure_clp()
options, _ = parser.parse_args(args="--constraint requirements1.txt".split())
assert options.constraint_files == ["requirements1.txt"]
def main():
pparser, resolver_options_builder = pexbin.configure_clp()
poptions, args = pparser.parse_args(sys.argv)
manifest_file = args[1]
manifest_text = open(manifest_file, 'r').read()
manifest = parse_manifest(manifest_text)
if poptions.pex_root:
ENV.set('PEX_ROOT', poptions.pex_root)
else:
poptions.pex_root = ENV.PEX_ROOT
if poptions.cache_dir:
poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir)
poptions.interpreter_cache_dir = pexbin.make_relative_to_root(
poptions.interpreter_cache_dir)
reqs = manifest.get('requirements', [])
with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)):
with TRACER.timed('Building pex'):
pex_builder = pexbin.build_pex(reqs, poptions,
resolver_options_builder)
# Add source files from the manifest
for modmap in manifest.get('modules', []):
src = modmap.get('src')
dst = modmap.get('dest')
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as err:
# Maybe we just can't use hardlinks? Try again.
if not pex_builder._copy:
pex_builder._copy = True
pex_builder.add_source(dereference_symlinks(src), dst)
else:
raise RuntimeError("Failed to add %s: %s" % (src, err))
# Add resources from the manifest
for reqmap in manifest.get('resources', []):
src = reqmap.get('src')
dst = reqmap.get('dest')
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add eggs/wheels from the manifest
for egg in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(egg)
except Exception as err:
raise RuntimeError("Failed to add %s: %s" % (egg, err))
# TODO(mikekap): Do something about manifest['nativeLibraries'].
pexbin.log('Saving PEX file to %s' % poptions.pex_name,
v=poptions.verbosity)
tmp_name = poptions.pex_name + '~'
safe_delete(tmp_name)
pex_builder.build(tmp_name)
os.rename(tmp_name, poptions.pex_name)