def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
resolveds = requirement_resolver(
options.requirements,
obtainer=build_obtainer(options),
interpreter=interpreter,
platform=options.platform)
if resolveds:
log('Resolved distributions:', v=options.verbosity)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
bdist = installer(source_dir).bdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_dist_location(bdist)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None):
"""Write a pex file that contains an executable entry point
:param td: temporary directory path
:param exe_contents: entry point python file
:type exe_contents: string
:param dists: distributions to include, typically sdists or bdists
:param sources: sources to include, as a list of pairs (env_filename, contents)
:param coverage: include coverage header
:param interpreter: a custom interpreter to use to build the pex
"""
dists = dists or []
sources = sources or []
safe_mkdir(td)
with open(os.path.join(td, 'exe.py'), 'w') as fp:
fp.write(exe_contents)
pb = PEXBuilder(path=td,
preamble=COVERAGE_PREAMBLE if coverage else None,
interpreter=interpreter)
for dist in dists:
pb.add_dist_location(dist.location)
for env_filename, contents in sources:
src_path = os.path.join(td, env_filename)
safe_mkdir(os.path.dirname(src_path))
with open(src_path, 'w') as fp:
fp.write(contents)
pb.add_source(src_path, env_filename)
pb.set_executable(os.path.join(td, 'exe.py'))
pb.freeze()
return pb
def create(cls, path):
# type: (str) -> Pip
"""Creates a pip tool with PEX isolation at path.
:param path: The path to build the pip tool pex at.
"""
pip_pex_path = os.path.join(path, isolated().pex_hash)
with atomic_directory(pip_pex_path, exclusive=True) as chroot:
if chroot is not None:
from pex.pex_builder import PEXBuilder
isolated_pip_builder = PEXBuilder(path=chroot)
for dist_location in third_party.expose(["pip", "setuptools", "wheel"]):
isolated_pip_builder.add_dist_location(dist=dist_location)
with open(os.path.join(isolated_pip_builder.path(), "run_pip.py"), "w") as fp:
fp.write(
dedent(
"""\
import os
import runpy
import sys
# Propagate un-vendored setuptools to pip for any legacy setup.py builds
# it needs to perform.
os.environ['__PEX_UNVENDORED__'] = '1'
os.environ['PYTHONPATH'] = os.pathsep.join(sys.path)
runpy.run_module('pip', run_name='__main__')
"""
)
)
isolated_pip_builder.set_executable(fp.name)
isolated_pip_builder.freeze()
return cls(pip_pex_path)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(
options.python,
PythonInterpreter.from_binary(options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def yield_pex_builder(zip_safe=True, interpreter=None):
# type: (bool, Optional[PythonInterpreter]) -> Iterator[PEXBuilder]
with temporary_dir() as td, make_bdist("p1", zip_safe=zip_safe, interpreter=interpreter) as p1:
pb = PEXBuilder(path=td, interpreter=interpreter)
pb.add_dist_location(p1.location)
yield pb
class PexBuilderWrapper:
"""Wraps PEXBuilder to provide an API that consumes targets and other BUILD file entities."""
class Factory(Subsystem):
options_scope = "pex-builder-wrapper"
@classmethod
def register_options(cls, register):
super(PexBuilderWrapper.Factory, cls).register_options(register)
# TODO: make an analogy to cls.register_jvm_tool that can be overridden for python subsystems
# by a python_requirement_library() target, not just via pants.ini!
register(
"--setuptools-version",
advanced=True,
default="40.6.3",
fingerprint=True,
help=
"The setuptools version to include in the pex if namespace packages need "
"to be injected.",
)
register(
"--pex-version",
advanced=True,
default=pex_version,
fingerprint=True,
help="The pex version to include in any generated ipex files. "
"NOTE: This should ideally be the same as the pex version which pants "
f"itself depends on, which right now is {pex_version}.",
)
@classmethod
def subsystem_dependencies(cls):
return super(PexBuilderWrapper.Factory,
cls).subsystem_dependencies() + (
PythonRepos,
PythonSetup,
)
@classmethod
def create(cls, builder, log=None, generate_ipex=False):
options = cls.global_instance().get_options()
setuptools_requirement = f"setuptools=={options.setuptools_version}"
pex_requirement = f"pex=={options.pex_version}"
log = log or logging.getLogger(__name__)
return PexBuilderWrapper(
builder=builder,
python_repos_subsystem=PythonRepos.global_instance(),
python_setup_subsystem=PythonSetup.global_instance(),
setuptools_requirement=PythonRequirement(
setuptools_requirement),
pex_requirement=PythonRequirement(pex_requirement),
log=log,
generate_ipex=generate_ipex,
)
def __init__(
self,
builder: PEXBuilder,
python_repos_subsystem: PythonRepos,
python_setup_subsystem: PythonSetup,
setuptools_requirement: PythonRequirement,
pex_requirement: PythonRequirement,
log,
generate_ipex: bool = False,
):
assert log is not None
self._builder = builder
self._python_repos_subsystem = python_repos_subsystem
self._python_setup_subsystem = python_setup_subsystem
self._setuptools_requirement = setuptools_requirement
self._pex_requirement = pex_requirement
self._log = log
self._distributions: Dict[str, Distribution] = {}
self._frozen = False
self._generate_ipex = generate_ipex
# If we generate a .ipex, we need to ensure all the code we copy into the underlying PEXBuilder
# is also added to the new PEXBuilder created in `._shuffle_original_build_info_into_ipex()`.
self._all_added_sources_resources: List[Path] = []
# If we generate a dehydrated "ipex" file, we need to make sure that it is aware of any special
# find_links repos attached to any single requirement, so it can later resolve those
# requirements when it is first bootstrapped, using the same resolve options.
self._all_find_links: OrderedSet[str] = OrderedSet()
def add_requirement_libs_from(self, req_libs, platforms=None):
"""Multi-platform dependency resolution for PEX files.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param req_libs: A list of :class:`PythonRequirementLibrary` targets to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
"""
reqs = [req for req_lib in req_libs for req in req_lib.requirements]
self.add_resolved_requirements(reqs, platforms=platforms)
class SingleDistExtractionError(Exception):
pass
def extract_single_dist_for_current_platform(self, reqs,
dist_key) -> Distribution:
"""Resolve a specific distribution from a set of requirements matching the current platform.
:param list reqs: A list of :class:`PythonRequirement` to resolve.
:param str dist_key: The value of `distribution.key` to match for a `distribution` from the
resolved requirements.
:return: The single :class:`pkg_resources.Distribution` matching `dist_key`.
:raises: :class:`self.SingleDistExtractionError` if no dists or multiple dists matched the
given `dist_key`.
"""
distributions = self.resolve_distributions(reqs, platforms=["current"])
try:
matched_dist = assert_single_element(
dist for dists in distributions.values() for dist in dists
if dist.key == dist_key)
except (StopIteration, ValueError) as e:
raise self.SingleDistExtractionError(
f"Exactly one dist was expected to match name {dist_key} in requirements {reqs}: {e!r}"
)
return matched_dist
def resolve_distributions(
self,
reqs: List[PythonRequirement],
platforms: Optional[List[Platform]] = None,
) -> Dict[str, List[Distribution]]:
"""Multi-platform dependency resolution.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param platforms: A list of platform strings to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
:returns: A tuple `(map, transitive_reqs)`, where `map` is a dict mapping distribution name
to a list of resolved distributions, and `reqs` contains all transitive ==
requirements
needed to resolve the initial given requirements `reqs` for the given platforms.
"""
deduped_reqs = OrderedSet(reqs)
find_links: OrderedSet[str] = OrderedSet()
for req in deduped_reqs:
self._log.debug(f" Dumping requirement: {req}")
self._builder.add_requirement(str(req.requirement))
if req.repository:
find_links.add(req.repository)
# Resolve the requirements into distributions.
distributions = self._resolve_multi(
self._builder.interpreter,
list(deduped_reqs),
platforms,
list(find_links),
)
return distributions
def add_resolved_requirements(
self,
reqs: List[PythonRequirement],
platforms: Optional[List[Platform]] = None,
override_ipex_build_do_actually_add_distribution: bool = False,
) -> None:
"""Multi-platform dependency resolution for PEX files.
:param builder: Dump the requirements into this builder.
:param interpreter: The :class:`PythonInterpreter` to resolve requirements for.
:param reqs: A list of :class:`PythonRequirement` to resolve.
:param log: Use this logger.
:param platforms: A list of :class:`Platform`s to resolve requirements for.
Defaults to the platforms specified by PythonSetup.
:param bool override_ipex_build_do_actually_add_distribution: When this PexBuilderWrapper is configured with
generate_ipex=True, this method won't add any distributions to
the output pex. The internal implementation of this class adds a
pex dependency to the output ipex file, and therefore needs to
override the default behavior of this method.
"""
distributions = self.resolve_distributions(reqs, platforms=platforms)
locations: Set[str] = set()
for platform, dists in distributions.items():
for dist in dists:
if dist.location not in locations:
if self._generate_ipex and not override_ipex_build_do_actually_add_distribution:
self._log.debug(
f" *AVOIDING* dumping distribution into ipex: .../{os.path.basename(dist.location)}"
)
self._register_distribution(dist)
else:
self._log.debug(
f" Dumping distribution: .../{os.path.basename(dist.location)}"
)
self.add_distribution(dist)
locations.add(dist.location)
def _resolve_multi(
self,
interpreter: PythonInterpreter,
requirements: List[PythonRequirement],
platforms: Optional[List[Platform]],
find_links: Optional[List[str]],
) -> Dict[str, List[Distribution]]:
"""Multi-platform dependency resolution for PEX files.
Returns a tuple containing a list of distributions that must be included in order to satisfy a
set of requirements, and the transitive == requirements for those distributions. This may
involve distributions for multiple platforms.
:param interpreter: The :class:`PythonInterpreter` to resolve for.
:param requirements: A list of :class:`PythonRequirement` objects to resolve.
:param platforms: A list of :class:`Platform`s to resolve for.
:param find_links: Additional paths to search for source packages during resolution.
:return: Map of platform name -> list of :class:`pkg_resources.Distribution` instances needed
to satisfy the requirements on that platform.
"""
python_setup = self._python_setup_subsystem
python_repos = self._python_repos_subsystem
platforms = platforms or python_setup.platforms
find_links = list(find_links) if find_links else []
find_links.extend(python_repos.repos)
# Individual requirements from pants may have a `repository` link attached to them, which is
# extracted in `self.resolve_distributions()`. When generating a .ipex file with
# `generate_ipex=True`, we want to ensure these repos are known to the ipex launcher when it
# tries to resolve all the requirements from BOOTSTRAP-PEX-INFO.
self._all_find_links.update(OrderedSet(find_links))
distributions: Dict[str, List[Distribution]] = defaultdict(list)
for platform in platforms:
requirements_cache_dir = os.path.join(
python_setup.resolver_cache_dir, str(interpreter.identity))
resolved_dists = resolve(
requirements=[str(req.requirement) for req in requirements],
interpreter=interpreter,
platform=platform,
indexes=python_repos.indexes,
find_links=find_links,
cache=requirements_cache_dir,
allow_prereleases=python_setup.resolver_allow_prereleases,
manylinux=python_setup.manylinux,
)
for resolved_dist in resolved_dists:
distributions[platform].append(resolved_dist.distribution)
return distributions
def _create_source_dumper(self, tgt: Target) -> Callable[[str], None]:
buildroot = get_buildroot()
def get_chroot_path(relpath: str) -> str:
if type(tgt) == Files:
# Loose `Files`, as opposed to `Resources` or `PythonTarget`s, have no (implied) package
# structure and so we chroot them relative to the build root so that they can be accessed
# via the normal Python filesystem APIs just as they would be accessed outside the
# chrooted environment. NB: This requires we mark the pex as not zip safe so
# these `Files` can still be accessed in the context of a built pex distribution.
self._builder.info.zip_safe = False
return relpath
return str(Path(relpath).relative_to(tgt.target_base))
def dump_source(relpath: str) -> None:
source_path = str(Path(buildroot, relpath))
dest_path = get_chroot_path(relpath)
self._all_added_sources_resources.append(Path(dest_path))
if has_resources(tgt):
self._builder.add_resource(filename=source_path,
env_filename=dest_path)
else:
self._builder.add_source(filename=source_path,
env_filename=dest_path)
return dump_source
def add_sources_from(self, tgt: Target) -> None:
dump_source = self._create_source_dumper(tgt)
self._log.debug(f" Dumping sources: {tgt}")
for relpath in tgt.sources_relative_to_buildroot():
try:
dump_source(relpath)
except OSError:
self._log.error(
f"Failed to copy {relpath} for target {tgt.address.spec}")
raise
if getattr(tgt, "_resource_target_specs", None) or getattr(
tgt, "_synthetic_resources_target", None):
# No one should be on old-style resources any more. And if they are,
# switching to the new python pipeline will be a great opportunity to fix that.
raise TaskError(
f"Old-style resources not supported for target {tgt.address.spec}. Depend on resources() "
"targets instead.")
def _prepare_inits(self) -> Set[str]:
chroot = self._builder.chroot()
sources = chroot.get("source") | chroot.get("resource")
missing_init_files = identify_missing_init_files(sources)
if missing_init_files:
with temporary_file(permissions=0o644) as ns_package:
ns_package.write(
b'__import__("pkg_resources").declare_namespace(__name__) # type: ignore[attr-defined]'
)
ns_package.flush()
for missing_init_file in missing_init_files:
self._all_added_sources_resources.append(
Path(missing_init_file))
self._builder.add_source(filename=ns_package.name,
env_filename=missing_init_file)
return missing_init_files
def set_emit_warnings(self, emit_warnings):
self._builder.info.emit_warnings = emit_warnings
def _set_major_minor_interpreter_constraint_for_ipex(
self,
info: PexInfo,
identity: PythonIdentity,
) -> PexInfo:
interpreter_name = identity.requirement.name
major, minor, _patch = identity.version
major_minor_only_constraint = f"{interpreter_name}=={major}.{minor}.*"
return ipex_launcher.modify_pex_info(
info, interpreter_constraints=[str(major_minor_only_constraint)])
def _shuffle_underlying_pex_builder(self) -> Tuple[PexInfo, Path]:
"""Replace the original builder with a new one, and just pull files from the old chroot."""
# Ensure that (the interpreter selected to resolve requirements when the ipex is first run) is
# (the exact same interpreter we used to resolve those requirements here). This is the only (?)
# way to ensure that the ipex bootstrap uses the *exact* same interpreter version.
self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
self._builder.info, self._builder.interpreter.identity)
# Remove all the original top-level requirements in favor of the transitive == requirements.
self._builder.info = ipex_launcher.modify_pex_info(self._builder.info,
requirements=[])
transitive_reqs = [
dist.as_requirement() for dist in self._distributions.values()
]
self.add_direct_requirements(transitive_reqs)
orig_info = self._builder.info.copy()
orig_chroot = self._builder.chroot()
# Mutate the PexBuilder object which is manipulated by this subsystem.
self._builder = PEXBuilder(interpreter=self._builder.interpreter)
self._builder.info = self._set_major_minor_interpreter_constraint_for_ipex(
self._builder.info, self._builder.interpreter.identity)
self._distributions = {}
return (orig_info, Path(orig_chroot.path()))
def _shuffle_original_build_info_into_ipex(self):
"""Create a "dehydrated" ipex file without any of its requirements, and specify that in two.
*-INFO files.
See ipex_launcher.py for details of how these files are used.
"""
orig_pex_info, orig_chroot = self._shuffle_underlying_pex_builder()
# Gather information needed to create IPEX-INFO.
all_code = [str(src) for src in self._all_added_sources_resources]
prefixed_code_paths = [
os.path.join(ipex_launcher.APP_CODE_PREFIX, src)
for src in all_code
]
for src, prefixed in zip(all_code, prefixed_code_paths):
# NB: Need to add under 'source' label for `self._prepare_inits()` to pick it up!
self._builder.chroot().copy(os.path.join(str(orig_chroot), src),
prefixed,
label="source")
python_repos = self._python_repos_subsystem
python_setup = self._python_setup_subsystem
# NB: self._all_find_links is updated on every call to self._resolve_multi(), and therefore
# includes all of the links from python_repos.repos, as well as any links added within any
# individual requirements from that resolve.
resolver_settings = dict(
indexes=list(python_repos.indexes),
find_links=list(self._all_find_links),
allow_prereleases=UnsetBool.coerce_bool(
python_setup.resolver_allow_prereleases, default=True),
manylinux=python_setup.manylinux,
)
# IPEX-INFO: A json mapping interpreted in ipex_launcher.py:
# {
# "code": [<which source files to add to the "hydrated" pex when bootstrapped>],
# "resolver_settings": {<which indices to search for requirements from when bootstrapping>},
# }
ipex_info = dict(
code=prefixed_code_paths,
resolver_settings=resolver_settings,
)
with temporary_file(permissions=0o644) as ipex_info_file:
ipex_info_file.write(json.dumps(ipex_info).encode())
ipex_info_file.flush()
self._builder.add_resource(filename=ipex_info_file.name,
env_filename="IPEX-INFO")
# BOOTSTRAP-PEX-INFO: The original PEX-INFO, which should be the PEX-INFO in the hydrated .pex
# file that is generated when the .ipex is first executed.
with temporary_file(permissions=0o644) as bootstrap_pex_info_file:
bootstrap_pex_info_file.write(orig_pex_info.dump().encode())
bootstrap_pex_info_file.flush()
self._builder.add_resource(filename=bootstrap_pex_info_file.name,
env_filename="BOOTSTRAP-PEX-INFO")
# ipex.py: The special bootstrap script to hydrate the .ipex with the fully resolved
# requirements when it is first executed.
# Extract the file contents of our custom app launcher script from the pants package.
parent_module = module_dirname(module_dirname(ipex_launcher.__name__))
ipex_launcher_provider = get_provider(parent_module)
ipex_launcher_script = ipex_launcher_provider.get_resource_string(
parent_module, "ipex/ipex_launcher.py")
with temporary_file(permissions=0o644) as ipex_launcher_file:
ipex_launcher_file.write(ipex_launcher_script)
ipex_launcher_file.flush()
# Our .ipex file will use our custom app launcher!
self._builder.set_executable(ipex_launcher_file.name,
env_filename="ipex.py")
# The PEX-INFO we generate shouldn't have any requirements (except pex itself), or they will
# fail to bootstrap because they were unable to find those distributions. Instead, the .pex file
# produced when the .ipex is first executed will read and resolve all those requirements from
# the BOOTSTRAP-PEX-INFO.
self.add_resolved_requirements(
[self._pex_requirement, self._setuptools_requirement],
override_ipex_build_do_actually_add_distribution=True,
)
def freeze(self) -> None:
if self._frozen:
return
if self._prepare_inits():
dist = self._distributions.get("setuptools")
if not dist:
self.add_resolved_requirements([self._setuptools_requirement])
if self._generate_ipex:
self._shuffle_original_build_info_into_ipex()
self._builder.freeze(bytecode_compile=False)
self._frozen = True
def set_entry_point(self, entry_point):
self._builder.set_entry_point(entry_point)
def build(self, safe_path):
self.freeze()
self._builder.build(safe_path,
bytecode_compile=False,
deterministic_timestamp=True)
def set_shebang(self, shebang):
self._builder.set_shebang(shebang)
def add_interpreter_constraint(self, constraint):
self._builder.add_interpreter_constraint(constraint)
def add_interpreter_constraints_from(self, constraint_tgts):
# TODO this would be a great place to validate the constraints and present a good error message
# if they are incompatible because all the sources of the constraints are available.
# See: https://github.com/pantsbuild/pex/blob/584b6e367939d24bc28aa9fa36eb911c8297dac8/pex/interpreter_constraints.py
constraint_tuples = {
self._python_setup_subsystem.compatibility_or_constraints(
tgt.compatibility)
for tgt in constraint_tgts
}
for constraint_tuple in constraint_tuples:
for constraint in constraint_tuple:
self.add_interpreter_constraint(constraint)
def add_direct_requirements(self, reqs):
for req in reqs:
self._builder.add_requirement(str(req))
def add_distribution(self, dist):
self._builder.add_distribution(dist)
self._register_distribution(dist)
def add_dist_location(self, location):
self._builder.add_dist_location(location)
dist = DistributionHelper.distribution_from_path(location)
self._register_distribution(dist)
def _register_distribution(self, dist):
self._distributions[dist.key] = dist
def set_script(self, script):
self._builder.set_script(script)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--directory', action='store_true', default=False)
parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True)
parser.add_option('--python', default='')
parser.add_option('--python-version', default='')
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace('.', ' ').split()
if len(python_version) == 3:
python_version.append('0')
identity = PythonIdentity.from_id_string(' '.join(python_version))
interpreter = PythonInterpreter(
options.python,
identity,
extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--directory', action='store_true', default=False)
parser.add_option('--no-zip-safe',
action='store_false',
dest='zip_safe',
default=True)
parser.add_option('--python', default='')
parser.add_option('--python-version', default='')
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
if not options.python:
options.python = sys.executable
identity = PythonIdentity.get()
elif not options.python_version:
# Note: this is expensive (~500ms). prefer passing --python-version when possible.
identity = PythonInterpreter.from_binary(options.python).identity
else:
# Convert "CPython 2.7" to "CPython 2 7 0"
python_version = options.python_version.replace('.', ' ').split()
if len(python_version) == 3:
python_version.append('0')
identity = PythonIdentity.from_id_string(' '.join(python_version))
interpreter = PythonInterpreter(options.python, identity, extras={})
pex_builder = PEXBuilder(
path=output if options.directory else None,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources` & `_markerlib`.
copy_package(pex_builder,
'pkg_resources',
prefix=pex_builder.BOOTSTRAP_DIR)
copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
if options.directory:
pex_builder.freeze(code_hash=False, bytecode_compile=False)
else:
pex_builder.build(output)
def main():
parser = optparse.OptionParser(usage="usage: %prog [options] output")
parser.add_option('--entry-point', default='__main__')
parser.add_option('--no-zip-safe',
action='store_false',
dest='zip_safe',
default=True)
parser.add_option('--python', default=sys.executable)
parser.add_option('--preload', action='append', default=[])
options, args = parser.parse_args()
if len(args) == 1:
output = args[0]
else:
parser.error("'output' positional argument is required")
return 1
# The manifest is passed via stdin, as it can sometimes get too large
# to be passed as a CLA.
manifest = json.load(sys.stdin)
# Setup a temp dir that the PEX builder will use as its scratch dir.
tmp_dir = tempfile.mkdtemp()
try:
# The version of pkg_resources.py (from setuptools) on some distros is
# too old for PEX. So we keep a recent version in the buck repo and
# force it into the process by constructing a custom PythonInterpreter
# instance using it.
interpreter = PythonInterpreter(options.python,
PythonInterpreter.from_binary(
options.python).identity,
extras={})
pex_builder = PEXBuilder(
path=tmp_dir,
interpreter=interpreter,
)
# Set whether this PEX as zip-safe, meaning everything will stayed zipped up
# and we'll rely on python's zip-import mechanism to load modules from
# the PEX. This may not work in some situations (e.g. native
# libraries, libraries that want to find resources via the FS).
pex_builder.info.zip_safe = options.zip_safe
# Set the starting point for this PEX.
pex_builder.info.entry_point = options.entry_point
# Copy in our version of `pkg_resources`.
copy_package(pex_builder,
'pkg_resources',
prefix=pex_builder.BOOTSTRAP_DIR)
# Add the sources listed in the manifest.
for dst, src in manifest['modules'].iteritems():
# NOTE(agallagher): calls the `add_source` and `add_resource` below
# hard-link the given source into the PEX temp dir. Since OS X and
# Linux behave different when hard-linking a source that is a
# symbolic link (Linux does *not* follow symlinks), resolve any
# layers of symlinks here to get consistent behavior.
try:
pex_builder.add_source(dereference_symlinks(src), dst)
except OSError as e:
raise Exception("Failed to add {}: {}".format(src, e))
# Add resources listed in the manifest.
for dst, src in manifest['resources'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Add prebuilt libraries listed in the manifest.
for req in manifest.get('prebuiltLibraries', []):
try:
pex_builder.add_dist_location(req)
except Exception as e:
raise Exception("Failed to add {}: {}".format(req, e))
# Add resources listed in the manifest.
for dst, src in manifest['nativeLibraries'].iteritems():
# NOTE(agallagher): see rationale above.
pex_builder.add_resource(dereference_symlinks(src), dst)
# Generate the PEX file.
pex_builder.build(output)
# Always try cleaning up the scratch dir, ignoring failures.
finally:
shutil.rmtree(tmp_dir, True)
def build_pex(args, options):
interpreter = interpreter_from_options(options)
pex_builder = PEXBuilder(
path=safe_mkdtemp(),
interpreter=interpreter,
)
pex_info = pex_builder.info
pex_info.zip_safe = options.zip_safe
pex_info.always_write_cache = options.always_write_cache
pex_info.ignore_errors = options.ignore_errors
pex_info.inherit_path = options.inherit_path
installer = WheelInstaller if options.use_wheel else EggInstaller
interpreter = interpreter_from_options(options)
fetchers = [Fetcher(options.repos)]
if options.pypi:
fetchers.append(PyPIFetcher())
if options.indices:
fetchers.extend(PyPIFetcher(index) for index in options.indices)
translator = translator_from_options(options)
if options.use_wheel:
precedence = (WheelPackage, EggPackage, SourcePackage)
else:
precedence = (EggPackage, SourcePackage)
with TRACER.timed('Resolving distributions'):
resolveds = requirement_resolver(
options.requirements,
fetchers=fetchers,
translator=translator,
interpreter=interpreter,
platform=options.platform,
precedence=precedence,
cache=options.cache_dir,
cache_ttl=options.cache_ttl)
for pkg in resolveds:
log(' %s' % pkg, v=options.verbosity)
pex_builder.add_distribution(pkg)
pex_builder.add_requirement(pkg.as_requirement())
for source_dir in options.source_dirs:
try:
bdist = installer(source_dir).bdist()
except installer.Error:
die('Failed to run installer for %s' % source_dir, CANNOT_DISTILL)
pex_builder.add_dist_location(bdist)
if options.entry_point is not None:
log('Setting entry point to %s' % options.entry_point, v=options.verbosity)
pex_builder.info.entry_point = options.entry_point
else:
log('Creating environment PEX.', v=options.verbosity)
return pex_builder
