def test_access_zipped_assets_integration(): test_executable = dedent(''' import os from _pex.util import DistributionHelper temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule') with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp: for line in fp: print(line) ''') with nested(temporary_dir(), temporary_dir()) as (td1, td2): pb = PEXBuilder(path=td1) with open(os.path.join(td1, 'exe.py'), 'w') as fp: fp.write(test_executable) pb.set_executable(fp.name) submodule = os.path.join(td1, 'my_package', 'submodule') safe_mkdir(submodule) mod_path = os.path.join(submodule, 'mod.py') with open(mod_path, 'w') as fp: fp.write('accessed') pb.add_source(fp.name, 'my_package/submodule/mod.py') pex = os.path.join(td2, 'app.pex') pb.build(pex) output, returncode = run_simple_pex(pex) try: output = output.decode('UTF-8') except ValueError: pass assert output == 'accessed\n' assert returncode == 0
def _add_test_hello_to_pex(ep): with tempfile.NamedTemporaryFile() as tf: tf.write(b'def hello(): print("hello")') tf.flush() pex_builder = PEXBuilder() pex_builder.add_source(tf.name, 'test.py') pex_builder.set_entry_point(ep) pex_builder.freeze() yield pex_builder
def build_and_check(path, copy): pb = PEXBuilder(path, copy=copy) pb.add_source(src, 'exe.py') s1 = os.stat(src) s2 = os.stat(os.path.join(path, 'exe.py')) is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV]) if copy: assert not is_link else: assert is_link
def write_simple_pex(td, exe_contents=None, dists=None, sources=None, coverage=False, interpreter=None, pex_info=None): """Write a pex file that optionally contains an executable entry point. :param str td: temporary directory path :param str exe_contents: entry point python file :param dists: distributions to include, typically sdists or bdists :type: list of :class:`pex.third_party.pkg_resources.Distribution` :param sources: sources to include, as a list of pairs (env_filename, contents) :type sources: list of (str, str) :param bool coverage: include coverage header :param interpreter: a custom interpreter to use to build the pex :type interpreter: :class:`pex.interpreter.PythonInterpreter` :param pex_info: a custom PexInfo to use to build the pex. :type pex_info: :class:`pex.pex_info.PexInfo` """ dists = dists or [] sources = sources or [] safe_mkdir(td) pb = PEXBuilder( path=td, preamble=COVERAGE_PREAMBLE if coverage else None, interpreter=interpreter, pex_info=pex_info, ) for dist in dists: pb.add_dist_location( dist.location if isinstance(dist, Distribution) else dist) for env_filename, contents in sources: src_path = os.path.join(td, env_filename) safe_mkdir(os.path.dirname(src_path)) with open(src_path, "w") as fp: fp.write(contents) pb.add_source(src_path, env_filename) if exe_contents: with open(os.path.join(td, "exe.py"), "w") as fp: fp.write(exe_contents) pb.set_executable(os.path.join(td, "exe.py")) pb.freeze() return pb
def write_simple_pex( td, # type: str exe_contents=None, # type: Optional[str] dists=None, # type: Optional[Iterable[Distribution]] sources=None, # type: Optional[Iterable[Tuple[str, str]]] coverage=False, # type: bool interpreter=None, # type: Optional[PythonInterpreter] pex_info=None, # type: Optional[PexInfo] ): # type: (...) -> PEXBuilder """Write a pex file that optionally contains an executable entry point. :param td: temporary directory path :param exe_contents: entry point python file :param dists: distributions to include, typically sdists or bdists :param sources: sources to include, as a list of pairs (env_filename, contents) :param coverage: include coverage header :param interpreter: a custom interpreter to use to build the pex :param pex_info: a custom PexInfo to use to build the pex. """ dists = dists or [] sources = sources or [] safe_mkdir(td) pb = PEXBuilder( path=td, preamble=COVERAGE_PREAMBLE if coverage else None, interpreter=interpreter, pex_info=pex_info, ) for dist in dists: pb.add_dist_location( dist.location if isinstance(dist, Distribution) else dist) for env_filename, contents in sources: src_path = os.path.join(td, env_filename) safe_mkdir(os.path.dirname(src_path)) with open(src_path, "w") as fp: fp.write(contents) pb.add_source(src_path, env_filename) if exe_contents: with open(os.path.join(td, "exe.py"), "w") as fp: fp.write(exe_contents) pb.set_executable(os.path.join(td, "exe.py")) pb.freeze() return pb
def build_and_check(path, copy): pb = PEXBuilder(path, copy=copy) pb.add_source(src, 'exe.py') path_clone = os.path.join(path, '__clone') pb.clone(into=path_clone) for root in path, path_clone: s1 = os.stat(src) s2 = os.stat(os.path.join(root, 'exe.py')) is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV]) if copy: assert not is_link else: assert is_link
def test_pex_builder_add_source_relpath_issues_1192( tmp_chroot, # type: str copy_mode, # type: CopyMode.Value ): # type: (...) -> None pb = PEXBuilder(copy_mode=copy_mode) with safe_open("src/main.py", "w") as fp: fp.write("import sys; sys.exit(42)") pb.add_source("src/main.py", "main.py") pb.set_entry_point("main") pb.build("test.pex") process = Executor.open_process(cmd=[os.path.abspath("test.pex")]) process.wait() assert 42 == process.returncode
def _add_test_hello_to_pex(ep): with temporary_dir() as td: hello_file = "\n".join([ "def hello():", " print('hello')", ]) with temporary_file(root_dir=td) as tf: with open(tf.name, 'w') as handle: handle.write(hello_file) pex_builder = PEXBuilder() pex_builder.add_source(tf.name, 'test.py') pex_builder.set_entry_point(ep) pex_builder.freeze() yield pex_builder
def test_execute_interpreter_dashm_module(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) with temporary_file(root_dir=pex_chroot) as fp: fp.write(b'import sys; print(" ".join(sys.argv))') fp.close() pex_builder.add_source(fp.name, 'foo/bar.py') pex_builder.freeze() process = PEX(pex_chroot).run(args=['-m', 'foo.bar', 'one', 'two'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False) stdout, stderr = process.communicate() assert 0 == process.returncode assert b'foo.bar one two\n' == stdout assert b'' == stderr
def _hydrate_pex_file(self, hydrated_pex_file): # We extract source files into a temporary directory before creating the pex. td = tempfile.mkdtemp() with open_zip(self) as zf: # Populate the pex with the pinned requirements and distribution names & hashes. bootstrap_info = PexInfo.from_json(zf.read("BOOTSTRAP-PEX-INFO")) bootstrap_builder = PEXBuilder(pex_info=bootstrap_info, interpreter=PythonInterpreter.get()) # Populate the pex with the needed code. try: ipex_info = json.loads(zf.read("IPEX-INFO").decode("utf-8")) for path in ipex_info["code"]: unzipped_source = zf.extract(path, td) bootstrap_builder.add_source( unzipped_source, env_filename=_strip_app_code_prefix(path)) except Exception as e: raise ValueError( "Error: {e}. The IPEX-INFO for this .ipex file was:\n{info}". format(e=e, info=json.dumps(ipex_info, indent=4))) # Perform a fully pinned intransitive resolve to hydrate the install cache. resolver_settings = ipex_info["resolver_settings"] sanitized_requirements = _sanitize_requirements( bootstrap_info.requirements) bootstrap_info = modify_pex_info(bootstrap_info, requirements=sanitized_requirements) bootstrap_builder.info = bootstrap_info resolved_distributions = resolver.resolve( requirements=bootstrap_info.requirements, cache=bootstrap_info.pex_root, platform="current", transitive=False, interpreter=bootstrap_builder.interpreter, **resolver_settings) # TODO: this shouldn't be necessary, as we should be able to use the same 'distributions' from # BOOTSTRAP-PEX-INFO. When the .ipex is executed, the normal pex bootstrap fails to see these # requirements or recognize that they should be pulled from the cache for some reason. for resolved_dist in resolved_distributions: bootstrap_builder.add_distribution(resolved_dist.distribution) bootstrap_builder.build(hydrated_pex_file, bytecode_compile=False)
def build_and_check(path, copy): # type: (str, bool) -> None pb = PEXBuilder(path=path, copy=copy) pb.add_source(src, "exe.py") path_clone = os.path.join(path, "__clone") pb.clone(into=path_clone) for root in path, path_clone: s1 = os.stat(src) s2 = os.stat(os.path.join(root, "exe.py")) is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV]) if copy: assert not is_link else: assert is_link
def build_and_check(path, precompile): pb = PEXBuilder(path) pb.add_source(src, 'lib/src.py') pb.set_executable(exe, 'exe.py') pb.freeze(bytecode_compile=precompile) for pyc_file in ('exe.pyc', 'lib/src.pyc', '__main__.pyc'): pyc_exists = os.path.exists(os.path.join(path, pyc_file)) if precompile: assert pyc_exists else: assert not pyc_exists bootstrap_dir = os.path.join(path, PEXBuilder.BOOTSTRAP_DIR) bootstrap_pycs = [] for _, _, files in os.walk(bootstrap_dir): bootstrap_pycs.extend(f for f in files if f.endswith('.pyc')) if precompile: assert len(bootstrap_pycs) > 0 else: assert 0 == len(bootstrap_pycs)
def write_simple_pex(td, exe_contents, dists=None, sources=None, coverage=False, interpreter=None): """Write a pex file that contains an executable entry point :param td: temporary directory path :param exe_contents: entry point python file :type exe_contents: string :param dists: distributions to include, typically sdists or bdists :param sources: sources to include, as a list of pairs (env_filename, contents) :param coverage: include coverage header :param interpreter: a custom interpreter to use to build the pex """ dists = dists or [] sources = sources or [] safe_mkdir(td) with open(os.path.join(td, 'exe.py'), 'w') as fp: fp.write(exe_contents) pb = PEXBuilder(path=td, preamble=COVERAGE_PREAMBLE if coverage else None, interpreter=interpreter) for dist in dists: pb.add_dist_location(dist.location) for env_filename, contents in sources: src_path = os.path.join(td, env_filename) safe_mkdir(os.path.dirname(src_path)) with open(src_path, 'w') as fp: fp.write(contents) pb.add_source(src_path, env_filename) pb.set_executable(os.path.join(td, 'exe.py')) pb.freeze() return pb
def build_and_check(path, precompile): # type: (str, bool) -> None pb = PEXBuilder(path=path) pb.add_source(src, "lib/src.py") pb.set_executable(exe, "exe.py") pb.freeze(bytecode_compile=precompile) for pyc_file in ("exe.pyc", "lib/src.pyc", "__main__.pyc"): pyc_exists = os.path.exists(os.path.join(path, pyc_file)) if precompile: assert pyc_exists else: assert not pyc_exists bootstrap_dir = os.path.join(path, pb.info.bootstrap) bootstrap_pycs = [] # type: List[str] for _, _, files in os.walk(bootstrap_dir): bootstrap_pycs.extend(f for f in files if f.endswith(".pyc")) if precompile: assert len(bootstrap_pycs) > 0 else: assert 0 == len(bootstrap_pycs)
def test_execute_interpreter_dashm_module(): with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot) pex_builder.add_source(None, "foo/__init__.py") with tempfile.NamedTemporaryFile() as fp: fp.write(b'import sys; print(" ".join(sys.argv))') fp.flush() pex_builder.add_source(fp.name, "foo/bar.py") pex_builder.freeze() pex = PEX(pex_chroot) process = pex.run( args=["-m", "foo.bar", "one", "two"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, blocking=False, ) stdout, stderr = process.communicate() assert 0 == process.returncode assert b"foo.bar one two\n" == stdout assert b"" == stderr
def assert_access_zipped_assets(distribution_helper_import): test_executable = dedent(""" import os {distribution_helper_import} temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule') with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp: for line in fp: print(line) """.format(distribution_helper_import=distribution_helper_import)) with nested(temporary_dir(), temporary_dir()) as (td1, td2): pb = PEXBuilder(path=td1) with open(os.path.join(td1, 'exe.py'), 'w') as fp: fp.write(test_executable) pb.set_executable(fp.name) submodule = os.path.join(td1, 'my_package', 'submodule') safe_mkdir(submodule) mod_path = os.path.join(submodule, 'mod.py') with open(mod_path, 'w') as fp: fp.write('accessed') pb.add_source(fp.name, 'my_package/submodule/mod.py') pb.add_source(None, 'my_package/__init__.py') pb.add_source(None, 'my_package/submodule/__init__.py') pex = os.path.join(td2, 'app.pex') pb.build(pex) process = PEX(pex, interpreter=pb.interpreter).run(blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert process.returncode == 0 assert b'accessed\n' == stdout return stderr
def assert_access_zipped_assets(distribution_helper_import): # type: (str) -> bytes test_executable = dedent(""" import os {distribution_helper_import} temp_dir = DistributionHelper.access_zipped_assets('my_package', 'submodule') with open(os.path.join(temp_dir, 'mod.py'), 'r') as fp: for line in fp: print(line) """.format(distribution_helper_import=distribution_helper_import)) with temporary_dir() as td1, temporary_dir() as td2: pb = PEXBuilder(path=td1) with open(os.path.join(td1, "exe.py"), "w") as fp: fp.write(test_executable) pb.set_executable(fp.name) submodule = os.path.join(td1, "my_package", "submodule") safe_mkdir(submodule) mod_path = os.path.join(submodule, "mod.py") with open(mod_path, "w") as fp: fp.write("accessed") pb.add_source(fp.name, "my_package/submodule/mod.py") pb.add_source(None, "my_package/__init__.py") pb.add_source(None, "my_package/submodule/__init__.py") pex = os.path.join(td2, "app.pex") pb.build(pex) process = PEX(pex, interpreter=pb.interpreter).run(blocking=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert process.returncode == 0 assert b"accessed\n" == stdout return cast(bytes, stderr)
def build_and_check(copy_mode): # type: (CopyMode.Value) -> None pb = PEXBuilder(copy_mode=copy_mode) path = pb.path() pb.add_source(src, "exe.py") path_clone = os.path.join(path, "__clone") pb.clone(into=path_clone) for root in path, path_clone: s1 = os.stat(src) s2 = os.stat(os.path.join(root, "exe.py")) is_link = (s1[stat.ST_INO], s1[stat.ST_DEV]) == (s2[stat.ST_INO], s2[stat.ST_DEV]) if copy_mode == CopyMode.COPY: assert not is_link else: # Since os.stat follows symlinks; so in CopyMode.SYMLINK, this just proves # the symlink points to the original file. Going further and checking path # and path_clone for the presence of a symlink (an os.islink test) is # trickier since a Linux hardlink of a symlink produces a symlink whereas a # macOS hardlink of a symlink produces a hardlink. assert is_link
def build_pex(reqs, options, cache=None): interpreters = None # Default to the current interpreter. pex_python_path = options.python_path # If None, this will result in using $PATH. # TODO(#1075): stop looking at PEX_PYTHON_PATH and solely consult the `--python-path` flag. if pex_python_path is None and (options.rc_file or not ENV.PEX_IGNORE_RCFILES): rc_variables = Variables(rc=options.rc_file) pex_python_path = rc_variables.PEX_PYTHON_PATH # NB: options.python and interpreter constraints cannot be used together. if options.python: with TRACER.timed("Resolving interpreters", V=2): def to_python_interpreter(full_path_or_basename): if os.path.isfile(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interp = PythonInterpreter.from_env(full_path_or_basename) if interp is None: die("Failed to find interpreter: %s" % full_path_or_basename) return interp interpreters = [ to_python_interpreter(interp) for interp in options.python ] elif options.interpreter_constraint: with TRACER.timed("Resolving interpreters", V=2): constraints = options.interpreter_constraint validate_constraints(constraints) try: interpreters = list( iter_compatible_interpreters( path=pex_python_path, interpreter_constraints=constraints)) except UnsatisfiableInterpreterConstraintsError as e: die( e.create_message( "Could not find a compatible interpreter."), CANNOT_SETUP_INTERPRETER, ) platforms = OrderedSet(options.platforms) interpreters = interpreters or [] if options.platforms and options.resolve_local_platforms: with TRACER.timed( "Searching for local interpreters matching {}".format( ", ".join(map(str, platforms)))): candidate_interpreters = OrderedSet( iter_compatible_interpreters(path=pex_python_path)) candidate_interpreters.add(PythonInterpreter.get()) for candidate_interpreter in candidate_interpreters: resolved_platforms = candidate_interpreter.supported_platforms.intersection( platforms) if resolved_platforms: for resolved_platform in resolved_platforms: TRACER.log("Resolved {} for platform {}".format( candidate_interpreter, resolved_platform)) platforms.remove(resolved_platform) interpreters.append(candidate_interpreter) if platforms: TRACER.log( "Could not resolve a local interpreter for {}, will resolve only binary distributions " "for {}.".format( ", ".join(map(str, platforms)), "this platform" if len(platforms) == 1 else "these platforms", )) interpreter = (PythonInterpreter.latest_release_of_min_compatible_version( interpreters) if interpreters else None) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None pex_builder = PEXBuilder( path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble, copy_mode=CopyMode.SYMLINK, include_tools=options.include_tools or options.venv, ) if options.resources_directory: pex_warnings.warn( "The `-R/--resources-directory` option is deprecated. Resources should be added via " "`-D/--sources-directory` instead.") for directory in OrderedSet(options.sources_directory + options.resources_directory): src_dir = os.path.normpath(directory) for root, _, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) pex_builder.add_source(src_file_path, dst_path) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.unzip = options.unzip pex_info.venv = bool(options.venv) pex_info.venv_bin_path = options.venv pex_info.venv_copies = options.venv_copies pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = InheritPath.for_value(options.inherit_path) pex_info.pex_root = options.runtime_pex_root pex_info.strip_pex_env = options.strip_pex_env if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) indexes = compute_indexes(options) for requirements_pex in options.requirements_pexes: pex_builder.add_from_requirements_pex(requirements_pex) with TRACER.timed( "Resolving distributions ({})".format(reqs + options.requirement_files)): if options.cache_ttl: pex_warnings.warn( "The --cache-ttl option is deprecated and no longer has any effect." ) if options.headers: pex_warnings.warn( "The --header option is deprecated and no longer has any effect." ) network_configuration = NetworkConfiguration( retries=options.retries, timeout=options.timeout, proxy=options.proxy, cert=options.cert, client_cert=options.client_cert, ) try: if options.pex_repository: with TRACER.timed("Resolving requirements from PEX {}.".format( options.pex_repository)): resolveds = resolve_from_pex( pex=options.pex_repository, requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, network_configuration=network_configuration, transitive=options.transitive, interpreters=interpreters, platforms=list(platforms), manylinux=options.manylinux, ignore_errors=options.ignore_errors, ) else: with TRACER.timed("Resolving requirements."): resolveds = resolve_multi( requirements=reqs, requirement_files=options.requirement_files, constraint_files=options.constraint_files, allow_prereleases=options.allow_prereleases, transitive=options.transitive, interpreters=interpreters, platforms=list(platforms), indexes=indexes, find_links=options.find_links, resolver_version=ResolverVersion.for_value( options.resolver_version), network_configuration=network_configuration, cache=cache, build=options.build, use_wheel=options.use_wheel, compile=options.compile, manylinux=options.manylinux, max_parallel_jobs=options.max_parallel_jobs, ignore_errors=options.ignore_errors, ) for resolved_dist in resolveds: pex_builder.add_distribution(resolved_dist.distribution) if resolved_dist.direct_requirement: pex_builder.add_requirement( resolved_dist.direct_requirement) except Unsatisfiable as e: die(str(e)) if options.entry_point and options.script: die("Must specify at most one entry point or script.", INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default=sys.executable) parser.add_option('--preload', action='append', default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. interpreter = PythonInterpreter( options.python, PythonInterpreter.from_binary(options.python).identity, extras={}) pex_builder = PEXBuilder( path=tmp_dir, interpreter=interpreter, ) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources` & `_markerlib`. copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR) copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # Add resources listed in the manifest. for dst, src in manifest['nativeLibraries'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option("--entry-point", default="__main__") parser.add_option("--directory", action="store_true", default=False) parser.add_option( "--no-zip-safe", action="store_false", dest="zip_safe", default=True ) parser.add_option("--python", default="") parser.add_option("--python-version", default="") parser.add_option("--python-shebang", default=None) parser.add_option("--preload", action="append", default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. if not options.python: options.python = sys.executable identity = PythonIdentity.get() elif not options.python_version: # Note: this is expensive (~500ms). prefer passing --python-version when possible. identity = PythonInterpreter.from_binary(options.python).identity else: # Convert "CPython 2.7" to "CPython 2 7 0" python_version = options.python_version.replace(".", " ").split() if len(python_version) == 3: python_version.append("0") identity = PythonIdentity.from_id_string(" ".join(python_version)) interpreter = PythonInterpreter(options.python, identity, extras={}) pex_builder = PEXBuilder( path=output if options.directory else None, interpreter=interpreter ) if options.python_shebang is not None: pex_builder.set_shebang(options.python_shebang) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources` & `_markerlib`. copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR) copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest["modules"].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest["resources"].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add resources listed in the manifest. for dst, src in manifest["nativeLibraries"].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) if options.directory: pex_builder.freeze(code_hash=False, bytecode_compile=False) else: pex_builder.build(output)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option("--entry-point", default="__main__") parser.add_option("--directory", action="store_true", default=False) parser.add_option( "--no-zip-safe", action="store_false", dest="zip_safe", default=True ) parser.add_option("--python", default="") parser.add_option("--python-version", default="") parser.add_option("--python-shebang", default=None) parser.add_option("--preload", action="append", default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. if not options.python: options.python = sys.executable identity = PythonIdentity.get() elif not options.python_version: # Note: this is expensive (~500ms). prefer passing --python-version when possible. identity = PythonInterpreter.from_binary(options.python).identity else: # Convert "CPython 2.7" to "CPython 2 7 0" python_version = options.python_version.replace(".", " ").split() if len(python_version) == 3: python_version.append("0") identity = PythonIdentity.from_id_string(" ".join(python_version)) interpreter = PythonInterpreter(options.python, identity, extras={}) pex_builder = PEXBuilder( path=output if options.directory else None, interpreter=interpreter ) if options.python_shebang is not None: pex_builder.set_shebang(options.python_shebang) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources` & `_markerlib`. copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR) copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest["modules"].items(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest["resources"].items(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add resources listed in the manifest. for dst, src in manifest["nativeLibraries"].items(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) if options.directory: pex_builder.freeze(code_hash=False, bytecode_compile=False) else: pex_builder.build(output)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default=sys.executable) parser.add_option('--preload', action='append', default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. interpreter = PythonInterpreter(options.python, PythonInterpreter.from_binary( options.python).identity, extras={}) pex_builder = PEXBuilder( path=tmp_dir, interpreter=interpreter, ) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources`. copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # Add resources listed in the manifest. for dst, src in manifest['nativeLibraries'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)