def check_resolve_venv(real_interpreter): # type: (PythonInterpreter) -> None tmpdir = safe_mkdtemp() def create_venv( interpreter, # type: PythonInterpreter rel_path, # type: str ): # type: (...) -> List[str] venv_dir = os.path.join(tmpdir, rel_path) interpreter.execute(["-m", "venv", venv_dir]) return glob.glob(os.path.join(venv_dir, "bin", "python*")) assert not real_interpreter.is_venv assert real_interpreter is real_interpreter.resolve_base_interpreter() for index, python in enumerate(create_venv(real_interpreter, "first-level")): venv_interpreter = PythonInterpreter.from_binary(python) assert venv_interpreter.is_venv assert venv_interpreter != real_interpreter.binary assert real_interpreter == venv_interpreter.resolve_base_interpreter() for nested_python in create_venv(venv_interpreter, "second-level{}".format(index)): nested_venv_interpreter = PythonInterpreter.from_binary( nested_python) assert nested_venv_interpreter.is_venv assert nested_venv_interpreter != venv_interpreter assert nested_venv_interpreter != real_interpreter assert real_interpreter == nested_venv_interpreter.resolve_base_interpreter( )
def test_venv_copies(tmpdir): # type: (Any) -> None python36 = ensure_python_interpreter(PY36) pex_file = os.path.join(str(tmpdir), "venv.pex") result = run_pex_command(args=["-o", pex_file, "--include-tools"], python=python36) result.assert_success() PEX_TOOLS = make_env(PEX_TOOLS=1) venv_symlinks = os.path.join(str(tmpdir), "venv.symlinks") subprocess.check_call(args=[python36, pex_file, "venv", venv_symlinks], env=PEX_TOOLS) venv_symlinks_interpreter = PythonInterpreter.from_binary( os.path.join(venv_symlinks, "bin", "python")) assert os.path.islink(venv_symlinks_interpreter.binary) venv_copies = os.path.join(str(tmpdir), "venv.copies") subprocess.check_call( args=[python36, pex_file, "venv", "--copies", venv_copies], env=PEX_TOOLS) venv_copies_interpreter = PythonInterpreter.from_binary( os.path.join(venv_copies, "bin", "python")) assert not os.path.islink(venv_copies_interpreter.binary)
def _do_test_exact_requirements_interpreter_change(self, sdist: bool) -> None: python36 = PythonInterpreter.from_binary(python_interpreter_path(PY_36)) python37 = PythonInterpreter.from_binary(python_interpreter_path(PY_37)) with self.plugin_resolution( interpreter=python36, plugins=[("jake", "1.2.3"), ("jane", "3.4.5")], sdist=sdist ) as results: working_set, chroot, repo_dir, cache_dir = results safe_rmtree(repo_dir) with pytest.raises(Unsatisfiable): with self.plugin_resolution( interpreter=python37, chroot=chroot, plugins=[("jake", "1.2.3"), ("jane", "3.4.5")], ): pytest.fail( "Plugin re-resolution is expected for an incompatible interpreter and it is " "expected to fail since we removed the dist `repo_dir` above." ) # But for a compatible interpreter the exact resolve results should be re-used and load # directly from the still in-tact cache. with self.plugin_resolution( interpreter=python36, chroot=chroot, plugins=[("jake", "1.2.3"), ("jane", "3.4.5")] ) as results2: working_set2, _, _, _ = results2 assert list(working_set) == list(working_set2)
def test_detect_pyvenv(tmpdir): # type: (Any) -> None venv = str(tmpdir) py35 = ensure_python_interpreter(PY35) real_interpreter = PythonInterpreter.from_binary(py35) real_interpreter.execute(["-m", "venv", venv]) with pytest.raises(Executor.NonZeroExit): real_interpreter.execute(["-c", "import colors"]) venv_bin_dir = os.path.join(venv, "bin") subprocess.check_call( [os.path.join(venv_bin_dir, "pip"), "install", "ansicolors==1.1.8"]) canonical_to_python = defaultdict(set) for python in glob.glob(os.path.join(venv_bin_dir, "python*")): venv_interpreter = PythonInterpreter.from_binary(python) canonical_to_python[venv_interpreter.binary].add(python) venv_interpreter.execute(["-c", "import colors"]) assert (len(canonical_to_python) == 1 ), "Expected exactly one canonical venv python, found: {}".format( canonical_to_python) canonical, pythons = canonical_to_python.popitem() real_python = os.path.realpath(py35) assert canonical != real_python assert os.path.dirname(canonical) == venv_bin_dir assert os.path.realpath(canonical) == real_python assert len( pythons ) >= 2, "Expected at least two virtualenv python binaries, found: {}".format( pythons)
def test_iter_interpreter_some(self, test_interpreter1, test_interpreter2): # type: (str, str) -> None assert [ PythonInterpreter.from_binary(test_interpreter1), PythonInterpreter.from_binary(test_interpreter2), ] == list( PythonInterpreter.iter_candidates( paths=[test_interpreter1, test_interpreter2]))
def test_interpreter_caching(self, test_interpreter1, test_interpreter2): # type: (str, str) -> None py_interpreter1 = PythonInterpreter.from_binary(test_interpreter1) py_interpreter2 = PythonInterpreter.from_binary(test_interpreter2) assert py_interpreter1 is not py_interpreter2 assert py_interpreter2.identity.version == self.TEST_INTERPRETER2_VERSION_TUPLE py_interpreter3 = PythonInterpreter.from_binary(test_interpreter1) assert py_interpreter1 is py_interpreter3
def test_resolve_current_and_foreign_platforms(p537_resolve_cache): # type: (str) -> None foreign_platform = "macosx-10.13-x86_64-cp-37-m" if IS_LINUX else "manylinux1_x86_64-cp-37-m" resolve_current_and_foreign = functools.partial( resolve_p537_wheel_names, cache=p537_resolve_cache, platforms=["current", foreign_platform]) assert 2 == len(resolve_current_and_foreign()) other_python_version = PY36 if PY_VER == (3, 5) else PY35 other_python = PythonInterpreter.from_binary( ensure_python_interpreter(other_python_version)) current_python = PythonInterpreter.get() assert 2 == len(resolve_current_and_foreign(interpreters=[current_python])) assert 2 == len(resolve_current_and_foreign(interpreters=[other_python])) assert 2 == len( resolve_current_and_foreign( interpreters=[current_python, current_python])) # Here we have 2 local interpreters, satisfying current, but with different platforms and thus # different dists and then the foreign platform for 3 total dists. assert 3 == len( resolve_current_and_foreign( interpreters=[current_python, other_python]))
def wheel_installer(*mixins): bare_interpreter = PythonInterpreter.from_binary( ensure_python_interpreter(PY36)) with make_installer(installer_impl=OrderableInstaller, interpreter=bare_interpreter, mixins=list(mixins)) as installer: yield installer
def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel): interpreter = None if python_interpreter: if os.path.exists(python_interpreter): interpreter = PythonInterpreter.from_binary(python_interpreter) else: interpreter = PythonInterpreter.from_env(python_interpreter) if interpreter is None: die('Failed to find interpreter: %s' % python_interpreter) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def fake_interpreter(python_tag: str, abi_tag: str, version: Tuple[int, int, int]): interpreter_dir = safe_mkdtemp() binary = os.path.join(interpreter_dir, "python") values = dict( binary=binary, python_tag=python_tag, abi_tag=abi_tag, platform_tag="", version=version, supported_tags=[], env_markers={}, ) id_str = json.dumps(values) with open(binary, "w") as fp: fp.write( dedent( f""" #!{PythonInterpreter.get().binary} from __future__ import print_function print({id_str!r}) """ ).strip() ) chmod_plus_x(binary) return PythonInterpreter.from_binary(binary)
def find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH. """ if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: print( "Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary, file=sys.stderr) if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.' ) else: if not os.getenv('PATH', ''): # no $PATH, use sys.executable interpreters = [PythonInterpreter.get()] else: # get all qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() return list( matched_interpreters(interpreters, compatibility_constraints ) if compatibility_constraints else interpreters)
def bare_interpreter(): with temporary_dir() as interpreter_cache: yield setup_interpreter(interpreter=PythonInterpreter.from_binary( ensure_python_interpreter(PY36)), interpreter_cache_dir=interpreter_cache, repos=None, use_wheel=True)
def test_resolve_current_platform(p537_resolve_cache): # type: (str) -> None resolve_current = functools.partial(resolve_p537_wheel_names, cache=p537_resolve_cache, platforms=["current"]) other_python_version = PY36 if PY_VER == (3, 5) else PY35 other_python = PythonInterpreter.from_binary( ensure_python_interpreter(other_python_version)) current_python = PythonInterpreter.get() resolved_other = resolve_current(interpreters=[other_python]) resolved_current = resolve_current() assert 1 == len(resolved_other) assert 1 == len(resolved_current) assert resolved_other != resolved_current assert resolved_current == resolve_current(interpreters=[current_python]) assert resolved_current == resolve_current( interpreters=[current_python, current_python]) # Here we have 2 local interpreters satisfying current but with different platforms and thus # different dists for 2 total dists. assert 2 == len( resolve_current(interpreters=[current_python, other_python]))
def find_compatible_interpreters(pex_python_path=None, compatibility_constraints=None): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH. """ if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary, file=sys.stderr) if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: # We may have been invoked with a specific interpreter not on the $PATH, make sure our # sys.executable is included as a candidate in this case. interpreters = OrderedSet([PythonInterpreter.get()]) # Add all qualifying interpreters found in $PATH. interpreters.update(PythonInterpreter.all()) return list( matched_interpreters(interpreters, compatibility_constraints) if compatibility_constraints else interpreters )
def test_issues_892(): python27 = ensure_python_interpreter(PY27) program = dedent("""\ from __future__ import print_function import os import sys # This puts python3.6 stdlib on PYTHONPATH. os.environ['PYTHONPATH'] = os.pathsep.join(sys.path) from pex import resolver from pex.interpreter import PythonInterpreter python27 = PythonInterpreter.from_binary({python27!r}) result = resolver.resolve(requirements=['packaging==19.2'], interpreter=python27) print('Resolved: {{}}'.format(result)) """.format(python27=python27)) python36 = ensure_python_interpreter(PY36) cmd, process = PythonInterpreter.from_binary(python36).open_process( args=["-c", program], stderr=subprocess.PIPE) _, stderr = process.communicate() assert process.returncode == 0, dedent(""" Command {cmd} failed with {returncode}. STDERR ====== {stderr} """.format(cmd=cmd, returncode=process.returncode, stderr=stderr.decode("utf8")))
def test_pex_run_conflicting_custom_setuptools_useable(): # Here we use an older setuptools to build the pex which has a newer setuptools requirement. # These setuptools dists have different pkg_resources APIs: # $ diff \ # <(zipinfo -1 setuptools-20.3.1-py2.py3-none-any.whl | grep pkg_resources/ | sort) \ # <(zipinfo -1 setuptools-40.4.3-py2.py3-none-any.whl | grep pkg_resources/ | sort) # 2a3,4 # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py with temporary_dir() as resolve_cache: dists = [resolved_dist.distribution for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)] interpreter = PythonInterpreter.from_binary(sys.executable, path_extras=[dist.location for dist in dists], include_site_extras=False) dists = [resolved_dist.distribution for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, 'from pkg_resources import appdirs, py31compat', dists=dists, interpreter=interpreter ) rc = PEX(pex.path()).run() assert rc == 0
def test_activate_interpreter_different_from_current(): with temporary_dir() as pex_root: interp_version = PY36 if PY2 else PY27 custom_interpreter = setup_interpreter( interpreter=PythonInterpreter.from_binary(ensure_python_interpreter(interp_version)), interpreter_cache_dir=os.path.join(pex_root, 'interpreters'), repos=None, # Default to PyPI. use_wheel=True ) pex_info = PexInfo.default(custom_interpreter) pex_info.pex_root = pex_root with temporary_dir() as pex_chroot: pex_builder = PEXBuilder(path=pex_chroot, interpreter=custom_interpreter, pex_info=pex_info) with make_bdist(installer_impl=WheelInstaller, interpreter=custom_interpreter) as bdist: pex_builder.add_distribution(bdist) pex_builder.set_entry_point('sys:exit') pex_builder.freeze() pex = PEX(pex_builder.path(), interpreter=custom_interpreter) try: pex._activate() except SystemExit as e: pytest.fail('PEX activation of %s failed with %s' % (pex, e))
def test_iter_candidates_empty_paths(self, test_interpreter1): # type: (str) -> None # Whereas `paths=None` should inspect $PATH, `paths=[]` means to search nothing. with environment_as(PATH=test_interpreter1): assert [] == list(PythonInterpreter.iter_candidates(paths=[])) assert [PythonInterpreter.from_binary(test_interpreter1) ] == list(PythonInterpreter.iter_candidates(paths=None))
def interpreter_from_options(options): interpreter = None if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) else: interpreter = PythonInterpreter.from_env(options.python) if interpreter is None: die('Failed to find interpreter: %s' % options.python) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and options.use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def to_python_interpreter(full_path_or_basename): if os.path.isfile(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interp = PythonInterpreter.from_env(full_path_or_basename) if interp is None: die("Failed to find interpreter: %s" % full_path_or_basename) return interp
def resolve_pytest(python_version, pytest_version): interpreter = PythonInterpreter.from_binary(ensure_python_interpreter(python_version)) resolved_dists = resolve_multi( interpreters=[interpreter], requirements=["pytest=={}".format(pytest_version)] ) project_to_version = {rd.requirement.key: rd.distribution.version for rd in resolved_dists} assert project_to_version["pytest"] == pytest_version return project_to_version
def test_iter_interpreter_path_filter(self, test_interpreter1, test_interpreter2): # type: (str, str) -> None assert [PythonInterpreter.from_binary(test_interpreter2)] == list( PythonInterpreter.iter_candidates( paths=[test_interpreter1, test_interpreter2], path_filter=lambda path: path == test_interpreter2, ))
def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter
def _validate_good_interpreter_path_file(path): with open(path, 'r') as fp: lines = fp.readlines() binary = lines[0].strip() try: interpreter = PythonInterpreter.from_binary(binary) return True if interpreter else False except Executor.ExecutableNotFound: return False
def _get_interpreter(interpreter_path_file): with open(interpreter_path_file, 'r') as infile: lines = infile.readlines() binary = lines[0].strip() interpreter = PythonInterpreter.from_binary(binary, include_site_extras=False) for line in lines[1:]: dist_name, dist_version, location = line.strip().split('\t') interpreter = interpreter.with_extra(dist_name, dist_version, location) return interpreter
def _interpreter_from_path(self, path, filters): try: executable = os.readlink(os.path.join(path, 'python')) except OSError: return None interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False) if self._matches(interpreter, filters): return self._resolve(interpreter) return None
def _get_interpreter(interpreter_path_file): with open(interpreter_path_file, 'r') as infile: lines = infile.readlines() binary = lines[0].strip() interpreter = PythonInterpreter.from_binary( binary, include_site_extras=False) for line in lines[1:]: dist_name, dist_version, location = line.strip().split('\t') interpreter = interpreter.with_extra(dist_name, dist_version, location) return interpreter
def interpreter_from_options(options): interpreter = None if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) else: interpreter = PythonInterpreter.from_env(options.python) if interpreter is None: die('Failed to find interpreter: %s' % options.python) else: interpreter = PythonInterpreter.get() return interpreter
def _get_interpreter(self, interpreter_path_file, targets): if os.path.exists(interpreter_path_file): with open(interpreter_path_file, "r") as infile: binary = infile.read().strip() try: return PythonInterpreter.from_binary(binary) except PythonInterpreter.Error: self.context.log.info( "Stale interpreter reference detected: {}, removing reference and " "selecting a new interpreter.".format(binary)) os.remove(interpreter_path_file) return self._select_interpreter(interpreter_path_file, targets)
def create_bare_interpreter(binary_path): """Creates an interpreter for python binary at the given path. The interpreter is bare in that it has no extras associated with it. :returns: A bare python interpreter with no extras. :rtype: :class:`pex.interpreter.PythonInterpreter` """ # TODO(John Sirois): Replace with a more direct PythonInterpreter construction API call when # https://github.com/pantsbuild/pex/issues/510 is fixed. interpreter_with_extras = PythonInterpreter.from_binary(binary_path) return PythonInterpreter(binary_path, interpreter_with_extras.identity, extras=None)
def fake_interpreter(id_str): interpreter_dir = safe_mkdtemp() binary = os.path.join(interpreter_dir, 'binary') with open(binary, 'w') as fp: fp.write(dedent(""" #!{} from __future__ import print_function print({!r}) """.format(PythonInterpreter.get().binary, id_str)).strip()) chmod_plus_x(binary) return PythonInterpreter.from_binary(binary)
def _select_pex_python_interpreter(target_python, compatibility_constraints=None): target = find_in_path(target_python) if not target: die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) if compatibility_constraints: pi = PythonInterpreter.from_binary(target) if not list(matched_interpreters([pi], compatibility_constraints)): die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' 'interpreter constraints: %s' % (target, str(compatibility_constraints))) if not os.path.exists(target): die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target) return target
def _interpreter_from_relpath(self, path, filters=()): path = os.path.join(self._cache_dir, path) try: executable = os.readlink(os.path.join(path, 'python')) if not os.path.exists(executable): self._purge_interpreter(path) return None except OSError: return None interpreter = PythonInterpreter.from_binary(executable) if self._matches(interpreter, filters=filters): return interpreter return None
def _get_interpreter(self, interpreter_path_file, targets): if os.path.exists(interpreter_path_file): with open(interpreter_path_file, 'r') as infile: binary = infile.read().strip() try: return PythonInterpreter.from_binary(binary) except Executor.ExecutableNotFound: # TODO(John Sirois): Trap a more appropriate exception once available: # https://github.com/pantsbuild/pex/issues/672 self.context.log.info('Stale interpreter reference detected: {}, removing reference and ' 'selecting a new interpreter.'.format(binary)) os.remove(interpreter_path_file) return self._select_interpreter(interpreter_path_file, targets)
def _establish_interpreter(args): if args.python: if os.path.exists(args.python): interpreter = PythonInterpreter.from_binary(args.python) else: interpreter = PythonInterpreter.from_env(args.python) if interpreter is None: die('Failed to find interpreter: %s' % args.python) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, args.interpreter_cache_dir, args.repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and args.use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def interpreter_from_options(options): interpreter = None if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) else: interpreter = PythonInterpreter.from_env(options.python) if interpreter is None: die("Failed to find interpreter: %s" % options.python) else: interpreter = PythonInterpreter.get() with TRACER.timed("Setting up interpreter %s" % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and options.use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def main(): """ Main """ # Options that this wrapper will accept from the bazel rule parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True) parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default="python2.7") parser.add_option('--find-links', dest='find_links', default='') parser.add_option('--no-use-wheel', action='store_false', dest='use_wheel', default=True) parser.add_option('--pex-root', dest='pex_root', default=".pex") options, args = parser.parse_args() # The manifest is passed via stdin or a file, as it can sometimes get too # large to be passed as a CLA. if len(args) == 2: output = args[0] manifest_text = open(args[1], 'r').read() elif len(args) == 1: output = args[0] manifest_text = sys.stdin.read() else: parser.error("'output' positional argument is required") return 1 if manifest_text.startswith('"') and manifest_text.endswith('"'): manifest_text = manifest_text[1:len(manifest_text) - 1] manifest = parse_manifest(manifest_text) # These are the options that pex will use pparser, resolver_options_builder = configure_clp() poptions, preqs = pparser.parse_args(sys.argv) poptions.entry_point = options.entry_point poptions.find_links = options.find_links poptions.pypi = options.pypi poptions.python = options.python poptions.use_wheel = options.use_wheel poptions.zip_safe = options.zip_safe poptions.pex_root = options.pex_root poptions.cache_dir = options.pex_root + "/build" poptions.interpreter_cache_dir = options.pex_root + "/interpreters" # sys.stderr.write("pex options: %s\n" % poptions) os.environ["PATH"] = os.getenv("PATH", "%s:/bin:/usr/bin" % poptions.python) if os.path.exists(options.python): pybin = poptions.python else: pybin = distutils.spawn.find_executable(options.python) # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in and force it into the # process by constructing a custom PythonInterpreter instance using it. # interpreter = PythonInterpreter.from_binary(pybin, # [SETUPTOOLS_PATH, # WHEEL_PATH]) interpreter = PythonInterpreter( pybin, PythonInterpreter.from_binary(pybin).identity, extras={ # TODO: Fix this to resolve automatically ('setuptools', '18.0.1'): SETUPTOOLS_PATH, # FIXME: I don't think this accomplishes anything at all. ('wheel', '0.23.0'): WHEEL_PATH, }) # resolve setuptools interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions) # possibly resolve wheel if interpreter and poptions.use_wheel: interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions) # Add prebuilt libraries listed in the manifest. reqs = manifest.get('requirements', {}).keys() # if len(reqs) > 0: # sys.stderr.write("pex requirements: %s" % reqs) pex_builder = build_pex(reqs, poptions, resolver_options_builder, interpreter=interpreter) # Set whether this PEX is zip-safe, meaning everything will stay zipped # up and we'll rely on python's zip-import mechanism to load modules # from the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point pex_builder.add_source( dereference_symlinks(PKG_RESOURCES_PATH), os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py')) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: raise Exception("Failed to add {}: {}".format(src, err)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as err: raise Exception("Failed to add {}: {}".format(req, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. # Generate the PEX file. pex_builder.build(output)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option("--entry-point", default="__main__") parser.add_option("--directory", action="store_true", default=False) parser.add_option( "--no-zip-safe", action="store_false", dest="zip_safe", default=True ) parser.add_option("--python", default="") parser.add_option("--python-version", default="") parser.add_option("--python-shebang", default=None) parser.add_option("--preload", action="append", default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. if not options.python: options.python = sys.executable identity = PythonIdentity.get() elif not options.python_version: # Note: this is expensive (~500ms). prefer passing --python-version when possible. identity = PythonInterpreter.from_binary(options.python).identity else: # Convert "CPython 2.7" to "CPython 2 7 0" python_version = options.python_version.replace(".", " ").split() if len(python_version) == 3: python_version.append("0") identity = PythonIdentity.from_id_string(" ".join(python_version)) interpreter = PythonInterpreter(options.python, identity, extras={}) pex_builder = PEXBuilder( path=output if options.directory else None, interpreter=interpreter ) if options.python_shebang is not None: pex_builder.set_shebang(options.python_shebang) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources` & `_markerlib`. copy_package(pex_builder, "pkg_resources", prefix=pex_builder.BOOTSTRAP_DIR) copy_package(pex_builder, "_markerlib", prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest["modules"].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest["resources"].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add resources listed in the manifest. for dst, src in manifest["nativeLibraries"].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) if options.directory: pex_builder.freeze(code_hash=False, bytecode_compile=False) else: pex_builder.build(output)
def main(): # These are the options that this class will accept from the rule parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-pypi', action='store_false', dest='pypi', default=True) parser.add_option('--not-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default="/usr/bin/python2.7") parser.add_option('--find-links', dest='find_links', default='') options, args = parser.parse_args() if len(args) == 2: output = args[0] manifest_text = open(args[1], 'r').read() elif len(args) == 1: output = args[0] manifest_text = sys.stdin.read() else: parser.error("'output' positional argument is required") return 1 if manifest_text.startswith('"') and manifest_text.endswith('"'): manifest_text = manifest_text[1:len(manifest_text) - 1] # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. with open(os.path.join(tempfile.mkdtemp(dir="/tmp"), "stderr"), "w") as x: x.write(manifest_text) manifest = parse_manifest(manifest_text) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # These are the options that pex will use pparser, resolver_options_builder = configure_clp() # Disabling wheels since the PyYAML wheel is incompatible with the Travis CI linux host. # Enabling Trace logging in pex/tracer.py shows this upon failure: # pex: Target package WheelPackage('file:///tmp/tmpR_gDlG/PyYAML-3.11-cp27-cp27mu-linux_x86_64.whl') # is not compatible with CPython-2.7.3 / linux-x86_64 poptions, preqs = pparser.parse_args(['--no-use-wheel'] + sys.argv) poptions.entry_point = options.entry_point poptions.find_links = options.find_links poptions.pypi = options.pypi poptions.python = options.python poptions.zip_safe = options.zip_safe print("pex options: %s" % poptions) os.environ["PATH"] = ".:%s:/bin:/usr/bin" % poptions.python # The version of pkg_resources.py (from setuptools) on some distros is too old for PEX. So # we keep a recent version in and force it into the process by constructing a custom # PythonInterpreter instance using it. interpreter = PythonInterpreter( poptions.python, PythonInterpreter.from_binary(options.python).identity, extras={ # TODO: Fix this to resolve automatically ('setuptools', '18.0.1'): 'third_party/pex/setuptools-18.0.1-py2.py3-none-any.whl', ('wheel', '0.23.0'): 'third_party/pex/wheel-0.23.0-py2.7.egg' }) # resolve setuptools interpreter = resolve_or_die(interpreter, SETUPTOOLS_REQUIREMENT, poptions) # possibly resolve wheel if interpreter and poptions.use_wheel: interpreter = resolve_or_die(interpreter, WHEEL_REQUIREMENT, poptions) # Add prebuilt libraries listed in the manifest. reqs = manifest.get('requirements', {}).keys() if len(reqs) > 0: print("pex requirements: %s" % reqs) pex_builder = build_pex(reqs, poptions, resolver_options_builder, interpreter=interpreter) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point pex_builder.add_source( dereference_symlinks(pkg_resources_py), os.path.join(pex_builder.BOOTSTRAP_DIR, 'pkg_resources.py')) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # TODO(mikekap): Do something about manifest['nativeLibraries']. # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)
def main(): parser = optparse.OptionParser(usage="usage: %prog [options] output") parser.add_option('--entry-point', default='__main__') parser.add_option('--no-zip-safe', action='store_false', dest='zip_safe', default=True) parser.add_option('--python', default=sys.executable) parser.add_option('--preload', action='append', default=[]) options, args = parser.parse_args() if len(args) == 1: output = args[0] else: parser.error("'output' positional argument is required") return 1 # The manifest is passed via stdin, as it can sometimes get too large # to be passed as a CLA. manifest = json.load(sys.stdin) # Setup a temp dir that the PEX builder will use as its scratch dir. tmp_dir = tempfile.mkdtemp() try: # The version of pkg_resources.py (from setuptools) on some distros is # too old for PEX. So we keep a recent version in the buck repo and # force it into the process by constructing a custom PythonInterpreter # instance using it. interpreter = PythonInterpreter( options.python, PythonInterpreter.from_binary(options.python).identity, extras={}) pex_builder = PEXBuilder( path=tmp_dir, interpreter=interpreter, ) # Set whether this PEX as zip-safe, meaning everything will stayed zipped up # and we'll rely on python's zip-import mechanism to load modules from # the PEX. This may not work in some situations (e.g. native # libraries, libraries that want to find resources via the FS). pex_builder.info.zip_safe = options.zip_safe # Set the starting point for this PEX. pex_builder.info.entry_point = options.entry_point # Copy in our version of `pkg_resources` & `_markerlib`. copy_package(pex_builder, 'pkg_resources', prefix=pex_builder.BOOTSTRAP_DIR) copy_package(pex_builder, '_markerlib', prefix=pex_builder.BOOTSTRAP_DIR) # Add the sources listed in the manifest. for dst, src in manifest['modules'].iteritems(): # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as e: raise Exception("Failed to add {}: {}".format(src, e)) # Add resources listed in the manifest. for dst, src in manifest['resources'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Add prebuilt libraries listed in the manifest. for req in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(req) except Exception as e: raise Exception("Failed to add {}: {}".format(req, e)) # Add resources listed in the manifest. for dst, src in manifest['nativeLibraries'].iteritems(): # NOTE(agallagher): see rationale above. pex_builder.add_resource(dereference_symlinks(src), dst) # Generate the PEX file. pex_builder.build(output) # Always try cleaning up the scratch dir, ignoring failures. finally: shutil.rmtree(tmp_dir, True)