def test_pyenv_shims(self): # type: () -> None py35, _, run_pyenv = ensure_python_distribution(PY35) py36 = ensure_python_interpreter(PY36) pyenv_root = str(run_pyenv(["root"]).strip()) pyenv_shims = os.path.join(pyenv_root, "shims") def pyenv_global(*versions): run_pyenv(["global"] + list(versions)) def assert_shim(shim_name, expected_binary_path): python = PythonInterpreter.from_binary( os.path.join(pyenv_shims, shim_name)) assert expected_binary_path == python.binary with temporary_dir() as pex_root: with ENV.patch(PEX_ROOT=pex_root) as pex_env: with environment_as(PYENV_ROOT=pyenv_root, **pex_env): pyenv_global(PY35, PY36) assert_shim("python3", py35) pyenv_global(PY36, PY35) # The python3 shim is now pointing at python3.6 but the Pex cache has a valid # entry for the old python3.5 association (the interpreter still exists.) assert_shim("python3", py35) # The shim pointer is now invalid since python3.5 was uninstalled and so should # be re-read. py35_deleted = "{}.uninstalled".format(py35) os.rename(py35, py35_deleted) try: assert_shim("python3", py36) finally: os.rename(py35_deleted, py35)
def main(): parser, resolver_options_builder = configure_clp() # split arguments early because optparse is dumb args = sys.argv[1:] try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) return 0 if options.platform != Platform.current(): log('WARNING: attempting to run PEX with differing platform!') pex_builder.freeze() log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter) sys.exit(pex.run(args=list(cmdline)))
def test_make_relative(): with ENV.patch(PEX_ROOT='/pex_root'): assert '/pex_root/interpreters' == make_relative_to_root( '{pex_root}/interpreters') #Verify the user can specify arbitrary absolute paths. assert '/tmp/interpreters' == make_relative_to_root( '/tmp/interpreters')
def temporary_pex_root(): with temporary_dir() as pex_root, ENV.patch(PEX_ROOT=os.path.realpath(pex_root)) as env: original_isolated = third_party._ISOLATED try: third_party._ISOLATED = None yield os.path.realpath(pex_root), env finally: third_party._ISOLATED = original_isolated
def main(): pparser = pexbin.configure_clp() poptions, args = pparser.parse_args(sys.argv) manifest_file = args[1] manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) reqs = manifest.get('requirements', []) with ENV.patch(PEX_VERBOSE=str(poptions.verbosity), PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT): with TRACER.timed('Building pex'): pex_builder = pexbin.build_pex(reqs, poptions) # Add source files from the manifest for modmap in manifest.get('modules', []): src = modmap.get('src') dst = modmap.get('dest') # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: # Maybe we just can't use hardlinks? Try again. if not pex_builder._copy: pex_builder._copy = True pex_builder.add_source(dereference_symlinks(src), dst) else: raise RuntimeError("Failed to add %s: %s" % (src, err)) # Add resources from the manifest for reqmap in manifest.get('resources', []): src = reqmap.get('src') dst = reqmap.get('dest') pex_builder.add_resource(dereference_symlinks(src), dst) # Add eggs/wheels from the manifest for egg in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(egg) except Exception as err: raise RuntimeError("Failed to add %s: %s" % (egg, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, V=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) shutil.move(tmp_name, poptions.pex_name)
def temporary_pex_root(): # type: () -> Iterator[Tuple[str, Dict[str, str]]] with temporary_dir() as pex_root, ENV.patch( PEX_ROOT=os.path.realpath(pex_root)) as env: original_isolated = third_party._ISOLATED try: third_party._ISOLATED = None yield os.path.realpath(pex_root), env finally: third_party._ISOLATED = original_isolated
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.' ) with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root) as patched_env: # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options) pex_builder.freeze(bytecode_compile=options.compile) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, V=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build( tmp_name, bytecode_compile=options.compile, deterministic_timestamp=not options.use_system_time) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!', V=1) log('Running on platform {} but built for {}'.format( Platform.current(), ', '.join(map(str, options.platforms))), V=1) log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), V=options.verbosity) sys.exit(pex.run(args=list(cmdline), env=patched_env))
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser, resolver_options_builder = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.' ) if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root( options.interpreter_cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) pex_builder.freeze() pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!' ) log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) sys.exit(pex.run(args=list(cmdline)))
def test_project_name_and_version_fallback(tmpdir): # type: (Any) -> None def tmp_path(relpath): # type: (str) -> str return os.path.join(str(tmpdir), relpath) expected_metadata_project_name_and_version = ProjectNameAndVersion("foo", "1.2.3") pkg_info_src = tmp_path("PKG-INFO") with open(pkg_info_src, "w") as fp: fp.write("Name: {}\n".format(expected_metadata_project_name_and_version.project_name)) fp.write("Version: {}\n".format(expected_metadata_project_name_and_version.version)) sdist_path = tmp_path("bar-baz-4.5.6.tar.gz") with tarfile.open(sdist_path, mode="w:gz") as tf: # N.B.: Valid PKG-INFO at an invalid location. tf.add(pkg_info_src, arcname="PKG-INFO") with ENV.patch(PEX_EMIT_WARNINGS="True"), warnings.catch_warnings(record=True) as events: assert project_name_and_version(sdist_path, fallback_to_filename=False) is None assert 1 == len(events) warning = events[0] assert PEXWarning == warning.category assert "bar-baz-4.5.6/PKG-INFO" in str(warning.message) assert ProjectNameAndVersion("bar-baz", "4.5.6") == project_name_and_version( sdist_path, fallback_to_filename=True ) name_and_version = "eggs-7.8.9" pkf_info_path = "{}/PKG-INFO".format(name_and_version) def write_sdist_tgz(extension): sdist_path = tmp_path("{}.{}".format(name_and_version, extension)) with tarfile.open(sdist_path, mode="w:gz") as tf: tf.add(pkg_info_src, arcname=pkf_info_path) return sdist_path assert expected_metadata_project_name_and_version == project_name_and_version( write_sdist_tgz("tar.gz"), fallback_to_filename=False ) assert expected_metadata_project_name_and_version == project_name_and_version( write_sdist_tgz("sdist"), fallback_to_filename=False ) zip_sdist_path = tmp_path("{}.zip".format(name_and_version)) with open_zip(zip_sdist_path, mode="w") as zf: zf.write(pkg_info_src, arcname=pkf_info_path) assert expected_metadata_project_name_and_version == project_name_and_version( zip_sdist_path, fallback_to_filename=False )
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser, resolver_options_builder = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.') if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) pex_builder.freeze(bytecode_compile=options.compile) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, V=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build( tmp_name, bytecode_compile=options.compile, deterministic_timestamp=not options.use_system_time ) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!') log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), V=options.verbosity) sys.exit(pex.run(args=list(cmdline)))
def main(args=None): parser = create_parser() args = parser.parse_args(args) if args.build_distro: if not args.spark_home: die("No spark home given but building a distribution") spark_home = os.path.realpath(os.path.abspath(args.spark_home)) if not os.path.exists(spark_home): die("No spark home given but building a distribution") spark_name = os.path.basename(spark_home) args.spark_home = spark_home args.spark_name = spark_name else: spark_home = None spark_name = None spex_name = args.spex_name spex_file = spex_name + '.spex' with ENV.patch(PEX_VERBOSE=str(args.verbosity)): with TRACER.timed('Building spex'): with TRACER.timed('Building pex'): pex_builder = build_pex(args) with dump_args_as_config(args) as cfg: pex_builder.add_resource(cfg, 'SPEX-INFO') log('Saving PEX file to %s' % spex_file, verbose=args.verbosity) tmp_name = args.spex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, spex_file) if args.build_distro: with TRACER.timed('Building spark package'): spark_distro = uber_distro_location(spark_name) establish_spark_distro(spark_distro, spark_home, spark_name, spex_file, spex_name) log('Spark package built') with TRACER.timed('Building full distribution'): create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args) log('Saved full distribution to %s' % spark_distro) return 0
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity)): pex_builder = build_pex(reqs, options, options_builder) def split_and_strip(entry_point): console_script, entry_point = entry_point.split('=', 2) return console_script.strip(), entry_point.strip() try: console_scripts = dict(split_and_strip(script) for script in self.distribution.entry_points.get('console_scripts', [])) except ValueError: console_scripts = {} target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) # Update cache_dir with pex_root in case this is being called directly. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root( options.interpreter_cache_dir) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): pex_builder = build_pex(reqs, options, options_builder) console_scripts = self.parse_entry_points() target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def main(args=None): args = args or sys.argv[1:] parser, resolver_options_builder = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root( options.interpreter_cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) return 0 if options.platform and Platform.current() not in options.platform: log('WARNING: attempting to run PEX with incompatible platforms!') pex_builder.freeze() log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter) sys.exit(pex.run(args=list(cmdline)))
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) # Update cache_dir with pex_root in case this is being called directly. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): pex_builder = build_pex(reqs, options, options_builder) console_scripts = self.parse_entry_points() target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def test_identify_cwd_isolation_issues_1231(tmpdir): # type: (Any) -> None python36, pip = ensure_python_venv(PY36) polluted_cwd = os.path.join(str(tmpdir), "dir") subprocess.check_call( args=[pip, "install", "--target", polluted_cwd, "pex==2.1.16"]) pex_root = os.path.join(str(tmpdir), "pex_root") with pushd(polluted_cwd), ENV.patch(PEX_ROOT=pex_root): interp = PythonInterpreter.from_binary(python36) interp_info_files = { os.path.join(root, f) for root, _, files in os.walk(pex_root) for f in files if f == PythonInterpreter.INTERP_INFO_FILE } assert 1 == len(interp_info_files) with open(interp_info_files.pop()) as fp: assert interp.binary == json.load(fp)["binary"]
def main(args=None): args = args or sys.argv[1:] parser, resolver_options_builder = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) return 0 if options.platform != Platform.current(): log('WARNING: attempting to run PEX with differing platform!') pex_builder.freeze() log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter) sys.exit(pex.run(args=list(cmdline)))
def bootstrap_pex(entry_point): # type: (str) -> None pex_info = _bootstrap(entry_point) # ENV.PEX_ROOT is consulted by PythonInterpreter and Platform so set that up as early as # possible in the run. with ENV.patch(PEX_ROOT=pex_info.pex_root): if not ENV.PEX_TOOLS and pex_info.venv: try: target = find_compatible_interpreter( interpreter_constraints=pex_info.interpreter_constraints, ) except UnsatisfiableInterpreterConstraintsError as e: die(str(e)) from . import pex venv_pex = ensure_venv(pex.PEX(entry_point, interpreter=target)) os.execv(venv_pex, [venv_pex] + sys.argv[1:]) else: maybe_reexec_pex(pex_info.interpreter_constraints) from . import pex pex.PEX(entry_point).execute()
def main(): pparser, resolver_options_builder = pexbin.configure_clp() poptions, args = pparser.parse_args(sys.argv) manifest_file = args[1] manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) if poptions.pex_root: ENV.set('PEX_ROOT', poptions.pex_root) else: poptions.pex_root = ENV.PEX_ROOT if poptions.cache_dir: poptions.cache_dir = pexbin.make_relative_to_root(poptions.cache_dir) poptions.interpreter_cache_dir = pexbin.make_relative_to_root( poptions.interpreter_cache_dir) reqs = manifest.get('requirements', []) with ENV.patch(PEX_VERBOSE=str(poptions.verbosity)): with TRACER.timed('Building pex'): pex_builder = pexbin.build_pex(reqs, poptions, resolver_options_builder) # Add source files from the manifest for modmap in manifest.get('modules', []): src = modmap.get('src') dst = modmap.get('dest') # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: # Maybe we just can't use hardlinks? Try again. if not pex_builder._copy: pex_builder._copy = True pex_builder.add_source(dereference_symlinks(src), dst) else: raise RuntimeError("Failed to add %s: %s" % (src, err)) # Add resources from the manifest for reqmap in manifest.get('resources', []): src = reqmap.get('src') dst = reqmap.get('dest') pex_builder.add_resource(dereference_symlinks(src), dst) # Add eggs/wheels from the manifest for egg in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(egg) except Exception as err: raise RuntimeError("Failed to add %s: %s" % (egg, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, v=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, poptions.pex_name)
def test_make_relative(): with ENV.patch(PEX_ROOT='/pex_root'): assert '/pex_root/interpreters' == make_relative_to_root('{pex_root}/interpreters') #Verify the user can specify arbitrary absolute paths. assert '/tmp/interpreters' == make_relative_to_root('/tmp/interpreters')
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser = configure_clp() try: separator = args.index("--") args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options = parser.parse_args(args=args) # Ensure the TMPDIR is an absolute path (So subprocesses that change CWD can find it) and # that it exists. tmpdir = os.path.realpath(options.tmpdir) if not os.path.exists(tmpdir): die("The specified --tmpdir does not exist: {}".format(tmpdir)) if not os.path.isdir(tmpdir): die("The specified --tmpdir is not a directory: {}".format(tmpdir)) tempfile.tempdir = os.environ["TMPDIR"] = tmpdir if options.cache_dir: pex_warnings.warn( "The --cache-dir option is deprecated, use --pex-root instead.") if options.pex_root and options.cache_dir != options.pex_root: die("Both --cache-dir and --pex-root were passed with conflicting values. " "Just set --pex-root.") if options.disable_cache: def warn_ignore_pex_root(set_via): pex_warnings.warn( "The pex root has been set via {via} but --disable-cache is also set. " "Ignoring {via} and disabling caches.".format(via=set_via)) if options.cache_dir: warn_ignore_pex_root("--cache-dir") elif options.pex_root: warn_ignore_pex_root("--pex-root") elif os.environ.get("PEX_ROOT"): warn_ignore_pex_root("PEX_ROOT") pex_root = safe_mkdtemp() else: pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.' ) if options.pex_repository and (options.indexes or options.find_links): die('The "--pex-repository" option cannot be used together with the "--index" or ' '"--find-links" options.') with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root, TMPDIR=tmpdir) as patched_env: with TRACER.timed("Building pex"): pex_builder = build_pex(options.requirements, options, cache=ENV.PEX_ROOT) pex_builder.freeze(bytecode_compile=options.compile) interpreter = pex_builder.interpreter pex = PEX(pex_builder.path(), interpreter=interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log("Saving PEX file to %s" % options.pex_name, V=options.verbosity) pex_builder.build( options.pex_name, bytecode_compile=options.compile, deterministic_timestamp=not options.use_system_time, ) if options.seed != Seed.NONE: seed_info = seed_cache(options, pex, verbose=options.seed == Seed.VERBOSE) print(seed_info) else: if not _compatible_with_current_platform(interpreter, options.platforms): log("WARNING: attempting to run PEX with incompatible platforms!", V=1) log( "Running on platform {} but built for {}".format( interpreter.platform, ", ".join(map(str, options.platforms))), V=1, ) log( "Running PEX file at %s with args %s" % (pex_builder.path(), cmdline), V=options.verbosity, ) sys.exit(pex.run(args=list(cmdline), env=patched_env))
def test_pyenv_shims(self, tmpdir): # type: (Any) -> None py35, _, run_pyenv = ensure_python_distribution(PY35) py36 = ensure_python_interpreter(PY36) pyenv_root = str(run_pyenv(["root"]).strip()) pyenv_shims = os.path.join(pyenv_root, "shims") def pyenv_global(*versions): # type: (*str) -> None run_pyenv(["global"] + list(versions)) def pyenv_local(*versions): # type: (*str) -> None run_pyenv(["local"] + list(versions)) @contextmanager def pyenv_shell(*versions): # type: (*str) -> Iterator[None] with environment_as(PYENV_VERSION=":".join(versions)): yield pex_root = os.path.join(str(tmpdir), "pex_root") cwd = safe_mkdir(os.path.join(str(tmpdir), "home", "jake", "project")) with ENV.patch(PEX_ROOT=pex_root) as pex_env, environment_as( PYENV_ROOT=pyenv_root, PEX_PYTHON_PATH=pyenv_shims, **pex_env ), pyenv_shell(), pushd(cwd): pyenv = Pyenv.find() assert pyenv is not None assert pyenv_root == pyenv.root def interpreter_for_shim(shim_name): # type: (str) -> PythonInterpreter binary = os.path.join(pyenv_shims, shim_name) return PythonInterpreter.from_binary(binary, pyenv=pyenv) def assert_shim( shim_name, # type: str expected_binary_path, # type: str ): # type: (...) -> None python = interpreter_for_shim(shim_name) assert expected_binary_path == python.binary def assert_shim_inactive(shim_name): # type: (str) -> None with pytest.raises(PythonInterpreter.IdentificationError): interpreter_for_shim(shim_name) pyenv_global(PY35, PY36) assert_shim("python", py35) assert_shim("python3", py35) assert_shim("python3.5", py35) assert_shim("python3.6", py36) pyenv_global(PY36, PY35) assert_shim("python", py36) assert_shim("python3", py36) assert_shim("python3.6", py36) assert_shim("python3.5", py35) pyenv_local(PY35) assert_shim("python", py35) assert_shim("python3", py35) assert_shim("python3.5", py35) assert_shim_inactive("python3.6") with pyenv_shell(PY36): assert_shim("python", py36) assert_shim("python3", py36) assert_shim("python3.6", py36) assert_shim_inactive("python3.5") with pyenv_shell(PY35, PY36): assert_shim("python", py35) assert_shim("python3", py35) assert_shim("python3.5", py35) assert_shim("python3.6", py36) # The shim pointer is now invalid since python3.5 was uninstalled and so # should be re-read and found invalid. py35_version_dir = os.path.dirname(os.path.dirname(py35)) py35_deleted = "{}.uninstalled".format(py35_version_dir) os.rename(py35_version_dir, py35_deleted) try: assert_shim_inactive("python") assert_shim_inactive("python3") assert_shim_inactive("python3.5") finally: os.rename(py35_deleted, py35_version_dir) assert_shim("python", py35)
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.cache_dir: pex_warnings.warn('The --cache-dir option is deprecated, use --pex-root instead.') if options.pex_root and options.cache_dir != options.pex_root: die('Both --cache-dir and --pex-root were passed with conflicting values. ' 'Just set --pex-root.') if options.disable_cache: def warn_ignore_pex_root(set_via): pex_warnings.warn('The pex root has been set via {via} but --disable-cache is also set. ' 'Ignoring {via} and disabling caches.'.format(via=set_via)) if options.cache_dir: warn_ignore_pex_root('--cache-dir') elif options.pex_root: warn_ignore_pex_root('--pex-root') elif os.environ.get('PEX_ROOT'): warn_ignore_pex_root('PEX_ROOT') pex_root = safe_mkdtemp() else: pex_root = options.cache_dir or options.pex_root or ENV.PEX_ROOT if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.') with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=pex_root) as patched_env: with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, cache=ENV.PEX_ROOT) pex_builder.freeze(bytecode_compile=options.compile) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, V=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build( tmp_name, bytecode_compile=options.compile, deterministic_timestamp=not options.use_system_time ) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!', V=1) log('Running on platform {} but built for {}' .format(Platform.current(), ', '.join(map(str, options.platforms))), V=1) log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), V=options.verbosity) sys.exit(pex.run(args=list(cmdline), env=patched_env))
def main(args=None): args = args[:] if args else sys.argv[1:] args = [pexbin.transform_legacy_arg(arg) for arg in args] pparser = pexbin.configure_clp() try: separator = args.index("--") args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] pparser.add_argument( "--manifest-file", dest="manifest_file", default=None, metavar="FILE", type=str, help="pex_wrapper manifest file.", ) poptions = pparser.parse_args(args=args) manifest_file = poptions.manifest_file manifest_text = open(manifest_file, 'r').read() manifest = parse_manifest(manifest_text) reqs = manifest.get('requirements', []) requirement_configuration = RequirementConfiguration(requirements=reqs) try: resolver_configuration = resolver_options.configure(poptions) except resolver_options.InvalidConfigurationError as e: die(str(e)) try: target_configuration = target_options.configure(poptions) except target_options.InterpreterNotFound as e: die(str(e)) except target_options.InterpreterConstraintsNotSatisfied as e: die(str(e), exit_code=pexbin.CANNOT_SETUP_INTERPRETER) with ENV.patch(PEX_VERBOSE=str(poptions.verbosity), PEX_ROOT=poptions.pex_root or ENV.PEX_ROOT): with TRACER.timed('Building pex'): pex_builder = pexbin.build_pex( requirement_configuration=requirement_configuration, resolver_configuration=resolver_configuration, target_configuration=target_configuration, options=poptions) # Add source files from the manifest for modmap in manifest.get('modules', []): src = modmap.get('src') dst = modmap.get('dest') # NOTE(agallagher): calls the `add_source` and `add_resource` below # hard-link the given source into the PEX temp dir. Since OS X and # Linux behave different when hard-linking a source that is a # symbolic link (Linux does *not* follow symlinks), resolve any # layers of symlinks here to get consistent behavior. try: pex_builder.add_source(dereference_symlinks(src), dst) except OSError as err: # Maybe we just can't use hardlinks? Try again. if not pex_builder._copy: pex_builder._copy = True pex_builder.add_source(dereference_symlinks(src), dst) else: raise RuntimeError("Failed to add %s: %s" % (src, err)) # Add resources from the manifest for reqmap in manifest.get('resources', []): src = reqmap.get('src') dst = reqmap.get('dest') pex_builder.add_source(dereference_symlinks(src), dst) # Add eggs/wheels from the manifest for egg in manifest.get('prebuiltLibraries', []): try: pex_builder.add_dist_location(egg) except Exception as err: raise RuntimeError("Failed to add %s: %s" % (egg, err)) # TODO(mikekap): Do something about manifest['nativeLibraries']. pexbin.log('Saving PEX file to %s' % poptions.pex_name, V=poptions.verbosity) tmp_name = poptions.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) shutil.move(tmp_name, poptions.pex_name)