def _wrap_coverage(self, runner, *args): if not self._vars.PEX_COVERAGE and self._vars.PEX_COVERAGE_FILENAME is None: return runner(*args) try: import coverage except ImportError: die('Could not bootstrap coverage module, aborting.') pex_coverage_filename = self._vars.PEX_COVERAGE_FILENAME if pex_coverage_filename is not None: cov = coverage.coverage(data_file=pex_coverage_filename) else: cov = coverage.coverage(data_suffix=True) TRACER.log('Starting coverage.') cov.start() try: return runner(*args) finally: TRACER.log('Stopping coverage') cov.stop() # TODO(wickman) Post-process coverage to elide $PEX_ROOT and make # the report more useful/less noisy. #89 if pex_coverage_filename: cov.save() else: cov.report(show_missing=False, ignore_errors=True, file=sys.stdout)
def _execute(self): force_interpreter = self._vars.PEX_INTERPRETER # N.B.: This is set in `__main__.py` of the executed PEX by `PEXBuilder` when we've been # executed from within a PEX zip file in `--unzip` mode. We replace `sys.argv[0]` to avoid # confusion and allow the user code we hand off to to provide useful messages and fully valid # re-execs that always re-directed through the PEX file. sys.argv[0] = os.environ.pop('__PEX_EXE__', sys.argv[0]) self._clean_environment(strip_pex_env=self._pex_info.strip_pex_env) if force_interpreter: TRACER.log('PEX_INTERPRETER specified, dropping into interpreter') return self.execute_interpreter() if self._pex_info_overrides.script and self._pex_info_overrides.entry_point: die('Cannot specify both script and entry_point for a PEX!') if self._pex_info.script and self._pex_info.entry_point: die('Cannot specify both script and entry_point for a PEX!') if self._pex_info_overrides.script: return self.execute_script(self._pex_info_overrides.script) elif self._pex_info_overrides.entry_point: return self.execute_entry(self._pex_info_overrides.entry_point) elif self._pex_info.script: return self.execute_script(self._pex_info.script) elif self._pex_info.entry_point: return self.execute_entry(self._pex_info.entry_point) else: TRACER.log('No entry point specified, dropping into interpreter') return self.execute_interpreter()
def _execute(self): force_interpreter = self._vars.PEX_INTERPRETER self.clean_environment() if force_interpreter: TRACER.log('PEX_INTERPRETER specified, dropping into interpreter') return self.execute_interpreter() if self._pex_info_overrides.script and self._pex_info_overrides.entry_point: die('Cannot specify both script and entry_point for a PEX!') if self._pex_info.script and self._pex_info.entry_point: die('Cannot specify both script and entry_point for a PEX!') if self._pex_info_overrides.script: return self.execute_script(self._pex_info_overrides.script) elif self._pex_info_overrides.entry_point: return self.execute_entry(self._pex_info_overrides.entry_point) elif self._pex_info.script: return self.execute_script(self._pex_info.script) elif self._pex_info.entry_point: return self.execute_entry(self._pex_info.entry_point) else: TRACER.log('No entry point specified, dropping into interpreter') return self.execute_interpreter()
def execute_interpreter(self): args = sys.argv[1:] if args: # NB: We take care here to setup sys.argv to match how CPython does it for each case. arg = args[0] if arg == '-c': content = args[1] sys.argv = ['-c'] + args[2:] self.execute_content('-c <cmd>', content, argv0='-c') elif arg == '-m': module = args[1] sys.argv = args[1:] self.execute_module(module) else: try: if arg == '-': content = sys.stdin.read() else: with open(arg) as fp: content = fp.read() except IOError as e: die("Could not open %s in the environment [%s]: %s" % (arg, sys.argv[0], e)) sys.argv = args self.execute_content(arg, content) else: self.demote_bootstrap() import code code.interact()
def find_compatible_interpreters(pex_python_path=None, compatibility_constraints=None): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH. """ if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: print("Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary, file=sys.stderr) if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.') else: # We may have been invoked with a specific interpreter not on the $PATH, make sure our # sys.executable is included as a candidate in this case. interpreters = OrderedSet([PythonInterpreter.get()]) # Add all qualifying interpreters found in $PATH. interpreters.update(PythonInterpreter.all()) return list( matched_interpreters(interpreters, compatibility_constraints) if compatibility_constraints else interpreters )
def _get_int(self, variable, default=None): try: return int(self._environ[variable]) except ValueError: die('Invalid value for %s, must be an integer, got %r' % (variable, self._environ[variable])) except KeyError: return self._defaulted(default)
def _select_path_interpreter(path=None, compatibility_constraints=None): compatible_interpreters = find_compatible_interpreters( path=path, compatibility_constraints=compatibility_constraints) if not compatible_interpreters: die('Failed to find compatible interpreter on path {} for constraints: {}' .format(path or os.getenv('PATH'), compatibility_constraints)) return _select_interpreter(compatible_interpreters)
def resolve_or_die(interpreter, requirement, options): resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) interpreter = resolve(interpreter, requirement) if interpreter is None: die('Could not find compatible interpreter that meets requirement %s' % requirement, CANNOT_SETUP_INTERPRETER) return interpreter
def run(self): parser, options_builder = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die('Must not specify entry point, script or output file to --pex-args, given: {}' .format(' '.join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend( (script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') pex_specs.append( (name if name in console_scripts else None, target)) # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir ] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append('-v') for script_name, target in pex_specs: cmd = args + ['--output-file', target] if script_name: log.info('Writing %s to %s' % (script_name, target)) cmd += ['--script', script_name] else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: die( 'Failed to create pex via {}:\n{}'.format( ' '.join(cmd), stderr.decode('utf-8')), result)
def execute(self): # type: () -> None """Execute the PEX. This function makes assumptions that it is the last function called by the interpreter. """ teardown_verbosity = self._vars.PEX_TEARDOWN_VERBOSE # N.B.: This is set in `__main__.py` of the executed PEX by `PEXBuilder` when we've been # executed from within a PEX zip file in `--unzip` mode. We replace `sys.argv[0]` to avoid # confusion and allow the user code we hand off to to provide useful messages and fully # valid re-execs that always re-directed through the PEX file. sys.argv[0] = os.environ.pop("__PEX_EXE__", sys.argv[0]) try: if self._vars.PEX_TOOLS: try: from pex.tools import main as tools except ImportError as e: die( "The PEX_TOOLS environment variable was set, but this PEX was not built " "with tools (Re-build the PEX file with `pex --include-tools ...`):" " {}".format(e) ) exit_value = tools.main(pex=self, pex_prog_path=sys.argv[0]) else: self.activate() exit_value = self._wrap_coverage(self._wrap_profiling, self._execute) sys.exit(exit_value) except Exception: # Allow the current sys.excepthook to handle this app exception before we tear things # down in finally, then reraise so that the exit status is reflected correctly. sys.excepthook(*sys.exc_info()) raise except SystemExit as se: # Print a SystemExit error message, avoiding a traceback in python3. # This must happen here, as sys.stderr is about to be torn down if not isinstance(se.code, int) and se.code is not None: # type: ignore[unreachable] print(se.code, file=sys.stderr) # type: ignore[unreachable] raise finally: # squash all exceptions on interpreter teardown -- the primary type here are # atexit handlers failing to run because of things such as: # http://stackoverflow.com/questions/2572172/referencing-other-modules-in-atexit if not teardown_verbosity: sys.stderr.flush() sys.stderr = open(os.devnull, "w") if PY3: # Python 3 warns about unclosed resources. In this case we intentionally do not # close `/dev/null` since we want all stderr to flow there until the latest # stages of Python interpreter shutdown when the Python runtime will `del` the # open file and thus finally close the underlying file descriptor. As such, # suppress the warning. warnings.filterwarnings( action="ignore", message=r"unclosed file {}".format(re.escape(str(sys.stderr))), category=ResourceWarning, ) sys.excepthook = lambda *a, **kw: None
def get_interpreter(python_interpreter, interpreter_cache_dir, repos, use_wheel): interpreter = None if python_interpreter: if os.path.exists(python_interpreter): interpreter = PythonInterpreter.from_binary(python_interpreter) else: interpreter = PythonInterpreter.from_env(python_interpreter) if interpreter is None: die('Failed to find interpreter: %s' % python_interpreter) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, interpreter_cache_dir, repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def interpreter_from_options(options): interpreter = None if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) else: interpreter = PythonInterpreter.from_env(options.python) if interpreter is None: die('Failed to find interpreter: %s' % options.python) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and options.use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def find_compatible_interpreters(pex_python_path, compatibility_constraints): """Find all compatible interpreters on the system within the supplied constraints and use PEX_PYTHON_PATH if it is set. If not, fall back to interpreters on $PATH. """ if pex_python_path: interpreters = [] for binary in pex_python_path.split(os.pathsep): try: interpreters.append(PythonInterpreter.from_binary(binary)) except Executor.ExecutionError: print( "Python interpreter %s in PEX_PYTHON_PATH failed to load properly." % binary, file=sys.stderr) if not interpreters: die('PEX_PYTHON_PATH was defined, but no valid interpreters could be identified. Exiting.' ) else: if not os.getenv('PATH', ''): # no $PATH, use sys.executable interpreters = [PythonInterpreter.get()] else: # get all qualifying interpreters found in $PATH interpreters = PythonInterpreter.all() return list( matched_interpreters(interpreters, compatibility_constraints ) if compatibility_constraints else interpreters)
def run(self): parser = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die("Must not specify entry point, script or output file to --pex-args, given: {}" .format(" ".join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, "dist") console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend( (script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + "-" + version + ".pex") pex_specs.append( (name if name in console_scripts else None, target)) args = ["-m", "pex", package_dir] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append("-v") for script_name, target in pex_specs: pex_cmd = args + ["--output-file", target] if script_name: log.info("Writing %s to %s" % (script_name, target)) pex_cmd += ["--script", script_name] else: # The package has no namesake entry point, so build an environment pex. log.info("Writing environment pex into %s" % target) cmd, process = PythonInterpreter.get().open_process( args=pex_cmd, stderr=subprocess.PIPE, # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. pythonpath=sys.path, ) _, stderr = process.communicate() result = process.returncode if result != 0: die( "Failed to create pex via {}:\n{}".format( " ".join(cmd), stderr.decode("utf-8")), result, )
def _get_int(self, variable): # type: (str) -> int value = self._get_string(variable) try: return int(value) except ValueError: die("Invalid value for %s, must be an integer, got %r" % (variable, self._environ[variable]))
def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter
def to_python_interpreter(full_path_or_basename): if os.path.isfile(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interp = PythonInterpreter.from_env(full_path_or_basename) if interp is None: die("Failed to find interpreter: %s" % full_path_or_basename) return interp
def _select_pex_python_interpreter(pex_python, compatibility_constraints=None): compatible_interpreters = _filter_compatible_interpreters( _find_pex_python(pex_python), compatibility_constraints=compatibility_constraints) if not compatible_interpreters: die('Failed to find a compatible PEX_PYTHON={} for constraints: {}'. format(pex_python, compatibility_constraints)) return _select_interpreter(compatible_interpreters)
def _select_pex_python_interpreter(pex_python, compatibility_constraints=None): compatible_interpreters_iter = _compatible_interpreters_iter( _iter_pex_python(pex_python), compatibility_constraints=compatibility_constraints) selected = _select_interpreter(compatible_interpreters_iter) if not selected: die('Failed to find a compatible PEX_PYTHON={} for constraints: {}'. format(pex_python, compatibility_constraints)) return selected
def validate_constraints(constraints): # TODO: add check to see if constraints are mutually exclusive (bad) so no time is wasted: # https://github.com/pantsbuild/pex/issues/432 for req in constraints: # Check that the compatibility requirements are well-formed. try: PythonIdentity.parse_requirement(req) except ValueError as e: die("Compatibility requirements are not formatted properly: %s" % str(e))
def _get_int(self, variable, default=None): # type: (str, Optional[int]) -> Optional[int] try: return int(self._environ[variable]) except ValueError: die("Invalid value for %s, must be an integer, got %r" % (variable, self._environ[variable])) except KeyError: return self._defaulted(default) # type: ignore[return-value]
def resolve_or_die(interpreter, requirement, options): resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) interpreter = resolve(interpreter, requirement) if interpreter is None: die( 'Could not find compatible interpreter that meets requirement %s' % requirement, CANNOT_SETUP_INTERPRETER) return interpreter
def _maybe_get_bool(self, variable): # type: (str) -> Optional[bool] value = self._maybe_get_string(variable) if value is None: return None if value.lower() in ("0", "false"): return False if value.lower() in ("1", "true"): return True die("Invalid value for %s, must be 0/1/false/true, got %r" % (variable, value))
def _select_path_interpreter(path=None, compatibility_constraints=None): compatible_interpreters_iter = iter_compatible_interpreters( path=path, compatibility_constraints=compatibility_constraints) try: return _select_interpreter(compatible_interpreters_iter) except UnsatisfiableInterpreterConstraintsError as e: die( e.create_message( "Failed to find compatible interpreter on path {path}.".format( path=path or os.getenv("PATH"))))
def _get_bool(self, variable, default=False): value = self._environ.get(variable) if value is not None: if value.lower() in ('0', 'false'): return False elif value.lower() in ('1', 'true'): return True else: die('Invalid value for %s, must be 0/1/false/true, got %r' % (variable, value)) else: return self._defaulted(default)
def _get_bool(self, variable, default=False): # type: (str, Optional[bool]) -> Optional[bool] value = self._environ.get(variable) if value is None: return self._defaulted(default) # type: ignore[return-value] if value.lower() in ("0", "false"): return False if value.lower() in ("1", "true"): return True die("Invalid value for %s, must be 0/1/false/true, got %r" % (variable, value))
def _get_bool(self, variable, default=False): value = self._environ.get(variable) if value is not None: if value.lower() in ("0", "false"): return False elif value.lower() in ("1", "true"): return True else: die("Invalid value for %s, must be 0/1/false/true, got %r" % (variable, value)) else: return self._defaulted(default)
def _select_pex_python_interpreter(pex_python, compatibility_constraints=None): compatible_interpreters_iter = _compatible_interpreters_iter( _iter_pex_python(pex_python), compatibility_constraints=compatibility_constraints) try: return _select_interpreter(compatible_interpreters_iter) except UnsatisfiableInterpreterConstraintsError as e: die( e.create_message( "Failed to find a compatible PEX_PYTHON={pex_python}.".format( pex_python=pex_python)))
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.' ) with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root) as patched_env: # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options) pex_builder.freeze(bytecode_compile=options.compile) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, V=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build( tmp_name, bytecode_compile=options.compile, deterministic_timestamp=not options.use_system_time) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!', V=1) log('Running on platform {} but built for {}'.format( Platform.current(), ', '.join(map(str, options.platforms))), V=1) log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), V=options.verbosity) sys.exit(pex.run(args=list(cmdline), env=patched_env))
def _select_interpreter(pex_python_path=None, compatibility_constraints=None): compatible_interpreters = find_compatible_interpreters( pex_python_path=pex_python_path, compatibility_constraints=compatibility_constraints) if not compatible_interpreters: die('Failed to find compatible interpreter for constraints: %s' % str(compatibility_constraints)) # TODO: https://github.com/pantsbuild/pex/issues/430 target = min(compatible_interpreters).binary if os.path.exists(target): return target
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser, resolver_options_builder = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.' ) if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root( options.interpreter_cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) pex_builder.freeze() pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, v=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!' ) log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), v=options.verbosity) sys.exit(pex.run(args=list(cmdline)))
def run(self): parser, options_builder = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die('Must not specify entry point, script or output file to --pex-args, given: {}' .format(' '.join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') pex_specs.append((name if name in console_scripts else None, target)) # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append('-v') for script_name, target in pex_specs: cmd = args + ['--output-file', target] if script_name: log.info('Writing %s to %s' % (script_name, target)) cmd += ['--script', script_name] else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')), result)
def main(args=None): args = args[:] if args else sys.argv[1:] args = [transform_legacy_arg(arg) for arg in args] parser, resolver_options_builder = configure_clp() try: separator = args.index('--') args, cmdline = args[:separator], args[separator + 1:] except ValueError: args, cmdline = args, [] options, reqs = parser.parse_args(args=args) if options.python and options.interpreter_constraint: die('The "--python" and "--interpreter-constraint" options cannot be used together.') if options.pex_root: ENV.set('PEX_ROOT', options.pex_root) else: options.pex_root = ENV.PEX_ROOT # If option not specified fallback to env variable. # Don't alter cache if it is disabled. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) with ENV.patch(PEX_VERBOSE=str(options.verbosity)): with TRACER.timed('Building pex'): pex_builder = build_pex(reqs, options, resolver_options_builder) pex_builder.freeze(bytecode_compile=options.compile) pex = PEX(pex_builder.path(), interpreter=pex_builder.interpreter, verify_entry_point=options.validate_ep) if options.pex_name is not None: log('Saving PEX file to %s' % options.pex_name, V=options.verbosity) tmp_name = options.pex_name + '~' safe_delete(tmp_name) pex_builder.build( tmp_name, bytecode_compile=options.compile, deterministic_timestamp=not options.use_system_time ) os.rename(tmp_name, options.pex_name) else: if not _compatible_with_current_platform(options.platforms): log('WARNING: attempting to run PEX with incompatible platforms!') log('Running PEX file at %s with args %s' % (pex_builder.path(), cmdline), V=options.verbosity) sys.exit(pex.run(args=list(cmdline)))
def execute_content(cls, name, content, argv0=None): argv0 = argv0 or name try: ast = compile(content, name, 'exec', flags=0, dont_inherit=1) except SyntaxError: die('Unable to parse %s. PEX script support only supports Python scripts.' % name) cls.demote_bootstrap() from pex.compatibility import exec_function sys.argv[0] = argv0 globals_map = globals().copy() globals_map['__name__'] = '__main__' globals_map['__file__'] = name exec_function(ast, globals_map)
def build_pex(args): with TRACER.timed('Resolving interpreter', V=2): interpreter = _establish_interpreter(args) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=_PREAMBLE) pex_info = pex_builder.info pex_info.zip_safe = False pex_info.always_write_cache = True pex_info.inherit_path = False resolver_option_builder = _establish_resolver_options(args) reqs = args.reqs resolvables = [Resolvable.get(req, resolver_option_builder) for req in reqs] for requirements_txt in args.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=args.platform) _add_spex_deps(resolvables, pex_builder, resolver_option_builder=resolver_option_builder) if not args.disable_cache: resolver = CachingResolver(args.cache_dir, args.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) resolveds = [] with TRACER.timed('Resolving distributions'): try: resolveds = resolver.resolve(resolvables) except Unsatisfiable as exception: die(exception) for dist in resolveds: log(' %s' % dist, verbose=args.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) pex_builder.set_entry_point('spex:spex') if args.python_shebang: pex_builder.set_shebang(args.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreter', V=2): interpreter = interpreter_from_options(options) if interpreter is None: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) resolver_kwargs = dict(interpreter=interpreter, platform=options.platform) if options.cache_dir: resolver = CachingResolver(options.cache_dir, options.cache_ttl, **resolver_kwargs) else: resolver = Resolver(**resolver_kwargs) with TRACER.timed('Resolving distributions'): resolveds = resolver.resolve(resolvables) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def main(args=None): parser = create_parser() args = parser.parse_args(args) if args.build_distro: if not args.spark_home: die("No spark home given but building a distribution") spark_home = os.path.realpath(os.path.abspath(args.spark_home)) if not os.path.exists(spark_home): die("No spark home given but building a distribution") spark_name = os.path.basename(spark_home) args.spark_home = spark_home args.spark_name = spark_name else: spark_home = None spark_name = None spex_name = args.spex_name spex_file = spex_name + '.spex' with ENV.patch(PEX_VERBOSE=str(args.verbosity)): with TRACER.timed('Building spex'): with TRACER.timed('Building pex'): pex_builder = build_pex(args) with dump_args_as_config(args) as cfg: pex_builder.add_resource(cfg, 'SPEX-INFO') log('Saving PEX file to %s' % spex_file, verbose=args.verbosity) tmp_name = args.spex_name + '~' safe_delete(tmp_name) pex_builder.build(tmp_name) os.rename(tmp_name, spex_file) if args.build_distro: with TRACER.timed('Building spark package'): spark_distro = uber_distro_location(spark_name) establish_spark_distro(spark_distro, spark_home, spark_name, spex_file, spex_name) log('Spark package built') with TRACER.timed('Building full distribution'): create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args) log('Saved full distribution to %s' % spark_distro) return 0
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity)): pex_builder = build_pex(reqs, options, options_builder) def split_and_strip(entry_point): console_script, entry_point = entry_point.split('=', 2) return console_script.strip(), entry_point.strip() try: console_scripts = dict(split_and_strip(script) for script in self.distribution.entry_points.get('console_scripts', [])) except ValueError: console_scripts = {} target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def _resolve(self, working_set, reqs): reqs = reqs[:] unresolved_reqs = set() resolveds = set() environment = self._target_interpreter_env.copy() environment['extra'] = list(set(itertools.chain(*(req.extras for req in reqs)))) # Resolve them one at a time so that we can figure out which ones we need to elide should # there be an interpreter incompatibility. for req in reqs: if req.marker and not req.marker.evaluate(environment=environment): TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req) continue with TRACER.timed('Resolving %s' % req, V=2): try: resolveds.update(working_set.resolve([req], env=self)) except DistributionNotFound as e: TRACER.log('Failed to resolve a requirement: %s' % e) unresolved_reqs.add(e.req.project_name) if e.requirers: unresolved_reqs.update(e.requirers) unresolved_reqs = set([req.lower() for req in unresolved_reqs]) if unresolved_reqs: TRACER.log('Unresolved requirements:') for req in unresolved_reqs: TRACER.log(' - %s' % req) TRACER.log('Distributions contained within this pex:') if not self._pex_info.distributions: TRACER.log(' None') else: for dist in self._pex_info.distributions: TRACER.log(' - %s' % dist) if not self._pex_info.ignore_errors: die( 'Failed to execute PEX file, missing %s compatible dependencies for:\n%s' % ( Platform.current(), '\n'.join(str(r) for r in unresolved_reqs) ) ) return resolveds
def run(self): name = self.distribution.get_name() version = self.distribution.get_version() parser, options_builder = configure_clp() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') options, reqs = parser.parse_args(self.pex_args) # Update cache_dir with pex_root in case this is being called directly. if options.cache_dir: options.cache_dir = make_relative_to_root(options.cache_dir) options.interpreter_cache_dir = make_relative_to_root(options.interpreter_cache_dir) if options.entry_point or options.script: die('Must not specify entry_point or script to --pex-args') reqs = [package_dir] + reqs with ENV.patch(PEX_VERBOSE=str(options.verbosity), PEX_ROOT=options.pex_root): pex_builder = build_pex(reqs, options, options_builder) console_scripts = self.parse_entry_points() target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') if self.bdist_all: # Write all entry points into unversioned pex files. for script_name in console_scripts: target = os.path.join(self.bdist_dir, script_name) log.info('Writing %s to %s' % (script_name, target)) self._write(pex_builder, target, script=script_name) elif name in console_scripts: # The package has a namesake entry point, so use it. log.info('Writing %s to %s' % (name, target)) self._write(pex_builder, target, script=name) else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) self._write(pex_builder, target, script=None)
def parse_entry_points(self): def parse_entry_point_name(entry_point): script_name = entry_point.split('=', 1)[0] return script_name.strip() raw_entry_points = self.distribution.entry_points if isinstance(raw_entry_points, string): parser = ConfigParser() parser.readfp(StringIO(to_unicode(raw_entry_points))) if parser.has_section('console_scripts'): return tuple(parser.options('console_scripts')) elif isinstance(raw_entry_points, dict): try: return tuple(parse_entry_point_name(script) for script in raw_entry_points.get('console_scripts', [])) except ValueError: pass elif raw_entry_points is not None: die('When entry_points is provided, it must be a string or dict.') return ()
def _establish_interpreter(args): if args.python: if os.path.exists(args.python): interpreter = PythonInterpreter.from_binary(args.python) else: interpreter = PythonInterpreter.from_env(args.python) if interpreter is None: die('Failed to find interpreter: %s' % args.python) else: interpreter = PythonInterpreter.get() with TRACER.timed('Setting up interpreter %s' % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, args.interpreter_cache_dir, args.repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and args.use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def parse_entry_points(self): def split_and_strip(entry_point): console_script, entry_point = entry_point.split('=', 2) return console_script.strip(), entry_point.strip() raw_entry_points = self.distribution.entry_points if isinstance(raw_entry_points, string): parser = ConfigParser() parser.readfp(StringIO(raw_entry_points)) if parser.has_section('console_scripts'): return dict(parser.items('console_scripts')) elif isinstance(raw_entry_points, dict): try: return dict(split_and_strip(script) for script in raw_entry_points.get('console_scripts', [])) except ValueError: pass elif raw_entry_points is not None: die('When entry_points is provided, it must be a string or dict.') return {}
def create_distro_tarball(spark_distro, spark_name, spex_file, spex_name, args): with tarfile.open(spex_name + '-distro.tar.bz2', 'w:bz2') as tarball: log('Including spex file [%s] in full distribution' % spex_file) tarball.add(spex_file, arcname=os.path.join(spex_name, spex_file)) log('Including spark package [%s] in full distribution' % spark_distro) tarball.add(spark_distro, arcname=os.path.join(spex_name, spark_name + '.tar.bz2')) additional_artifacts = ( ('files', args.files), ('py_files', args.py_files), ('jars', args.jars)) log('Including additional artifacts in full distribution') for arcdir, artifacts in additional_artifacts: for artifact in (os.path.realpath(a) for a in artifacts): arcfile = os.path.basename(artifact) if os.path.exists(artifact): arcname = os.path.join(os.path.join(spex_name, arcdir), arcfile) log('Including ' + arcname) tarball.add(artifact, arcname=arcname) else: die('You asked me to include a %s that does not exist [%s]' % (arcdir, artifact))
def interpreter_from_options(options): interpreter = None if options.python: if os.path.exists(options.python): interpreter = PythonInterpreter.from_binary(options.python) else: interpreter = PythonInterpreter.from_env(options.python) if interpreter is None: die("Failed to find interpreter: %s" % options.python) else: interpreter = PythonInterpreter.get() with TRACER.timed("Setting up interpreter %s" % interpreter.binary, V=2): resolve = functools.partial(resolve_interpreter, options.interpreter_cache_dir, options.repos) # resolve setuptools interpreter = resolve(interpreter, SETUPTOOLS_REQUIREMENT) # possibly resolve wheel if interpreter and options.use_wheel: interpreter = resolve(interpreter, WHEEL_REQUIREMENT) return interpreter
def _select_pex_python_interpreter(target_python, compatibility_constraints=None): target = find_in_path(target_python) if not target: die('Failed to find interpreter specified by PEX_PYTHON: %s' % target) if compatibility_constraints: pi = PythonInterpreter.from_binary(target) if not list(matched_interpreters([pi], compatibility_constraints)): die('Interpreter specified by PEX_PYTHON (%s) is not compatible with specified ' 'interpreter constraints: %s' % (target, str(compatibility_constraints))) if not os.path.exists(target): die('Target interpreter specified by PEX_PYTHON %s does not exist. Exiting.' % target) return target
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): interpreters = [ get_interpreter(interpreter, options.interpreter_cache_dir, options.repos, options.use_wheel) for interpreter in options.python or [None] ] if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = _lowest_version_interpreter(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.inherit_path = options.inherit_path resolvables = [Resolvable.get(arg, resolver_option_builder) for arg in args] for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, resolver_option_builder)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, resolver_option_builder): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platform, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed) for dist in resolveds: log(' %s' % dist, v=options.verbosity) pex_builder.add_distribution(dist) pex_builder.add_requirement(dist.as_requirement()) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder
def build_pex(args, options, resolver_option_builder): with TRACER.timed('Resolving interpreters', V=2): def to_python_interpreter(full_path_or_basename): if os.path.exists(full_path_or_basename): return PythonInterpreter.from_binary(full_path_or_basename) else: interpreter = PythonInterpreter.from_env(full_path_or_basename) if interpreter is None: die('Failed to find interpreter: %s' % full_path_or_basename) return interpreter interpreters = [to_python_interpreter(interp) for interp in options.python or [sys.executable]] if options.interpreter_constraint: # NB: options.python and interpreter constraints cannot be used together, so this will not # affect usages of the interpreter(s) specified by the "--python" command line flag. constraints = options.interpreter_constraint validate_constraints(constraints) if options.rc_file or not ENV.PEX_IGNORE_RCFILES: rc_variables = Variables.from_rc(rc=options.rc_file) pex_python_path = rc_variables.get('PEX_PYTHON_PATH', '') else: pex_python_path = "" interpreters = find_compatible_interpreters(pex_python_path, constraints) if not interpreters: die('Could not find compatible interpreter', CANNOT_SETUP_INTERPRETER) try: with open(options.preamble_file) as preamble_fd: preamble = preamble_fd.read() except TypeError: # options.preamble_file is None preamble = None interpreter = min(interpreters) pex_builder = PEXBuilder(path=safe_mkdtemp(), interpreter=interpreter, preamble=preamble) def walk_and_do(fn, src_dir): src_dir = os.path.normpath(src_dir) for root, dirs, files in os.walk(src_dir): for f in files: src_file_path = os.path.join(root, f) dst_path = os.path.relpath(src_file_path, src_dir) fn(src_file_path, dst_path) for directory in options.sources_directory: walk_and_do(pex_builder.add_source, directory) for directory in options.resources_directory: walk_and_do(pex_builder.add_resource, directory) pex_info = pex_builder.info pex_info.zip_safe = options.zip_safe pex_info.pex_path = options.pex_path pex_info.always_write_cache = options.always_write_cache pex_info.ignore_errors = options.ignore_errors pex_info.emit_warnings = options.emit_warnings pex_info.inherit_path = options.inherit_path if options.interpreter_constraint: for ic in options.interpreter_constraint: pex_builder.add_interpreter_constraint(ic) resolvables = resolvables_from_iterable(args, resolver_option_builder, interpreter=interpreter) for requirements_txt in options.requirement_files: resolvables.extend(requirements_from_file(requirements_txt, builder=resolver_option_builder, interpreter=interpreter)) # pip states the constraints format is identical tor requirements # https://pip.pypa.io/en/stable/user_guide/#constraints-files for constraints_txt in options.constraint_files: constraints = [] for r in requirements_from_file(constraints_txt, builder=resolver_option_builder, interpreter=interpreter): r.is_constraint = True constraints.append(r) resolvables.extend(constraints) with TRACER.timed('Resolving distributions'): try: resolveds = resolve_multi(resolvables, interpreters=interpreters, platforms=options.platforms, cache=options.cache_dir, cache_ttl=options.cache_ttl, allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) for resolved_dist in resolveds: log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), V=options.verbosity) pex_builder.add_distribution(resolved_dist.distribution) pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) if options.entry_point and options.script: die('Must specify at most one entry point or script.', INVALID_OPTIONS) if options.entry_point: pex_builder.set_entry_point(options.entry_point) elif options.script: pex_builder.set_script(options.script) if options.python_shebang: pex_builder.set_shebang(options.python_shebang) return pex_builder