def test_executor_execute_nonzero(exit_code): # type: (int) -> None with pytest.raises(Executor.NonZeroExit) as exc: Executor.execute("exit %s" % exit_code, shell=True) if exit_code > 0: assert exc.value.exit_code == exit_code
def run(self): if self._installed is not None: return self._installed with TRACER.timed('Installing %s' % self._install_tmp, V=2): env = self._interpreter.sanitized_environment() mixins = OrderedSet(['setuptools'] + self.mixins) env['PYTHONPATH'] = os.pathsep.join(third_party.expose(mixins)) env['__PEX_UNVENDORED__'] = '1' command = [self._interpreter.binary, '-s', '-'] + self.setup_command() try: Executor.execute(command, env=env, cwd=self._source_dir, stdin_payload=self.setup_py_wrapper.encode('ascii')) self._installed = True except Executor.NonZeroExit as e: self._installed = False name = os.path.basename(self._source_dir) print('**** Failed to install %s (caused by: %r\n):' % (name, e), file=sys.stderr) print('stdout:\n%s\nstderr:\n%s\n' % (e.stdout, e.stderr), file=sys.stderr) return self._installed return self._installed
def test_executor_execute_dir(): with temporary_dir() as temp_dir: test_dir = os.path.realpath(os.path.join(temp_dir, 'tmp')) safe_mkdir(test_dir) assert os.path.isdir(test_dir) with pytest.raises(Executor.ExecutionError) as e: Executor.execute(test_dir) assert test_dir in str(e)
def test_executor_execute_stdio(): with temporary_dir() as tmp: with open(os.path.join(tmp, 'stdout'), 'w+b') as fake_stdout: with open(os.path.join(tmp, 'stderr'), 'w+b') as fake_stderr: Executor.execute('/bin/echo -n TEST | tee /dev/stderr', shell=True, stdout=fake_stdout, stderr=fake_stderr) fake_stdout.seek(0) fake_stderr.seek(0) assert fake_stdout.read().decode('utf-8') == 'TEST' assert fake_stderr.read().decode('utf-8') == 'TEST'
def test_executor_open_process_communicate(): process = Executor.open_process(["/bin/echo", "-n", "hello"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert stdout.decode("utf-8") == "hello" assert stderr.decode("utf-8") == ""
def compile(self, root, relpaths): """Compiles the given python source files using this compiler's interpreter. :param string root: The root path all the source files are found under. :param list relpaths: The realtive paths from the `root` of the source files to compile. :returns: A list of relative paths of the compiled bytecode files. :raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files. """ with named_temporary_file() as fp: fp.write( to_bytes(_COMPILER_MAIN % { 'root': root, 'relpaths': relpaths }, encoding='utf-8')) fp.flush() try: out, _ = Executor.execute( [self._interpreter.binary, '-sE', fp.name]) except Executor.NonZeroExit as e: raise self.CompilationFailure( 'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr)) return out.splitlines()
def test_executor_open_process_communicate(): process = Executor.open_process(['/bin/echo', '-n', 'hello'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = process.communicate() assert stdout.decode('utf-8') == 'hello' assert stderr.decode('utf-8') == ''
def test_executor_exceptions_nonzeroexit(cmd): exc = Executor.NonZeroExit(cmd=cmd, exit_code=TEST_CODE, stdout=TEST_STDOUT, stderr=TEST_STDERR) assert exc.executable == TEST_EXECUTABLE assert exc.cmd == cmd assert exc.exit_code == TEST_CODE assert exc.stdout == TEST_STDOUT assert exc.stderr == TEST_STDERR
def run(self, args=(), with_chroot=False, blocking=True, setsid=False, env=None, **kwargs): """Run the PythonEnvironment in an interpreter in a subprocess. :keyword args: Additional arguments to be passed to the application being invoked by the environment. :keyword with_chroot: Run with cwd set to the environment's working directory. :keyword blocking: If true, return the return code of the subprocess. If false, return the Popen object of the invoked subprocess. :keyword setsid: If true, run the PEX in a separate operating system session. :keyword env: An optional environment dict to use as the PEX subprocess environment. If none is passed, the ambient environment is inherited. Remaining keyword arguments are passed directly to subprocess.Popen. """ if env is not None: # If explicit env vars are passed, we don't want clean any of these. env = env.copy() else: env = os.environ.copy() self._clean_environment(env=env) cmdline = self.cmdline(args) TRACER.log("PEX.run invoking %s" % " ".join(cmdline)) process = Executor.open_process( cmdline, cwd=self._pex if with_chroot else os.getcwd(), preexec_fn=os.setsid if setsid else None, stdin=kwargs.pop("stdin", None), stdout=kwargs.pop("stdout", None), stderr=kwargs.pop("stderr", None), env=env, **kwargs ) return process.wait() if blocking else process
def run(self): parser, options_builder = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die('Must not specify entry point, script or output file to --pex-args, given: {}' .format(' '.join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname( os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend( (script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') pex_specs.append( (name if name in console_scripts else None, target)) # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir ] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append('-v') for script_name, target in pex_specs: cmd = args + ['--output-file', target] if script_name: log.info('Writing %s to %s' % (script_name, target)) cmd += ['--script', script_name] else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: die( 'Failed to create pex via {}:\n{}'.format( ' '.join(cmd), stderr.decode('utf-8')), result)
def run(self, args=(), with_chroot=False, blocking=True, setsid=False, **kwargs): """Run the PythonEnvironment in an interpreter in a subprocess. :keyword args: Additional arguments to be passed to the application being invoked by the environment. :keyword with_chroot: Run with cwd set to the environment's working directory. :keyword blocking: If true, return the return code of the subprocess. If false, return the Popen object of the invoked subprocess. :keyword setsid: If true, run the PEX in a separate operating system session. Remaining keyword arguments are passed directly to subprocess.Popen. """ self.clean_environment() cmdline = self.cmdline(args) TRACER.log('PEX.run invoking %s' % ' '.join(cmdline)) process = Executor.open_process( cmdline, cwd=self._pex if with_chroot else os.getcwd(), preexec_fn=os.setsid if setsid else None, stdin=kwargs.pop('stdin', None), stdout=kwargs.pop('stdout', None), stderr=kwargs.pop('stderr', None), **kwargs) return process.wait() if blocking else process
def test_executor_exceptions_executablenotfound(cmd): # type: (List[str]) -> None exc_cause = OSError("test") exc = Executor.ExecutableNotFound(cmd=cmd, exc=exc_cause) assert exc.executable == TEST_EXECUTABLE assert exc.cmd == cmd assert exc.exc == exc_cause
def open_process(self, args=None, pythonpath=None, env=None, **kwargs): cmd, env = self._create_isolated_cmd(self.binary, args=args, pythonpath=pythonpath, env=env) process = Executor.open_process(cmd, env=env, **kwargs) return cmd, process
def _from_binary_external(cls, binary): environ = cls.sanitized_environment() stdout, _ = Executor.execute([binary, '-sE'], env=environ, stdin_payload=_generate_identity_source()) identity = stdout.strip() if not identity: raise cls.IdentificationError('Could not establish identity of %s' % binary) return cls(binary, PythonIdentity.from_id_string(identity))
def run_pex_command(args, env=None): """Simulate running pex command for integration testing. This is different from run_simple_pex in that it calls the pex command rather than running a generated pex. This is useful for testing end to end runs with specific command line arguments or env options. """ cmd = [sys.executable, '-mpex', '-vvvvv'] + list(args) process = Executor.open_process(cmd=cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() return IntegResults(output.decode('utf-8'), error.decode('utf-8'), process.returncode)
def run(self): if self._installed is not None: return self._installed with TRACER.timed('Installing %s' % self._install_tmp, V=2): command = [self._interpreter.binary, '-sE', '-'] + self._setup_command() try: Executor.execute(command, env=self._interpreter.sanitized_environment(), cwd=self._source_dir, stdin_payload=self.bootstrap_script.encode('ascii')) self._installed = True except Executor.NonZeroExit as e: self._installed = False name = os.path.basename(self._source_dir) print('**** Failed to install %s (caused by: %r\n):' % (name, e), file=sys.stderr) print('stdout:\n%s\nstderr:\n%s\n' % (e.stdout, e.stderr), file=sys.stderr) return self._installed return self._installed
def test_executor_execute(): assert Executor.execute('/bin/echo -n stdout >&1', shell=True) == ('stdout', '') assert Executor.execute('/bin/echo -n stderr >&2', shell=True) == ('', 'stderr') assert Executor.execute('/bin/echo -n TEST | tee /dev/stderr', shell=True) == ('TEST', 'TEST') assert Executor.execute(['/bin/echo', 'hello']) == ('hello\n', '') assert Executor.execute(['/bin/echo', '-n', 'hello']) == ('hello', '') assert Executor.execute('/bin/echo -n $HELLO', env={'HELLO': 'hey'}, shell=True) == ('hey', '')
def run(self): parser, options_builder = configure_clp() options, reqs = parser.parse_args(self.pex_args) if options.entry_point or options.script or options.pex_name: die('Must not specify entry point, script or output file to --pex-args, given: {}' .format(' '.join(self.pex_args))) name = self.distribution.get_name() version = self.distribution.get_version() package_dir = os.path.dirname(os.path.realpath(os.path.expanduser( self.distribution.script_name))) if self.bdist_dir is None: self.bdist_dir = os.path.join(package_dir, 'dist') console_scripts = self.parse_entry_points() pex_specs = [] if self.bdist_all: # Write all entry points into unversioned pex files. pex_specs.extend((script_name, os.path.join(self.bdist_dir, script_name)) for script_name in console_scripts) else: target = os.path.join(self.bdist_dir, name + '-' + version + '.pex') pex_specs.append((name if name in console_scripts else None, target)) # In order for code to run to here, pex is on the sys.path - make sure to propagate the # sys.path so the subprocess can find us. env = os.environ.copy() env['PYTHONPATH'] = os.pathsep.join(sys.path) args = [sys.executable, '-s', '-m', 'pex.bin.pex', package_dir] + reqs + self.pex_args if self.get_log_level() < log.INFO and options.verbosity == 0: args.append('-v') for script_name, target in pex_specs: cmd = args + ['--output-file', target] if script_name: log.info('Writing %s to %s' % (script_name, target)) cmd += ['--script', script_name] else: # The package has no namesake entry point, so build an environment pex. log.info('Writing environment pex into %s' % target) log.debug('Building pex via: {}'.format(' '.join(cmd))) process = Executor.open_process(cmd, stderr=subprocess.PIPE, env=env) _, stderr = process.communicate() result = process.returncode if result != 0: die('Failed to create pex via {}:\n{}'.format(' '.join(cmd), stderr.decode('utf-8')), result)
def open_process( self, args=None, # type: Optional[Iterable[str]] pythonpath=None, # type: Optional[Iterable[str]] env=None, # type: Optional[Mapping[str, str]] **kwargs # type: Any ): # type: (...) -> Tuple[Iterable[str], subprocess.Popen] cmd, env = self.create_isolated_cmd(args=args, pythonpath=pythonpath, env=env) process = Executor.open_process(cmd, env=env, **kwargs) return cmd, process
def test_executor_execute(): # type: () -> None assert Executor.execute("/bin/echo -n stdout >&1", shell=True) == ("stdout", "") assert Executor.execute("/bin/echo -n stderr >&2", shell=True) == ("", "stderr") assert Executor.execute("/bin/echo -n TEST | tee /dev/stderr", shell=True) == ("TEST", "TEST") assert Executor.execute(["/bin/echo", "hello"]) == ("hello\n", "") assert Executor.execute(["/bin/echo", "-n", "hello"]) == ("hello", "") assert Executor.execute("/bin/echo -n $HELLO", env={"HELLO": "hey"}, shell=True) == ("hey", "")
def run_pex_command(args, env=None, python=None, quiet=False): """Simulate running pex command for integration testing. This is different from run_simple_pex in that it calls the pex command rather than running a generated pex. This is useful for testing end to end runs with specific command line arguments or env options. """ cmd = [python or sys.executable, '-mpex'] if not quiet: cmd.append('-vvvvv') cmd.extend(args) process = Executor.open_process(cmd=cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() return IntegResults(output.decode('utf-8'), error.decode('utf-8'), process.returncode)
def test_pex_builder_add_source_relpath_issues_1192( tmp_chroot, # type: str copy_mode, # type: CopyMode.Value ): # type: (...) -> None pb = PEXBuilder(copy_mode=copy_mode) with safe_open("src/main.py", "w") as fp: fp.write("import sys; sys.exit(42)") pb.add_source("src/main.py", "main.py") pb.set_entry_point("main") pb.build("test.pex") process = Executor.open_process(cmd=[os.path.abspath("test.pex")]) process.wait() assert 42 == process.returncode
def _execute(cls, binary, args=None, pythonpath=None, env=None, stdin_payload=None, **kwargs): cmd, env = cls._create_isolated_cmd(binary, args=args, pythonpath=pythonpath, env=env) stdout, stderr = Executor.execute(cmd, stdin_payload=stdin_payload, env=env, **kwargs) return cmd, stdout, stderr
def execute( self, args=None, # type: Optional[Iterable[str]] stdin_payload=None, # type: Optional[AnyStr] pythonpath=None, # type: Optional[Iterable[str]] env=None, # type: Optional[Mapping[str, str]] **kwargs # type: Any ): # type: (...) -> Tuple[Iterable[str], str, str] cmd, env = self.create_isolated_cmd(args=args, pythonpath=pythonpath, env=env) stdout, stderr = Executor.execute(cmd, stdin_payload=stdin_payload, env=env, **kwargs) return cmd, stdout, stderr
def compile(self, root, relpaths): """Compiles the given python source files using this compiler's interpreter. :param string root: The root path all the source files are found under. :param list relpaths: The realtive paths from the `root` of the source files to compile. :returns: A list of relative paths of the compiled bytecode files. :raises: A :class:`Compiler.Error` if there was a problem bytecode compiling any of the files. """ with named_temporary_file() as fp: fp.write(to_bytes(_COMPILER_MAIN % {'root': root, 'relpaths': relpaths}, encoding='utf-8')) fp.flush() try: out, _ = Executor.execute([self._interpreter.binary, '-sE', fp.name]) except Executor.NonZeroExit as e: raise self.CompilationFailure( 'encountered %r during bytecode compilation.\nstderr was:\n%s\n' % (e, e.stderr) ) return out.splitlines()
def run(self, args=(), with_chroot=False, blocking=True, setsid=False, **kwargs): """Run the PythonEnvironment in an interpreter in a subprocess. :keyword args: Additional arguments to be passed to the application being invoked by the environment. :keyword with_chroot: Run with cwd set to the environment's working directory. :keyword blocking: If true, return the return code of the subprocess. If false, return the Popen object of the invoked subprocess. :keyword setsid: If true, run the PEX in a separate operating system session. Remaining keyword arguments are passed directly to subprocess.Popen. """ self.clean_environment() cmdline = self.cmdline(args) TRACER.log('PEX.run invoking %s' % ' '.join(cmdline)) process = Executor.open_process(cmdline, cwd=self._pex if with_chroot else os.getcwd(), preexec_fn=os.setsid if setsid else None, stdin=kwargs.pop('stdin', None), stdout=kwargs.pop('stdout', None), stderr=kwargs.pop('stderr', None), **kwargs) return process.wait() if blocking else process
def test_executor_open_process_wait_return(): process = Executor.open_process('exit 8', shell=True) exit_code = process.wait() assert exit_code == 8
def test_executor_execute_zero(): # type: () -> None Executor.execute("exit 0", shell=True)
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode("utf-8").strip() if not identity: raise cls.IdentificationError( "Could not establish identity of %s" % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode("utf-8")) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, "interpreters") os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed("GCing interpreter cache from prior OS version"): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) # Some distributions include more than one copy of the same interpreter via a hard link (e.g.: # python3.7 is a hardlink to python3.7m). To ensure a deterministic INTERP-INFO file we must # emit a separate INTERP-INFO for each link since INTERP-INFO contains the interpreter path and # would otherwise be unstable. # # See cls._REGEXEN for a related affordance. path_id = binary.replace(os.sep, ".").lstrip(".") cache_dir = os.path.join(os_cache_dir, interpreter_hash, path_id) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, "rb") as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(["pex"]) cmd, env = cls._create_isolated_cmd( binary, args=[ "-c", dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)), ], pythonpath=pythonpath, ) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def test_executor_open_process_wait_return(): # type: () -> None process = Executor.open_process("exit 8", shell=True) exit_code = process.wait() assert exit_code == 8
def test_executor_execute_zero(): Executor.execute('exit 0', shell=True)
def test_executor_execute_nonzero(exit_code): with pytest.raises(Executor.NonZeroExit) as exc: Executor.execute('exit %s' % exit_code, shell=True) if exit_code > 0: assert exc.value.exit_code == exit_code
def _spawn_from_binary_external(cls, binary): def create_interpreter(stdout): identity = stdout.decode('utf-8').strip() if not identity: raise cls.IdentificationError( 'Could not establish identity of %s' % binary) return cls(PythonIdentity.decode(identity)) # Part of the PythonInterpreter data are environment markers that depend on the current OS # release. That data can change when the OS is upgraded but (some of) the installed interpreters # remain the same. As such, include the OS in the hash structure for cached interpreters. os_digest = hashlib.sha1() for os_identifier in platform.release(), platform.version(): os_digest.update(os_identifier.encode('utf-8')) os_hash = os_digest.hexdigest() interpreter_cache_dir = os.path.join(ENV.PEX_ROOT, 'interpreters') os_cache_dir = os.path.join(interpreter_cache_dir, os_hash) if os.path.isdir( interpreter_cache_dir) and not os.path.isdir(os_cache_dir): with TRACER.timed('GCing interpreter cache from prior OS version'): safe_rmtree(interpreter_cache_dir) interpreter_hash = CacheHelper.hash(binary) cache_dir = os.path.join(os_cache_dir, interpreter_hash) cache_file = os.path.join(cache_dir, cls.INTERP_INFO_FILE) if os.path.isfile(cache_file): try: with open(cache_file, 'rb') as fp: return SpawnedJob.completed(create_interpreter(fp.read())) except (IOError, OSError, cls.Error, PythonIdentity.Error): safe_rmtree(cache_dir) return cls._spawn_from_binary_external(binary) else: pythonpath = third_party.expose(['pex']) cmd, env = cls._create_isolated_cmd(binary, args=[ '-c', dedent("""\ import os import sys from pex.common import atomic_directory, safe_open from pex.interpreter import PythonIdentity encoded_identity = PythonIdentity.get().encode() sys.stdout.write(encoded_identity) with atomic_directory({cache_dir!r}) as cache_dir: if cache_dir: with safe_open(os.path.join(cache_dir, {info_file!r}), 'w') as fp: fp.write(encoded_identity) """.format(cache_dir=cache_dir, info_file=cls.INTERP_INFO_FILE)) ], pythonpath=pythonpath) process = Executor.open_process(cmd, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) job = Job(command=cmd, process=process) return SpawnedJob.stdout(job, result_func=create_interpreter)
def test_executor_exceptions_executablenotfound(cmd): exc_cause = OSError('test') exc = Executor.ExecutableNotFound(cmd=cmd, exc=exc_cause) assert exc.executable == TEST_EXECUTABLE assert exc.cmd == cmd assert exc.exc == exc_cause