def run_script( self, script, arg_str, catch_stderr=False, with_retcode=False, catch_timeout=False, # FIXME A timeout of zero or disabling timeouts may not return results! timeout=15, raw=False, popen_kwargs=None, log_output=None): ''' Execute a script with the given argument string The ``log_output`` argument is ternary, it can be True, False, or None. If the value is boolean, then it forces the results to either be logged or not logged. If it is None, then the return code of the subprocess determines whether or not to log results. ''' import salt.utils.platform script_path = self.get_script_path(script) if not os.path.isfile(script_path): return False popen_kwargs = popen_kwargs or {} if salt.utils.platform.is_windows(): cmd = 'python ' if 'cwd' not in popen_kwargs: popen_kwargs['cwd'] = os.getcwd() if 'env' not in popen_kwargs: popen_kwargs['env'] = os.environ.copy() if sys.version_info[0] < 3: popen_kwargs['env'][b'PYTHONPATH'] = CODE_DIR.encode() else: popen_kwargs['env']['PYTHONPATH'] = CODE_DIR else: cmd = 'PYTHONPATH=' python_path = os.environ.get('PYTHONPATH', None) if python_path is not None: cmd += '{0}:'.format(python_path) if sys.version_info[0] < 3: cmd += '{0} '.format(':'.join(sys.path[1:])) else: cmd += '{0} '.format(':'.join(sys.path[0:])) cmd += 'python{0}.{1} '.format(*sys.version_info) cmd += '{0} '.format(script_path) cmd += '{0} '.format(arg_str) tmp_file = tempfile.SpooledTemporaryFile() popen_kwargs = dict( { 'shell': True, 'stdout': tmp_file, 'universal_newlines': True, }, **popen_kwargs) if catch_stderr is True: popen_kwargs['stderr'] = subprocess.PIPE if not sys.platform.lower().startswith('win'): popen_kwargs['close_fds'] = True def detach_from_parent_group(): # detach from parent group (no more inherited signals!) os.setpgrp() popen_kwargs['preexec_fn'] = detach_from_parent_group def format_return(retcode, stdout, stderr=None, timed_out=False): ''' DRY helper to log script result if it failed, and then return the desired output based on whether or not stderr was desired, and wither or not a retcode was desired. ''' log_func = log.debug if timed_out: log.error( 'run_script timed out after %d seconds (process killed)', timeout) log_func = log.error if log_output is True \ or timed_out \ or (log_output is None and retcode != 0): log_func( 'run_script results for: %s %s\n' 'return code: %s\n' 'stdout:\n' '%s\n\n' 'stderr:\n' '%s', script, arg_str, retcode, stdout, stderr) stdout = stdout or '' stderr = stderr or '' if not raw: stdout = stdout.splitlines() stderr = stderr.splitlines() ret = [stdout] if catch_stderr: ret.append(stderr) if with_retcode: ret.append(retcode) if catch_timeout: ret.append(timed_out) return ret[0] if len(ret) == 1 else tuple(ret) process = subprocess.Popen(cmd, **popen_kwargs) if timeout is not None: stop_at = datetime.now() + timedelta(seconds=timeout) term_sent = False while True: process.poll() time.sleep(0.1) if datetime.now() <= stop_at: # We haven't reached the timeout yet if process.returncode is not None: break else: # We've reached the timeout if term_sent is False: # Kill the process group since sending the term signal # would only terminate the shell, not the command # executed in the shell if salt.utils.platform.is_windows(): _, alive = win32_kill_process_tree(process.pid) if alive: log.error("Child processes still alive: %s", alive) else: os.killpg(os.getpgid(process.pid), signal.SIGINT) term_sent = True continue try: # As a last resort, kill the process group if salt.utils.platform.is_windows(): _, alive = win32_kill_process_tree(process.pid) if alive: log.error("Child processes still alive: %s", alive) else: os.killpg(os.getpgid(process.pid), signal.SIGINT) except OSError as exc: if exc.errno != errno.ESRCH: # If errno is not "no such process", raise raise return format_return(process.returncode, *process.communicate(), timed_out=True) tmp_file.seek(0) if sys.version_info >= (3, ): try: out = tmp_file.read().decode(__salt_system_encoding__) except (NameError, UnicodeDecodeError): # Let's cross our fingers and hope for the best out = tmp_file.read().decode('utf-8') else: out = tmp_file.read() if catch_stderr: if sys.version_info < (2, 7): # On python 2.6, the subprocess'es communicate() method uses # select which, is limited by the OS to 1024 file descriptors # We need more available descriptors to run the tests which # need the stderr output. # So instead of .communicate() we wait for the process to # finish, but, as the python docs state "This will deadlock # when using stdout=PIPE and/or stderr=PIPE and the child # process generates enough output to a pipe such that it # blocks waiting for the OS pipe buffer to accept more data. # Use communicate() to avoid that." <- a catch, catch situation # # Use this work around were it's needed only, python 2.6 process.wait() err = process.stderr.read() else: _, err = process.communicate() # Force closing stderr/stdout to release file descriptors if process.stdout is not None: process.stdout.close() if process.stderr is not None: process.stderr.close() # pylint: disable=maybe-no-member try: return format_return(process.returncode, out, err or '') finally: try: if os.path.exists(tmp_file.name): if isinstance(tmp_file.name, six.string_types): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) else: # Clean up file handles tmp_file.close() process.terminate() except OSError as err: # process already terminated pass # pylint: enable=maybe-no-member # TODO Remove this? process.communicate() if process.stdout is not None: process.stdout.close() try: return format_return(process.returncode, out) finally: try: if os.path.exists(tmp_file.name): if isinstance(tmp_file.name, six.string_types): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) else: # Clean up file handles tmp_file.close() process.terminate() except OSError as err: # process already terminated pass
def run( self, args=None, catch_stderr=False, with_retcode=False, timeout=None, raw=False, env=None, verbatim_args=False, verbatim_env=False, ): ''' Execute a command possibly using a supplied environment. :param args: A command string or a command sequence of arguments for the program. :param catch_stderr: A boolean whether to capture and return stderr. :param with_retcode: A boolean whether to return the exit code. :param timeout: A float of how long to wait for the process to complete before it is killed. :param raw: A boolean whether to return buffer strings for stdout and stderr or sequences of output lines. :param env: A dictionary of environment key/value settings for the command. :param verbatim_args: A boolean whether to automatically add inferred arguments. :param verbatim_env: A boolean whether to automatically add inferred environment values. :return list: (stdout [,stderr] [,retcode]) ''' # unused for now _ = verbatim_args self.setup() if args is None: args = [] if env is None: env = {} env_delta = {} env_delta.update(self.env) env_delta.update(env) if not verbatim_env: env_pypath = env_delta.get('PYTHONPATH', os.environ.get('PYTHONPATH')) if not env_pypath: env_pypath = sys.path else: env_pypath = env_pypath.split(':') for path in sys.path: if path not in env_pypath: env_pypath.append(path) # Always ensure that the test tree is searched first for python modules if CODE_DIR != env_pypath[0]: env_pypath.insert(0, CODE_DIR) env_delta['PYTHONPATH'] = ':'.join(env_pypath) cmd_env = dict(os.environ) cmd_env.update(env_delta) popen_kwargs = { 'shell': self.shell, 'stdout': subprocess.PIPE, 'env': cmd_env, } if catch_stderr is True: popen_kwargs['stderr'] = subprocess.PIPE if not sys.platform.lower().startswith('win'): popen_kwargs['close_fds'] = True def detach_from_parent_group(): ''' A utility function that prevents child process from getting parent signals. ''' os.setpgrp() popen_kwargs['preexec_fn'] = detach_from_parent_group self.argv = [self.program] self.argv.extend(args) log.debug('TestProgram.run: %s Environment %s', self.argv, env_delta) process = subprocess.Popen(self.argv, **popen_kwargs) self.process = process if timeout is not None: stop_at = datetime.now() + timedelta(seconds=timeout) term_sent = False while True: process.poll() if datetime.now() > stop_at: if term_sent is False: if salt.utils.platform.is_windows(): _, alive = win32_kill_process_tree(process.pid) if alive: log.error("Child processes still alive: %s", alive) else: # Kill the process group since sending the term signal # would only terminate the shell, not the command # executed in the shell os.killpg(os.getpgid(process.pid), signal.SIGINT) term_sent = True continue try: if salt.utils.platform.is_windows(): _, alive = win32_kill_process_tree(process.pid) if alive: log.error("Child processes still alive: %s", alive) else: # As a last resort, kill the process group os.killpg(os.getpgid(process.pid), signal.SIGKILL) process.wait() except OSError as exc: if exc.errno != errno.ESRCH: raise out = process.stdout.read().splitlines() out.extend([ 'Process took more than {0} seconds to complete. ' 'Process Killed!'.format(timeout) ]) if catch_stderr: err = process.stderr.read().splitlines() if with_retcode: return out, err, process.returncode else: return out, err if with_retcode: return out, process.returncode else: return out if process.returncode is not None: break if catch_stderr: if sys.version_info < (2, 7): # On python 2.6, the subprocess'es communicate() method uses # select which, is limited by the OS to 1024 file descriptors # We need more available descriptors to run the tests which # need the stderr output. # So instead of .communicate() we wait for the process to # finish, but, as the python docs state "This will deadlock # when using stdout=PIPE and/or stderr=PIPE and the child # process generates enough output to a pipe such that it # blocks waiting for the OS pipe buffer to accept more data. # Use communicate() to avoid that." <- a catch, catch situation # # Use this work around were it's needed only, python 2.6 process.wait() out = process.stdout.read() err = process.stderr.read() else: out, err = process.communicate() # Force closing stderr/stdout to release file descriptors if process.stdout is not None: process.stdout.close() if process.stderr is not None: process.stderr.close() # pylint: disable=maybe-no-member try: if with_retcode: if out is not None and err is not None: if not raw: return out.splitlines(), err.splitlines( ), process.returncode else: return out, err, process.returncode return out.splitlines(), [], process.returncode else: if out is not None and err is not None: if not raw: return out.splitlines(), err.splitlines() else: return out, err if not raw: return out.splitlines(), [] else: return out, [] finally: try: process.terminate() except OSError as err: # process already terminated pass # pylint: enable=maybe-no-member data = process.communicate() process.stdout.close() try: if with_retcode: if not raw: return data[0].splitlines(), process.returncode else: return data[0], process.returncode else: if not raw: return data[0].splitlines() else: return data[0] finally: try: process.terminate() except OSError as err: # process already terminated pass
def run_script( self, script, arg_str, catch_stderr=False, with_retcode=False, # FIXME A timeout of zero or disabling timeouts may not return results! timeout=15, raw=False): ''' Execute a script with the given argument string ''' script_path = self.get_script_path(script) if not os.path.isfile(script_path): return False python_path = os.environ.get('PYTHONPATH', None) if sys.platform.startswith('win'): cmd = 'set PYTHONPATH=' else: cmd = 'PYTHONPATH=' if python_path is not None: cmd += '{0}:'.format(python_path) if sys.version_info[0] < 3: cmd += '{0} '.format(':'.join(sys.path[1:])) else: cmd += '{0} '.format(':'.join(sys.path[0:])) cmd += 'python{0}.{1} '.format(*sys.version_info) cmd += '{0} '.format(script_path) cmd += '{0} '.format(arg_str) tmp_file = tempfile.SpooledTemporaryFile() popen_kwargs = { 'shell': True, 'stdout': tmp_file, 'universal_newlines': True } if catch_stderr is True: popen_kwargs['stderr'] = subprocess.PIPE if not sys.platform.lower().startswith('win'): popen_kwargs['close_fds'] = True def detach_from_parent_group(): # detach from parent group (no more inherited signals!) os.setpgrp() popen_kwargs['preexec_fn'] = detach_from_parent_group process = subprocess.Popen(cmd, **popen_kwargs) # Late import import salt.utils.platform if timeout is not None: stop_at = datetime.now() + timedelta(seconds=timeout) term_sent = False while True: process.poll() time.sleep(0.1) if datetime.now() > stop_at: if term_sent is False: # Kill the process group since sending the term signal # would only terminate the shell, not the command # executed in the shell if salt.utils.platform.is_windows(): _, alive = win32_kill_process_tree(process.pid) if alive: log.error("Child processes still alive: %s", alive) else: os.killpg(os.getpgid(process.pid), signal.SIGINT) term_sent = True continue try: # As a last resort, kill the process group if salt.utils.platform.is_windows(): _, alive = win32_kill_process_tree(process.pid) if alive: log.error("Child processes still alive: %s", alive) else: os.killpg(os.getpgid(process.pid), signal.SIGINT) except OSError as exc: if exc.errno != errno.ESRCH: # If errno is not "no such process", raise raise out = [ 'Process took more than {0} seconds to complete. ' 'Process Killed!'.format(timeout) ] if catch_stderr: err = ['Process killed, unable to catch stderr output'] if with_retcode: return out, err, process.returncode else: return out, err if with_retcode: return out, process.returncode else: return out if process.returncode is not None: break tmp_file.seek(0) if sys.version_info >= (3, ): try: out = tmp_file.read().decode(__salt_system_encoding__) except (NameError, UnicodeDecodeError): # Let's cross our fingers and hope for the best out = tmp_file.read().decode('utf-8') else: out = tmp_file.read() if catch_stderr: if sys.version_info < (2, 7): # On python 2.6, the subprocess'es communicate() method uses # select which, is limited by the OS to 1024 file descriptors # We need more available descriptors to run the tests which # need the stderr output. # So instead of .communicate() we wait for the process to # finish, but, as the python docs state "This will deadlock # when using stdout=PIPE and/or stderr=PIPE and the child # process generates enough output to a pipe such that it # blocks waiting for the OS pipe buffer to accept more data. # Use communicate() to avoid that." <- a catch, catch situation # # Use this work around were it's needed only, python 2.6 process.wait() err = process.stderr.read() else: _, err = process.communicate() # Force closing stderr/stdout to release file descriptors if process.stdout is not None: process.stdout.close() if process.stderr is not None: process.stderr.close() # pylint: disable=maybe-no-member try: if with_retcode: if out is not None and err is not None: if not raw: return out.splitlines(), err.splitlines( ), process.returncode else: return out, err, process.returncode return out.splitlines(), [], process.returncode else: if out is not None and err is not None: if not raw: return out.splitlines(), err.splitlines() else: return out, err if not raw: return out.splitlines(), [] else: return out, [] finally: try: if os.path.exists(tmp_file.name): if isinstance(tmp_file.name, six.string_types): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) else: # Clean up file handles tmp_file.close() process.terminate() except OSError as err: # process already terminated pass # pylint: enable=maybe-no-member # TODO Remove this? process.communicate() if process.stdout is not None: process.stdout.close() try: if with_retcode: if not raw: return out.splitlines(), process.returncode else: return out, process.returncode else: if not raw: return out.splitlines() else: return out finally: try: if os.path.exists(tmp_file.name): if isinstance(tmp_file.name, six.string_types): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) else: # Clean up file handles tmp_file.close() process.terminate() except OSError as err: # process already terminated pass