def run_script(
self,
script,
arg_str,
catch_stderr=False,
with_retcode=False,
catch_timeout=False,
# FIXME A timeout of zero or disabling timeouts may not return results!
timeout=15,
raw=False,
popen_kwargs=None,
log_output=None):
'''
Execute a script with the given argument string
The ``log_output`` argument is ternary, it can be True, False, or None.
If the value is boolean, then it forces the results to either be logged
or not logged. If it is None, then the return code of the subprocess
determines whether or not to log results.
'''
import salt.utils.platform
script_path = self.get_script_path(script)
if not os.path.isfile(script_path):
return False
popen_kwargs = popen_kwargs or {}
if salt.utils.platform.is_windows():
cmd = 'python '
if 'cwd' not in popen_kwargs:
popen_kwargs['cwd'] = os.getcwd()
if 'env' not in popen_kwargs:
popen_kwargs['env'] = os.environ.copy()
if sys.version_info[0] < 3:
popen_kwargs['env'][b'PYTHONPATH'] = CODE_DIR.encode()
else:
popen_kwargs['env']['PYTHONPATH'] = CODE_DIR
else:
cmd = 'PYTHONPATH='
python_path = os.environ.get('PYTHONPATH', None)
if python_path is not None:
cmd += '{0}:'.format(python_path)
if sys.version_info[0] < 3:
cmd += '{0} '.format(':'.join(sys.path[1:]))
else:
cmd += '{0} '.format(':'.join(sys.path[0:]))
cmd += 'python{0}.{1} '.format(*sys.version_info)
cmd += '{0} '.format(script_path)
cmd += '{0} '.format(arg_str)
tmp_file = tempfile.SpooledTemporaryFile()
popen_kwargs = dict(
{
'shell': True,
'stdout': tmp_file,
'universal_newlines': True,
}, **popen_kwargs)
if catch_stderr is True:
popen_kwargs['stderr'] = subprocess.PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
# detach from parent group (no more inherited signals!)
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
def format_return(retcode, stdout, stderr=None, timed_out=False):
'''
DRY helper to log script result if it failed, and then return the
desired output based on whether or not stderr was desired, and
wither or not a retcode was desired.
'''
log_func = log.debug
if timed_out:
log.error(
'run_script timed out after %d seconds (process killed)',
timeout)
log_func = log.error
if log_output is True \
or timed_out \
or (log_output is None and retcode != 0):
log_func(
'run_script results for: %s %s\n'
'return code: %s\n'
'stdout:\n'
'%s\n\n'
'stderr:\n'
'%s', script, arg_str, retcode, stdout, stderr)
stdout = stdout or ''
stderr = stderr or ''
if not raw:
stdout = stdout.splitlines()
stderr = stderr.splitlines()
ret = [stdout]
if catch_stderr:
ret.append(stderr)
if with_retcode:
ret.append(retcode)
if catch_timeout:
ret.append(timed_out)
return ret[0] if len(ret) == 1 else tuple(ret)
process = subprocess.Popen(cmd, **popen_kwargs)
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
time.sleep(0.1)
if datetime.now() <= stop_at:
# We haven't reached the timeout yet
if process.returncode is not None:
break
else:
# We've reached the timeout
if term_sent is False:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s",
alive)
else:
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
# As a last resort, kill the process group
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s",
alive)
else:
os.killpg(os.getpgid(process.pid), signal.SIGINT)
except OSError as exc:
if exc.errno != errno.ESRCH:
# If errno is not "no such process", raise
raise
return format_return(process.returncode,
*process.communicate(),
timed_out=True)
tmp_file.seek(0)
if sys.version_info >= (3, ):
try:
out = tmp_file.read().decode(__salt_system_encoding__)
except (NameError, UnicodeDecodeError):
# Let's cross our fingers and hope for the best
out = tmp_file.read().decode('utf-8')
else:
out = tmp_file.read()
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
err = process.stderr.read()
else:
_, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
return format_return(process.returncode, out, err or '')
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, six.string_types):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
# TODO Remove this?
process.communicate()
if process.stdout is not None:
process.stdout.close()
try:
return format_return(process.returncode, out)
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, six.string_types):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
def run(
self,
args=None,
catch_stderr=False,
with_retcode=False,
timeout=None,
raw=False,
env=None,
verbatim_args=False,
verbatim_env=False,
):
'''
Execute a command possibly using a supplied environment.
:param args:
A command string or a command sequence of arguments for the program.
:param catch_stderr: A boolean whether to capture and return stderr.
:param with_retcode: A boolean whether to return the exit code.
:param timeout: A float of how long to wait for the process to
complete before it is killed.
:param raw: A boolean whether to return buffer strings for stdout and
stderr or sequences of output lines.
:param env: A dictionary of environment key/value settings for the
command.
:param verbatim_args: A boolean whether to automatically add inferred arguments.
:param verbatim_env: A boolean whether to automatically add inferred
environment values.
:return list: (stdout [,stderr] [,retcode])
'''
# unused for now
_ = verbatim_args
self.setup()
if args is None:
args = []
if env is None:
env = {}
env_delta = {}
env_delta.update(self.env)
env_delta.update(env)
if not verbatim_env:
env_pypath = env_delta.get('PYTHONPATH',
os.environ.get('PYTHONPATH'))
if not env_pypath:
env_pypath = sys.path
else:
env_pypath = env_pypath.split(':')
for path in sys.path:
if path not in env_pypath:
env_pypath.append(path)
# Always ensure that the test tree is searched first for python modules
if CODE_DIR != env_pypath[0]:
env_pypath.insert(0, CODE_DIR)
env_delta['PYTHONPATH'] = ':'.join(env_pypath)
cmd_env = dict(os.environ)
cmd_env.update(env_delta)
popen_kwargs = {
'shell': self.shell,
'stdout': subprocess.PIPE,
'env': cmd_env,
}
if catch_stderr is True:
popen_kwargs['stderr'] = subprocess.PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
'''
A utility function that prevents child process from getting parent signals.
'''
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
self.argv = [self.program]
self.argv.extend(args)
log.debug('TestProgram.run: %s Environment %s', self.argv, env_delta)
process = subprocess.Popen(self.argv, **popen_kwargs)
self.process = process
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
if datetime.now() > stop_at:
if term_sent is False:
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s",
alive)
else:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s",
alive)
else:
# As a last resort, kill the process group
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
process.wait()
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
out = process.stdout.read().splitlines()
out.extend([
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout)
])
if catch_stderr:
err = process.stderr.read().splitlines()
if with_retcode:
return out, err, process.returncode
else:
return out, err
if with_retcode:
return out, process.returncode
else:
return out
if process.returncode is not None:
break
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
out = process.stdout.read()
err = process.stderr.read()
else:
out, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
if with_retcode:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines(
), process.returncode
else:
return out, err, process.returncode
return out.splitlines(), [], process.returncode
else:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines()
else:
return out, err
if not raw:
return out.splitlines(), []
else:
return out, []
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
data = process.communicate()
process.stdout.close()
try:
if with_retcode:
if not raw:
return data[0].splitlines(), process.returncode
else:
return data[0], process.returncode
else:
if not raw:
return data[0].splitlines()
else:
return data[0]
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
def run_script(
self,
script,
arg_str,
catch_stderr=False,
with_retcode=False,
# FIXME A timeout of zero or disabling timeouts may not return results!
timeout=15,
raw=False):
'''
Execute a script with the given argument string
'''
script_path = self.get_script_path(script)
if not os.path.isfile(script_path):
return False
python_path = os.environ.get('PYTHONPATH', None)
if sys.platform.startswith('win'):
cmd = 'set PYTHONPATH='
else:
cmd = 'PYTHONPATH='
if python_path is not None:
cmd += '{0}:'.format(python_path)
if sys.version_info[0] < 3:
cmd += '{0} '.format(':'.join(sys.path[1:]))
else:
cmd += '{0} '.format(':'.join(sys.path[0:]))
cmd += 'python{0}.{1} '.format(*sys.version_info)
cmd += '{0} '.format(script_path)
cmd += '{0} '.format(arg_str)
tmp_file = tempfile.SpooledTemporaryFile()
popen_kwargs = {
'shell': True,
'stdout': tmp_file,
'universal_newlines': True
}
if catch_stderr is True:
popen_kwargs['stderr'] = subprocess.PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
# detach from parent group (no more inherited signals!)
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
process = subprocess.Popen(cmd, **popen_kwargs)
# Late import
import salt.utils.platform
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
time.sleep(0.1)
if datetime.now() > stop_at:
if term_sent is False:
# Kill the process group since sending the term signal
# would only terminate the shell, not the command
# executed in the shell
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s",
alive)
else:
os.killpg(os.getpgid(process.pid), signal.SIGINT)
term_sent = True
continue
try:
# As a last resort, kill the process group
if salt.utils.platform.is_windows():
_, alive = win32_kill_process_tree(process.pid)
if alive:
log.error("Child processes still alive: %s",
alive)
else:
os.killpg(os.getpgid(process.pid), signal.SIGINT)
except OSError as exc:
if exc.errno != errno.ESRCH:
# If errno is not "no such process", raise
raise
out = [
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout)
]
if catch_stderr:
err = ['Process killed, unable to catch stderr output']
if with_retcode:
return out, err, process.returncode
else:
return out, err
if with_retcode:
return out, process.returncode
else:
return out
if process.returncode is not None:
break
tmp_file.seek(0)
if sys.version_info >= (3, ):
try:
out = tmp_file.read().decode(__salt_system_encoding__)
except (NameError, UnicodeDecodeError):
# Let's cross our fingers and hope for the best
out = tmp_file.read().decode('utf-8')
else:
out = tmp_file.read()
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
err = process.stderr.read()
else:
_, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
if with_retcode:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines(
), process.returncode
else:
return out, err, process.returncode
return out.splitlines(), [], process.returncode
else:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines()
else:
return out, err
if not raw:
return out.splitlines(), []
else:
return out, []
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, six.string_types):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
# TODO Remove this?
process.communicate()
if process.stdout is not None:
process.stdout.close()
try:
if with_retcode:
if not raw:
return out.splitlines(), process.returncode
else:
return out, process.returncode
else:
if not raw:
return out.splitlines()
else:
return out
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, six.string_types):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
