def shutdown(self, signum=signal.SIGTERM, timeout=10, wait_for_orphans=0):
'''Shutdown a running daemon'''
if not self._shutdown:
try:
pid = self.wait_for_daemon_pid(timeout)
terminate_process(pid=pid, kill_children=True)
except TimeoutError:
pass
if self.process:
terminate_process(pid=self.process.pid, kill_children=True)
self.process.wait()
if wait_for_orphans:
# NOTE: The process for finding orphans is greedy, it just
# looks for processes with the same cmdline which are owned by
# PID 1.
orphans = self.find_orphans(self.argv)
last = time.time()
while True:
if orphans:
log.debug('Terminating orphaned child processes: %s',
orphans)
terminate_process_list(orphans)
last = time.time()
if (time.time() - last) >= wait_for_orphans:
break
time.sleep(0.25)
orphans = self.find_orphans(self.argv)
self.process = None
self._shutdown = True
def shutdown(self, signum=signal.SIGTERM, timeout=10):
'''Shutdown a running daemon'''
if not self._shutdown:
try:
pid = self.wait_for_daemon_pid(timeout)
terminate_process(pid=pid)
except TimeoutError:
pass
if self.process:
terminate_process(pid=self.process.pid)
self.process.wait()
self.process = None
self._shutdown = True
def test_master_startup(self):
proc = NonBlockingPopen(
[
self.get_script_path("master"),
"-c",
RUNTIME_VARS.TMP_CONF_DIR,
"-l",
"info",
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out = six.b("")
err = six.b("")
# Testing this should never be longer than 1 minute
max_time = time.time() + 60
try:
while True:
if time.time() > max_time:
assert False, "Max timeout ocurred"
time.sleep(0.5)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if six.b("DeprecationWarning: object() takes no parameters") in out:
self.fail(
"'DeprecationWarning: object() takes no parameters' was seen in output"
)
if six.b("TypeError: object() takes no parameters") in out:
self.fail(
"'TypeError: object() takes no parameters' was seen in output"
)
if six.b("Setting up the master communication server") in out:
# We got past the place we need, stop the process
break
if out is None and err is None:
break
if proc.poll() is not None:
break
finally:
terminate_process(proc.pid, kill_children=True)
def test_master_startup(self):
proc = NonBlockingPopen(
[
self.get_script_path('master'),
'-c',
self.config_dir,
'-l',
'info'
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
)
out = six.b('')
err = six.b('')
# Testing this should never be longer than 1 minute
max_time = time.time() + 60
try:
while True:
if time.time() > max_time:
assert False, 'Max timeout ocurred'
time.sleep(0.5)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if six.b('DeprecationWarning: object() takes no parameters') in out:
self.fail('\'DeprecationWarning: object() takes no parameters\' was seen in output')
if six.b('TypeError: object() takes no parameters') in out:
self.fail('\'TypeError: object() takes no parameters\' was seen in output')
if six.b('Setting up the master communication server') in out:
# We got past the place we need, stop the process
break
if out is None and err is None:
break
if proc.poll() is not None:
break
finally:
terminate_process(proc.pid, kill_children=True)
def finalize(self, exit_code=0):
'''
Run the finalization procedures. Show report, clean-up file-system, etc
'''
# Collect any child processes still laying around
children = processes.collect_child_processes(os.getpid())
if self.options.no_report is False:
self.print_overall_testsuite_report()
self.post_execution_cleanup()
# Brute force approach to terminate this process and its children
if children:
log.info('Terminating test suite child processes: %s', children)
processes.terminate_process(children=children, kill_children=True)
children = processes.collect_child_processes(os.getpid())
if children:
log.info('Second run at terminating test suite child processes: %s', children)
processes.terminate_process(children=children, kill_children=True)
exit_msg = 'Test suite execution finalized with exit code: {}'.format(exit_code)
log.info(exit_msg)
self.exit(status=exit_code, msg=exit_msg + '\n')
def tearDown(self):
subprocess_list = self.subprocess_list
processes = subprocess_list.processes
self.schedule.reset()
del self.schedule
for proc in processes:
if proc.is_alive():
terminate_process(proc.pid, kill_children=True, slow_stop=True)
subprocess_list.cleanup()
processes = subprocess_list.processes
if processes:
for proc in processes:
if proc.is_alive():
terminate_process(proc.pid,
kill_children=True,
slow_stop=False)
subprocess_list.cleanup()
processes = subprocess_list.processes
if processes:
log.warning("Processes left running: %s", processes)
def test_event_return(self):
# Once salt is py3 only, the warnings part of this test no longer applies
evt = None
try:
with warnings.catch_warnings(record=True) as w:
# Cause all warnings to always be triggered.
warnings.simplefilter("always")
evt = None
try:
evt = salt.utils.event.EventReturn(
salt.config.DEFAULT_MASTER_OPTS.copy())
evt.start()
except TypeError as exc:
if "object" in str(exc):
self.fail("'{}' TypeError should have not been raised".
format(exc))
for warning in w:
if warning.category is DeprecationWarning:
assert "object() takes no parameters" not in warning.message
finally:
if evt is not None:
terminate_process(evt.pid, kill_children=True)
def run_script(
self,
script,
arg_str,
catch_stderr=False,
with_retcode=False,
catch_timeout=False,
# FIXME A timeout of zero or disabling timeouts may not return results!
timeout=15,
raw=False,
popen_kwargs=None,
log_output=None,
**kwargs):
"""
Execute a script with the given argument string
The ``log_output`` argument is ternary, it can be True, False, or None.
If the value is boolean, then it forces the results to either be logged
or not logged. If it is None, then the return code of the subprocess
determines whether or not to log results.
"""
import salt.utils.platform
script_path = self.get_script_path(script)
if not os.path.isfile(script_path):
return False
popen_kwargs = popen_kwargs or {}
if salt.utils.platform.is_windows():
cmd = "python "
if "cwd" not in popen_kwargs:
popen_kwargs["cwd"] = os.getcwd()
if "env" not in popen_kwargs:
popen_kwargs["env"] = os.environ.copy()
if sys.version_info[0] < 3:
popen_kwargs["env"][b"PYTHONPATH"] = CODE_DIR.encode()
else:
popen_kwargs["env"]["PYTHONPATH"] = CODE_DIR
else:
cmd = "PYTHONPATH="
python_path = os.environ.get("PYTHONPATH", None)
if python_path is not None:
cmd += "{0}:".format(python_path)
if sys.version_info[0] < 3:
cmd += "{0} ".format(":".join(sys.path[1:]))
else:
cmd += "{0} ".format(":".join(sys.path[0:]))
cmd += "python{0}.{1} ".format(*sys.version_info)
cmd += "{0} ".format(script_path)
cmd += "{0} ".format(arg_str)
if kwargs:
# late import
import salt.utils.json
for key, value in kwargs.items():
cmd += "'{0}={1} '".format(key, salt.utils.json.dumps(value))
tmp_file = tempfile.SpooledTemporaryFile()
popen_kwargs = dict(
{
"shell": True,
"stdout": tmp_file,
"universal_newlines": True
}, **popen_kwargs)
if catch_stderr is True:
popen_kwargs["stderr"] = subprocess.PIPE
if not sys.platform.lower().startswith("win"):
popen_kwargs["close_fds"] = True
def detach_from_parent_group():
# detach from parent group (no more inherited signals!)
os.setpgrp()
popen_kwargs["preexec_fn"] = detach_from_parent_group
def format_return(retcode, stdout, stderr=None, timed_out=False):
"""
DRY helper to log script result if it failed, and then return the
desired output based on whether or not stderr was desired, and
wither or not a retcode was desired.
"""
log_func = log.debug
if timed_out:
log.error(
"run_script timed out after %d seconds (process killed)",
timeout)
log_func = log.error
if log_output is True or timed_out or (log_output is None
and retcode != 0):
log_func(
"run_script results for: %s %s\n"
"return code: %s\n"
"stdout:\n"
"%s\n\n"
"stderr:\n"
"%s",
script,
arg_str,
retcode,
stdout,
stderr,
)
stdout = stdout or ""
stderr = stderr or ""
if not raw:
stdout = stdout.splitlines()
stderr = stderr.splitlines()
ret = [stdout]
if catch_stderr:
ret.append(stderr)
if with_retcode:
ret.append(retcode)
if catch_timeout:
ret.append(timed_out)
return ret[0] if len(ret) == 1 else tuple(ret)
process = subprocess.Popen(cmd, **popen_kwargs)
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
time.sleep(0.1)
if datetime.now() <= stop_at:
# We haven't reached the timeout yet
if process.returncode is not None:
break
else:
terminate_process(process.pid, kill_children=True)
return format_return(process.returncode,
*process.communicate(),
timed_out=True)
tmp_file.seek(0)
if sys.version_info >= (3, ):
try:
out = tmp_file.read().decode(__salt_system_encoding__)
except (NameError, UnicodeDecodeError):
# Let's cross our fingers and hope for the best
out = tmp_file.read().decode("utf-8")
else:
out = tmp_file.read()
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
err = process.stderr.read()
else:
_, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
return format_return(process.returncode, out, err or "")
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, six.string_types):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
# TODO Remove this?
process.communicate()
if process.stdout is not None:
process.stdout.close()
try:
return format_return(process.returncode, out)
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, six.string_types):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
def run(
self,
args=None,
catch_stderr=False,
with_retcode=False,
timeout=None,
raw=False,
env=None,
verbatim_args=False,
verbatim_env=False,
):
'''
Execute a command possibly using a supplied environment.
:param args:
A command string or a command sequence of arguments for the program.
:param catch_stderr: A boolean whether to capture and return stderr.
:param with_retcode: A boolean whether to return the exit code.
:param timeout: A float of how long to wait for the process to
complete before it is killed.
:param raw: A boolean whether to return buffer strings for stdout and
stderr or sequences of output lines.
:param env: A dictionary of environment key/value settings for the
command.
:param verbatim_args: A boolean whether to automatically add inferred arguments.
:param verbatim_env: A boolean whether to automatically add inferred
environment values.
:return list: (stdout [,stderr] [,retcode])
'''
# unused for now
_ = verbatim_args
self.setup()
if args is None:
args = []
if env is None:
env = {}
env_delta = {}
env_delta.update(self.env)
env_delta.update(env)
if not verbatim_env:
env_pypath = env_delta.get('PYTHONPATH',
os.environ.get('PYTHONPATH'))
if not env_pypath:
env_pypath = sys.path
else:
env_pypath = env_pypath.split(':')
for path in sys.path:
if path not in env_pypath:
env_pypath.append(path)
# Always ensure that the test tree is searched first for python modules
if RUNTIME_VARS.CODE_DIR != env_pypath[0]:
env_pypath.insert(0, RUNTIME_VARS.CODE_DIR)
if salt.utils.platform.is_windows():
env_delta['PYTHONPATH'] = ';'.join(env_pypath)
else:
env_delta['PYTHONPATH'] = ':'.join(env_pypath)
cmd_env = dict(os.environ)
cmd_env.update(env_delta)
if salt.utils.platform.is_windows() and six.PY2:
for k, v in cmd_env.items():
if isinstance(k, six.text_type) or isinstance(
v, six.text_type):
cmd_env[k.encode('ascii')] = v.encode('ascii')
popen_kwargs = {
'shell': self.shell,
'stdout': subprocess.PIPE,
'env': cmd_env,
}
if catch_stderr is True:
popen_kwargs['stderr'] = subprocess.PIPE
if not sys.platform.lower().startswith('win'):
popen_kwargs['close_fds'] = True
def detach_from_parent_group():
'''
A utility function that prevents child process from getting parent signals.
'''
os.setpgrp()
popen_kwargs['preexec_fn'] = detach_from_parent_group
if salt.utils.platform.is_windows():
self.argv = ['python.exe', self.program]
else:
self.argv = [self.program]
self.argv.extend(args)
log.debug('TestProgram.run: %s Environment %s', self.argv, env_delta)
process = subprocess.Popen(self.argv, **popen_kwargs)
self.process = process
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
term_sent = False
while True:
process.poll()
if datetime.now() > stop_at:
try:
terminate_process(pid=process.pid, kill_children=True)
process.wait()
except OSError as exc:
if exc.errno != errno.ESRCH:
raise
out = process.stdout.read().splitlines()
out.extend([
'Process took more than {0} seconds to complete. '
'Process Killed!'.format(timeout)
])
if catch_stderr:
err = process.stderr.read().splitlines()
if with_retcode:
return out, err, process.returncode
else:
return out, err
if with_retcode:
return out, process.returncode
else:
return out
if process.returncode is not None:
break
if catch_stderr:
if sys.version_info < (2, 7):
# On python 2.6, the subprocess'es communicate() method uses
# select which, is limited by the OS to 1024 file descriptors
# We need more available descriptors to run the tests which
# need the stderr output.
# So instead of .communicate() we wait for the process to
# finish, but, as the python docs state "This will deadlock
# when using stdout=PIPE and/or stderr=PIPE and the child
# process generates enough output to a pipe such that it
# blocks waiting for the OS pipe buffer to accept more data.
# Use communicate() to avoid that." <- a catch, catch situation
#
# Use this work around were it's needed only, python 2.6
process.wait()
out = process.stdout.read()
err = process.stderr.read()
else:
out, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
if with_retcode:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines(
), process.returncode
else:
return out, err, process.returncode
return out.splitlines(), [], process.returncode
else:
if out is not None and err is not None:
if not raw:
return out.splitlines(), err.splitlines()
else:
return out, err
if not raw:
return out.splitlines(), []
else:
return out, []
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
data = process.communicate()
process.stdout.close()
try:
if with_retcode:
if not raw:
return data[0].splitlines(), process.returncode
else:
return data[0], process.returncode
else:
if not raw:
return data[0].splitlines()
else:
return data[0]
finally:
try:
process.terminate()
except OSError as err:
# process already terminated
pass
