def hgfs_setup_and_teardown():
"""
build up and tear down hg repos to test with.
"""
initial_child_processes = psutil.Process().children()
source_dir = Path(__file__).resolve().parent.joinpath("files")
tempdir = tempfile.TemporaryDirectory()
tempsubdir = tempdir.name / Path("subdir/")
tempsubdir.mkdir()
tempdirPath = Path(tempdir.name)
for file in source_dir.iterdir():
to_file = tempdirPath / file.name
to_file2 = tempsubdir / file.name
shutil.copy(file.as_posix(), to_file.as_posix())
shutil.copy(file.as_posix(), to_file2.as_posix())
client = hglib.init(bytes(tempdirPath.as_posix(), encoding="utf8"))
client.close()
with hglib.open(bytes(tempdirPath.as_posix(), encoding="utf8")) as repo:
repo.add(bytes(tempdirPath.as_posix(), encoding="utf8"))
repo.commit(b"init commit", user="******")
repo.tag(b"test", user="******")
repo.branch(b"test")
repo.commit(b"create test branch", user="******")
repo.bookmark(b"bookmark_test")
try:
yield tempdirPath.as_uri()
finally:
tempdir.cleanup()
for child in psutil.Process().children():
if child not in initial_child_processes:
terminate_process(process=child, kill_children=True)
def schedule(tmp_path):
subprocess_list = None
try:
subprocess_list = SubprocessList()
root_dir = tmp_path / "schedule-unit-tests"
sock_dir = str(root_dir / "test-socks")
default_config = salt.config.minion_config(None)
default_config["conf_dir"] = str(root_dir)
default_config["root_dir"] = str(root_dir)
default_config["sock_dir"] = sock_dir
default_config["pki_dir"] = str(root_dir / "pki")
default_config["cachedir"] = str(root_dir / "cache")
with patch("salt.utils.schedule.clean_proc_dir",
MagicMock(return_value=None)):
functions = {"test.ping": ping}
_schedule = salt.utils.schedule.Schedule(
copy.deepcopy(default_config),
functions,
returners={},
new_instance=True,
)
_schedule.opts["loop_interval"] = 1
_schedule._subprocess_list = subprocess_list
yield _schedule
finally:
if subprocess_list:
processes = subprocess_list.processes
_schedule.reset()
del _schedule
for proc in processes:
if proc.is_alive():
terminate_process(proc.pid,
kill_children=True,
slow_stop=True)
subprocess_list.cleanup()
processes = subprocess_list.processes
if processes:
for proc in processes:
if proc.is_alive():
terminate_process(proc.pid,
kill_children=True,
slow_stop=False)
subprocess_list.cleanup()
processes = subprocess_list.processes
if processes:
log.warning("Processes left running: %s", processes)
del default_config
del subprocess_list
def req_server_channel(salt_master, req_channel_crypt):
req_server_channel_process = ReqServerChannelProcess(
salt_master.config.copy(), req_channel_crypt)
try:
with req_server_channel_process:
yield
finally:
terminate_process(pid=req_server_channel_process.pid,
kill_children=True,
slow_stop=False)
def test_event_return():
evt = None
try:
evt = salt.utils.event.EventReturn(
salt.config.DEFAULT_MASTER_OPTS.copy())
evt.start()
except TypeError as exc:
if "object" in str(exc):
pytest.fail(
"'{}' TypeError should have not been raised".format(exc))
finally:
if evt is not None:
terminate_process(evt.pid, kill_children=True)
def close(self):
if self._closing:
return
self._closing = True
if self.req_server_channel is not None:
self.req_server_channel.close()
self.req_server_channel = None
if self.process_manager is not None:
self.process_manager.terminate()
# Really terminate any process still left behind
for pid in self.process_manager._process_map:
terminate_process(pid=pid, kill_children=True, slow_stop=False)
self.process_manager = None
def close(self):
if self._closing:
return
self._closing = True
if self.process_manager is None:
return
self.process_manager.terminate()
if hasattr(self.pub_server_channel, "pub_close"):
self.pub_server_channel.pub_close()
# Really terminate any process still left behind
for pid in self.process_manager._process_map:
terminate_process(pid=pid, kill_children=True, slow_stop=False)
self.process_manager = None
def test_master_startup(self):
proc = NonBlockingPopen(
[
sys.executable,
self.get_script_path("master"),
"-c",
RUNTIME_VARS.TMP_CONF_DIR,
"-l",
"info",
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out = b""
err = b""
# Testing this should never be longer than 1 minute
max_time = time.time() + 60
try:
while True:
if time.time() > max_time:
assert False, "Max timeout ocurred"
time.sleep(0.5)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if b"DeprecationWarning: object() takes no parameters" in out:
self.fail(
"'DeprecationWarning: object() takes no parameters' was seen in"
" output")
if b"TypeError: object() takes no parameters" in out:
self.fail(
"'TypeError: object() takes no parameters' was seen in output"
)
if b"Setting up the master communication server" in out:
# We got past the place we need, stop the process
break
if out is None and err is None:
break
if proc.poll() is not None:
break
finally:
terminate_process(proc.pid, kill_children=True)
def test_interrupt_on_long_running_job(salt_cli, salt_master, salt_minion):
"""
Ensure that a call to ``salt`` that is taking too long, when a user
hits CTRL-C, that the JID is printed to the console.
Refer to https://github.com/saltstack/salt/issues/60963 for more details
"""
# Ensure test.sleep is working as supposed
start = time.time()
ret = salt_cli.run("test.sleep", "1", minion_tgt=salt_minion.id)
stop = time.time()
assert ret.returncode == 0
assert ret.data is True
assert stop - start > 1, "The command should have taken more than 1 second"
# Now the real test
terminal_stdout = tempfile.SpooledTemporaryFile(512000, buffering=0)
terminal_stderr = tempfile.SpooledTemporaryFile(512000, buffering=0)
cmdline = [
sys.executable,
salt_cli.get_script_path(),
"--config-dir={}".format(salt_master.config_dir),
salt_minion.id,
"test.sleep",
"30",
]
# If this test starts failing, commend the following block of code
proc = subprocess.Popen(
cmdline,
shell=False,
stdout=terminal_stdout,
stderr=terminal_stderr,
universal_newlines=True,
)
# and uncomment the following block of code
# with default_signals(signal.SIGINT, signal.SIGTERM):
# proc = subprocess.Popen(
# cmdline,
# shell=False,
# stdout=terminal_stdout,
# stderr=terminal_stderr,
# universal_newlines=True,
# )
# What this means is that something in salt or the test suite is setting
# the SIGTERM and SIGINT signals to SIG_IGN, ignore.
# Check which line of code is doing that and fix it
start = time.time()
try:
# Make sure it actually starts
proc.wait(1)
except subprocess.TimeoutExpired:
pass
else:
terminate_process(proc.pid, kill_children=True)
pytest.fail("The test process failed to start")
time.sleep(2)
# Send CTRL-C to the process
os.kill(proc.pid, signal.SIGINT)
with proc:
# Wait for the process to terminate, to avoid zombies.
# Shouldn't really take the 30 seconds
proc.wait(30)
# poll the terminal so the right returncode is set on the popen object
proc.poll()
# This call shouldn't really be necessary
proc.communicate()
stop = time.time()
terminal_stdout.flush()
terminal_stdout.seek(0)
if sys.version_info < (3, 6): # pragma: no cover
stdout = proc._translate_newlines(terminal_stdout.read(),
__salt_system_encoding__)
else:
stdout = proc._translate_newlines(terminal_stdout.read(),
__salt_system_encoding__,
sys.stdout.errors)
terminal_stdout.close()
terminal_stderr.flush()
terminal_stderr.seek(0)
if sys.version_info < (3, 6): # pragma: no cover
stderr = proc._translate_newlines(terminal_stderr.read(),
__salt_system_encoding__)
else:
stderr = proc._translate_newlines(terminal_stderr.read(),
__salt_system_encoding__,
sys.stderr.errors)
terminal_stderr.close()
ret = ProcessResult(returncode=proc.returncode,
stdout=stdout,
stderr=stderr,
cmdline=proc.args)
log.debug(ret)
# If the minion ID is on stdout it means that the command finished and wasn't terminated
assert (
salt_minion.id not in ret.stdout
), "The command wasn't actually terminated. Took {} seconds.".format(
round(stop - start, 2))
# Make sure the ctrl+c exited gracefully
assert "Exiting gracefully on Ctrl-c" in ret.stderr
assert "Exception ignored in" not in ret.stderr
assert "This job's jid is" in ret.stderr
def test_setup(salt_ssh_cli, ssh_container_name, ssh_sub_container_name,
ssh_password):
"""
Test salt-ssh grains id work for localhost.
"""
# Provide the passwd from the CLI to allow the key deploy
possible_ids = (ssh_container_name, ssh_sub_container_name)
ret = salt_ssh_cli.run("--passwd",
ssh_password,
"--key-deploy",
"grains.get",
"id",
minion_tgt="*")
assert ret.returncode == 0
for id in possible_ids:
assert id in ret.data
assert ret.data[id] == id
# Run it again without the key deploy
ret = salt_ssh_cli.run("grains.get", "id", minion_tgt="*")
assert ret.returncode == 0
for id in possible_ids:
assert id in ret.data
assert ret.data[id] == id
# Run a test.sleep and kill it
sleep_time = 15
cmdline = salt_ssh_cli.cmdline("test.sleep", sleep_time, minion_tgt="*")
terminal_stdout = tempfile.SpooledTemporaryFile(512000, buffering=0)
terminal_stderr = tempfile.SpooledTemporaryFile(512000, buffering=0)
proc = subprocess.Popen(
cmdline,
shell=False,
stdout=terminal_stdout,
stderr=terminal_stderr,
universal_newlines=True,
)
start = time.time()
try:
# Make sure it actually starts
proc.wait(1)
except subprocess.TimeoutExpired:
pass
else:
terminate_process(proc.pid, kill_children=True)
pytest.fail("The test process failed to start")
time.sleep(2)
# Send CTRL-C to the process
os.kill(proc.pid, signal.SIGINT)
with proc:
# Wait for the process to terminate, to avoid zombies.
# Shouldn't really take the 30 seconds
proc.wait(sleep_time * 2)
# poll the terminal so the right returncode is set on the popen object
proc.poll()
# This call shouldn't really be necessary
proc.communicate()
stop = time.time()
terminal_stdout.flush()
terminal_stdout.seek(0)
if sys.version_info < (3, 6): # pragma: no cover
stdout = proc._translate_newlines(terminal_stdout.read(),
__salt_system_encoding__)
else:
stdout = proc._translate_newlines(terminal_stdout.read(),
__salt_system_encoding__,
sys.stdout.errors)
terminal_stdout.close()
terminal_stderr.flush()
terminal_stderr.seek(0)
if sys.version_info < (3, 6): # pragma: no cover
stderr = proc._translate_newlines(terminal_stderr.read(),
__salt_system_encoding__)
else:
stderr = proc._translate_newlines(terminal_stderr.read(),
__salt_system_encoding__,
sys.stderr.errors)
terminal_stderr.close()
ret = ProcessResult(returncode=proc.returncode,
stdout=stdout,
stderr=stderr,
cmdline=proc.args)
log.debug(ret)
# If the minion ID is on stdout it means that the command finished and wasn't terminated
for id in possible_ids:
assert (
id not in ret.stdout
), "The command wasn't actually terminated. Took {} seconds.".format(
round(stop - start, 2))
def _terminate(self):
"""
This method actually terminates the started daemon.
"""
# We completely override the parent class method because we're not using the
# self._terminal property, it's a systemd service
if self._process is None: # pragma: no cover
if TYPE_CHECKING:
# Make mypy happy
assert self._terminal_result
return self._terminal_result # pylint: disable=access-member-before-definition
atexit.unregister(self.terminate)
log.info("Stopping %s", self.factory)
pid = self.pid
# Collect any child processes information before terminating the process
with contextlib.suppress(psutil.NoSuchProcess):
for child in psutil.Process(pid).children(recursive=True):
if child not in self._children: # pylint: disable=access-member-before-definition
self._children.append(child) # pylint: disable=access-member-before-definition
if self._process.is_running(): # pragma: no cover
cmdline = self._process.cmdline()
else:
# The main pid is not longer alive, try to get the cmdline from systemd
ret = self._internal_run("systemctl", "show", "-p", "ExecStart",
self.get_service_name())
cmdline = ret.stdout.split("argv[]=")[-1].split(
";")[0].strip().split()
# Tell systemd to stop the service
self._internal_run("systemctl", "stop", self.get_service_name())
if self._process.is_running(): # pragma: no cover
cmdline = self._process.cmdline()
try:
self._process.wait()
except psutil.TimeoutExpired:
self._process.terminate()
try:
self._process.wait()
except psutil.TimeoutExpired:
pass
exitcode = self._process.wait() or 0
# Dereference the internal _process attribute
self._process = None
# Lets log and kill any child processes left behind, including the main subprocess
# if it failed to properly stop
terminate_process(
pid=pid,
kill_children=True,
children=self._children, # pylint: disable=access-member-before-definition
slow_stop=self.factory.slow_stop,
)
if self._terminal_stdout is not None:
self._terminal_stdout.close() # pylint: disable=access-member-before-definition
if self._terminal_stderr is not None:
self._terminal_stderr.close() # pylint: disable=access-member-before-definition
stdout = ""
ret = self._internal_run("journalctl", "--no-pager", "-u",
self.get_service_name())
stderr = ret.stdout
try:
self._terminal_result = ProcessResult(returncode=exitcode,
stdout=stdout,
stderr=stderr,
cmdline=cmdline)
log.info("%s %s", self.factory.__class__.__name__,
self._terminal_result)
return self._terminal_result
finally:
self._terminal = None
self._terminal_stdout = None
self._terminal_stderr = None
self._terminal_timeout = None
self._children = []
def test_deferred_write_on_atexit(tmp_path):
# Python will .flush() and .close() all logging handlers at interpreter shutdown.
# This should be enough to flush our deferred messages.
pyscript = dedent(r"""
import sys
import time
import logging
CODE_DIR = {!r}
if CODE_DIR in sys.path:
sys.path.remove(CODE_DIR)
sys.path.insert(0, CODE_DIR)
from salt._logging.handlers import DeferredStreamHandler
# Reset any logging handlers we might have already
logging.root.handlers[:] = []
handler = DeferredStreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
logging.root.addHandler(handler)
log = logging.getLogger(__name__)
sys.stdout.write('STARTED\n')
sys.stdout.flush()
log.debug('Foo')
sys.exit(0)
""".format(RUNTIME_VARS.CODE_DIR))
script_path = tmp_path / "atexit_deferred_logging_test.py"
script_path.write_text(pyscript, encoding="utf-8")
proc = NonBlockingPopen(
[sys.executable, str(script_path)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out = b""
err = b""
# This test should never take more than 5 seconds
execution_time = 5
max_time = time.time() + execution_time
try:
# Just loop consuming output
while True:
if time.time() > max_time:
pytest.fail("Script didn't exit after {} second".format(
execution_time))
time.sleep(0.125)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if _out is None and _err is None:
# The script exited
break
if proc.poll() is not None:
# The script exited
break
finally:
terminate_process(proc.pid, kill_children=True)
if b"Foo" not in err:
pytest.fail("'Foo' should be in stderr and it's not: {}".format(err))
def test_deferred_write_on_sigint(tmp_path):
pyscript = dedent(r"""
import sys
import time
import signal
import logging
CODE_DIR = {!r}
if CODE_DIR in sys.path:
sys.path.remove(CODE_DIR)
sys.path.insert(0, CODE_DIR)
from salt._logging.handlers import DeferredStreamHandler
# Reset any logging handlers we might have already
logging.root.handlers[:] = []
handler = DeferredStreamHandler(sys.stderr)
handler.setLevel(logging.DEBUG)
logging.root.addHandler(handler)
if signal.getsignal(signal.SIGINT) != signal.default_int_handler:
# Looking at you Debian based distros :/
signal.signal(signal.SIGINT, signal.default_int_handler)
log = logging.getLogger(__name__)
start_printed = False
while True:
try:
log.debug('Foo')
if start_printed is False:
sys.stdout.write('STARTED\n')
sys.stdout.write('SIGINT HANDLER: {{!r}}\n'.format(signal.getsignal(signal.SIGINT)))
sys.stdout.flush()
start_printed = True
time.sleep(0.125)
except (KeyboardInterrupt, SystemExit):
log.info('KeyboardInterrupt caught')
sys.stdout.write('KeyboardInterrupt caught\n')
sys.stdout.flush()
break
log.info('EXITING')
sys.stdout.write('EXITING\n')
sys.stdout.flush()
sys.exit(0)
""".format(RUNTIME_VARS.CODE_DIR))
script_path = tmp_path / "sigint_deferred_logging_test.py"
script_path.write_text(pyscript, encoding="utf-8")
proc = NonBlockingPopen(
[sys.executable, str(script_path)],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
out = b""
err = b""
# Test should take less than 20 seconds, way less
execution_time = 10
start = time.time()
max_time = time.time() + execution_time
try:
signalled = False
log.info("Starting Loop")
while True:
time.sleep(0.125)
_out = proc.recv()
_err = proc.recv_err()
if _out:
out += _out
if _err:
err += _err
if b"STARTED" in out and not signalled:
# Enough time has passed
proc.send_signal(signal.SIGINT)
signalled = True
log.debug("Sent SIGINT after: %s", time.time() - start)
if signalled is False:
if out:
pytest.fail(
"We have stdout output when there should be none: {}".
format(out))
if err:
pytest.fail(
"We have stderr output when there should be none: {}".
format(err))
if _out is None and _err is None:
log.info("_out and _err are None")
if b"Foo" not in err:
pytest.fail(
"No more output and 'Foo' should be in stderr and it's not: {}"
.format(err))
break
if proc.poll() is not None:
log.debug("poll() is not None")
if b"Foo" not in err:
pytest.fail(
"Process terminated and 'Foo' should be in stderr and it's not: {}"
.format(err))
break
if time.time() > max_time:
log.debug("Reached max time")
if b"Foo" not in err:
pytest.fail(
"'Foo' should be in stderr and it's not:\n{0}\nSTDERR:\n{0}\n{1}\n{0}\nSTDOUT:\n{0}\n{2}\n{0}"
.format("-" * 80, err, out))
finally:
terminate_process(proc.pid, kill_children=True)
log.debug("Test took %s seconds", time.time() - start)
def run_script(
self,
script,
arg_str,
catch_stderr=False,
with_retcode=False,
catch_timeout=False,
# FIXME A timeout of zero or disabling timeouts may not return results!
timeout=15,
raw=False,
popen_kwargs=None,
log_output=None,
config_dir=None,
**kwargs):
"""
Execute a script with the given argument string
The ``log_output`` argument is ternary, it can be True, False, or None.
If the value is boolean, then it forces the results to either be logged
or not logged. If it is None, then the return code of the subprocess
determines whether or not to log results.
"""
import salt.utils.platform
script_path = self.get_script_path(script)
if not os.path.isfile(script_path):
return False
popen_kwargs = popen_kwargs or {}
python_path_env_var = os.environ.get("PYTHONPATH") or None
if python_path_env_var is None:
python_path_entries = [RUNTIME_VARS.CODE_DIR]
else:
python_path_entries = python_path_env_var.split(os.pathsep)
if RUNTIME_VARS.CODE_DIR in python_path_entries:
python_path_entries.remove(RUNTIME_VARS.CODE_DIR)
python_path_entries.insert(0, RUNTIME_VARS.CODE_DIR)
python_path_entries.extend(sys.path[0:])
if "env" not in popen_kwargs:
popen_kwargs["env"] = os.environ.copy()
popen_kwargs["env"]["PYTHONPATH"] = os.pathsep.join(
python_path_entries)
if "cwd" not in popen_kwargs:
popen_kwargs["cwd"] = RUNTIME_VARS.TMP
if salt.utils.platform.is_windows():
cmd = "python "
else:
cmd = "python{}.{} ".format(*sys.version_info)
cmd += "{} --config-dir={} {} ".format(
script_path, config_dir or RUNTIME_VARS.TMP_CONF_DIR, arg_str)
if kwargs:
# late import
import salt.utils.json
for key, value in kwargs.items():
cmd += "'{}={} '".format(key, salt.utils.json.dumps(value))
tmp_file = tempfile.SpooledTemporaryFile()
popen_kwargs = dict(
{
"shell": True,
"stdout": tmp_file,
"universal_newlines": True
}, **popen_kwargs)
if catch_stderr is True:
popen_kwargs["stderr"] = subprocess.PIPE
if salt.utils.platform.is_windows():
# Windows does not support closing FDs
close_fds = False
elif salt.utils.platform.is_freebsd() and sys.version_info < (3, 9):
# Closing FDs in FreeBSD before Py3.9 can be slow
# https://bugs.python.org/issue38061
close_fds = False
else:
close_fds = True
popen_kwargs["close_fds"] = close_fds
if not salt.utils.platform.is_windows():
def detach_from_parent_group():
# detach from parent group (no more inherited signals!)
os.setpgrp()
popen_kwargs["preexec_fn"] = detach_from_parent_group
def format_return(retcode, stdout, stderr=None, timed_out=False):
"""
DRY helper to log script result if it failed, and then return the
desired output based on whether or not stderr was desired, and
wither or not a retcode was desired.
"""
log_func = log.debug
if timed_out:
log.error(
"run_script timed out after %d seconds (process killed)",
timeout)
log_func = log.error
if log_output is True or timed_out or (log_output is None
and retcode != 0):
log_func(
"run_script results for: %s %s\n"
"return code: %s\n"
"stdout:\n"
"%s\n\n"
"stderr:\n"
"%s",
script,
arg_str,
retcode,
stdout,
stderr,
)
stdout = stdout or ""
stderr = stderr or ""
if not raw:
stdout = stdout.splitlines()
stderr = stderr.splitlines()
ret = [stdout]
if catch_stderr:
ret.append(stderr)
if with_retcode:
ret.append(retcode)
if catch_timeout:
ret.append(timed_out)
return ret[0] if len(ret) == 1 else tuple(ret)
log.debug("Running Popen(%r, %r)", cmd, popen_kwargs)
process = subprocess.Popen(cmd, **popen_kwargs)
if timeout is not None:
stop_at = datetime.now() + timedelta(seconds=timeout)
while True:
process.poll()
time.sleep(0.1)
if datetime.now() <= stop_at:
# We haven't reached the timeout yet
if process.returncode is not None:
break
else:
terminate_process(process.pid, kill_children=True)
return format_return(process.returncode,
*process.communicate(),
timed_out=True)
tmp_file.seek(0)
try:
out = tmp_file.read().decode(__salt_system_encoding__)
except (NameError, UnicodeDecodeError):
# Let's cross our fingers and hope for the best
out = tmp_file.read().decode("utf-8")
if catch_stderr:
_, err = process.communicate()
# Force closing stderr/stdout to release file descriptors
if process.stdout is not None:
process.stdout.close()
if process.stderr is not None:
process.stderr.close()
# pylint: disable=maybe-no-member
try:
return format_return(process.returncode, out, err or "")
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, str):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass
# pylint: enable=maybe-no-member
# TODO Remove this?
process.communicate()
if process.stdout is not None:
process.stdout.close()
try:
return format_return(process.returncode, out)
finally:
try:
if os.path.exists(tmp_file.name):
if isinstance(tmp_file.name, str):
# tmp_file.name is an int when using SpooledTemporaryFiles
# int types cannot be used with os.remove() in Python 3
os.remove(tmp_file.name)
else:
# Clean up file handles
tmp_file.close()
process.terminate()
except OSError as err:
# process already terminated
pass