def hgfs_setup_and_teardown(): """ build up and tear down hg repos to test with. """ initial_child_processes = psutil.Process().children() source_dir = Path(__file__).resolve().parent.joinpath("files") tempdir = tempfile.TemporaryDirectory() tempsubdir = tempdir.name / Path("subdir/") tempsubdir.mkdir() tempdirPath = Path(tempdir.name) for file in source_dir.iterdir(): to_file = tempdirPath / file.name to_file2 = tempsubdir / file.name shutil.copy(file.as_posix(), to_file.as_posix()) shutil.copy(file.as_posix(), to_file2.as_posix()) client = hglib.init(bytes(tempdirPath.as_posix(), encoding="utf8")) client.close() with hglib.open(bytes(tempdirPath.as_posix(), encoding="utf8")) as repo: repo.add(bytes(tempdirPath.as_posix(), encoding="utf8")) repo.commit(b"init commit", user="******") repo.tag(b"test", user="******") repo.branch(b"test") repo.commit(b"create test branch", user="******") repo.bookmark(b"bookmark_test") try: yield tempdirPath.as_uri() finally: tempdir.cleanup() for child in psutil.Process().children(): if child not in initial_child_processes: terminate_process(process=child, kill_children=True)
def schedule(tmp_path): subprocess_list = None try: subprocess_list = SubprocessList() root_dir = tmp_path / "schedule-unit-tests" sock_dir = str(root_dir / "test-socks") default_config = salt.config.minion_config(None) default_config["conf_dir"] = str(root_dir) default_config["root_dir"] = str(root_dir) default_config["sock_dir"] = sock_dir default_config["pki_dir"] = str(root_dir / "pki") default_config["cachedir"] = str(root_dir / "cache") with patch("salt.utils.schedule.clean_proc_dir", MagicMock(return_value=None)): functions = {"test.ping": ping} _schedule = salt.utils.schedule.Schedule( copy.deepcopy(default_config), functions, returners={}, new_instance=True, ) _schedule.opts["loop_interval"] = 1 _schedule._subprocess_list = subprocess_list yield _schedule finally: if subprocess_list: processes = subprocess_list.processes _schedule.reset() del _schedule for proc in processes: if proc.is_alive(): terminate_process(proc.pid, kill_children=True, slow_stop=True) subprocess_list.cleanup() processes = subprocess_list.processes if processes: for proc in processes: if proc.is_alive(): terminate_process(proc.pid, kill_children=True, slow_stop=False) subprocess_list.cleanup() processes = subprocess_list.processes if processes: log.warning("Processes left running: %s", processes) del default_config del subprocess_list
def req_server_channel(salt_master, req_channel_crypt): req_server_channel_process = ReqServerChannelProcess( salt_master.config.copy(), req_channel_crypt) try: with req_server_channel_process: yield finally: terminate_process(pid=req_server_channel_process.pid, kill_children=True, slow_stop=False)
def test_event_return(): evt = None try: evt = salt.utils.event.EventReturn( salt.config.DEFAULT_MASTER_OPTS.copy()) evt.start() except TypeError as exc: if "object" in str(exc): pytest.fail( "'{}' TypeError should have not been raised".format(exc)) finally: if evt is not None: terminate_process(evt.pid, kill_children=True)
def close(self): if self._closing: return self._closing = True if self.req_server_channel is not None: self.req_server_channel.close() self.req_server_channel = None if self.process_manager is not None: self.process_manager.terminate() # Really terminate any process still left behind for pid in self.process_manager._process_map: terminate_process(pid=pid, kill_children=True, slow_stop=False) self.process_manager = None
def close(self): if self._closing: return self._closing = True if self.process_manager is None: return self.process_manager.terminate() if hasattr(self.pub_server_channel, "pub_close"): self.pub_server_channel.pub_close() # Really terminate any process still left behind for pid in self.process_manager._process_map: terminate_process(pid=pid, kill_children=True, slow_stop=False) self.process_manager = None
def test_master_startup(self): proc = NonBlockingPopen( [ sys.executable, self.get_script_path("master"), "-c", RUNTIME_VARS.TMP_CONF_DIR, "-l", "info", ], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) out = b"" err = b"" # Testing this should never be longer than 1 minute max_time = time.time() + 60 try: while True: if time.time() > max_time: assert False, "Max timeout ocurred" time.sleep(0.5) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if b"DeprecationWarning: object() takes no parameters" in out: self.fail( "'DeprecationWarning: object() takes no parameters' was seen in" " output") if b"TypeError: object() takes no parameters" in out: self.fail( "'TypeError: object() takes no parameters' was seen in output" ) if b"Setting up the master communication server" in out: # We got past the place we need, stop the process break if out is None and err is None: break if proc.poll() is not None: break finally: terminate_process(proc.pid, kill_children=True)
def test_interrupt_on_long_running_job(salt_cli, salt_master, salt_minion): """ Ensure that a call to ``salt`` that is taking too long, when a user hits CTRL-C, that the JID is printed to the console. Refer to https://github.com/saltstack/salt/issues/60963 for more details """ # Ensure test.sleep is working as supposed start = time.time() ret = salt_cli.run("test.sleep", "1", minion_tgt=salt_minion.id) stop = time.time() assert ret.returncode == 0 assert ret.data is True assert stop - start > 1, "The command should have taken more than 1 second" # Now the real test terminal_stdout = tempfile.SpooledTemporaryFile(512000, buffering=0) terminal_stderr = tempfile.SpooledTemporaryFile(512000, buffering=0) cmdline = [ sys.executable, salt_cli.get_script_path(), "--config-dir={}".format(salt_master.config_dir), salt_minion.id, "test.sleep", "30", ] # If this test starts failing, commend the following block of code proc = subprocess.Popen( cmdline, shell=False, stdout=terminal_stdout, stderr=terminal_stderr, universal_newlines=True, ) # and uncomment the following block of code # with default_signals(signal.SIGINT, signal.SIGTERM): # proc = subprocess.Popen( # cmdline, # shell=False, # stdout=terminal_stdout, # stderr=terminal_stderr, # universal_newlines=True, # ) # What this means is that something in salt or the test suite is setting # the SIGTERM and SIGINT signals to SIG_IGN, ignore. # Check which line of code is doing that and fix it start = time.time() try: # Make sure it actually starts proc.wait(1) except subprocess.TimeoutExpired: pass else: terminate_process(proc.pid, kill_children=True) pytest.fail("The test process failed to start") time.sleep(2) # Send CTRL-C to the process os.kill(proc.pid, signal.SIGINT) with proc: # Wait for the process to terminate, to avoid zombies. # Shouldn't really take the 30 seconds proc.wait(30) # poll the terminal so the right returncode is set on the popen object proc.poll() # This call shouldn't really be necessary proc.communicate() stop = time.time() terminal_stdout.flush() terminal_stdout.seek(0) if sys.version_info < (3, 6): # pragma: no cover stdout = proc._translate_newlines(terminal_stdout.read(), __salt_system_encoding__) else: stdout = proc._translate_newlines(terminal_stdout.read(), __salt_system_encoding__, sys.stdout.errors) terminal_stdout.close() terminal_stderr.flush() terminal_stderr.seek(0) if sys.version_info < (3, 6): # pragma: no cover stderr = proc._translate_newlines(terminal_stderr.read(), __salt_system_encoding__) else: stderr = proc._translate_newlines(terminal_stderr.read(), __salt_system_encoding__, sys.stderr.errors) terminal_stderr.close() ret = ProcessResult(returncode=proc.returncode, stdout=stdout, stderr=stderr, cmdline=proc.args) log.debug(ret) # If the minion ID is on stdout it means that the command finished and wasn't terminated assert ( salt_minion.id not in ret.stdout ), "The command wasn't actually terminated. Took {} seconds.".format( round(stop - start, 2)) # Make sure the ctrl+c exited gracefully assert "Exiting gracefully on Ctrl-c" in ret.stderr assert "Exception ignored in" not in ret.stderr assert "This job's jid is" in ret.stderr
def test_setup(salt_ssh_cli, ssh_container_name, ssh_sub_container_name, ssh_password): """ Test salt-ssh grains id work for localhost. """ # Provide the passwd from the CLI to allow the key deploy possible_ids = (ssh_container_name, ssh_sub_container_name) ret = salt_ssh_cli.run("--passwd", ssh_password, "--key-deploy", "grains.get", "id", minion_tgt="*") assert ret.returncode == 0 for id in possible_ids: assert id in ret.data assert ret.data[id] == id # Run it again without the key deploy ret = salt_ssh_cli.run("grains.get", "id", minion_tgt="*") assert ret.returncode == 0 for id in possible_ids: assert id in ret.data assert ret.data[id] == id # Run a test.sleep and kill it sleep_time = 15 cmdline = salt_ssh_cli.cmdline("test.sleep", sleep_time, minion_tgt="*") terminal_stdout = tempfile.SpooledTemporaryFile(512000, buffering=0) terminal_stderr = tempfile.SpooledTemporaryFile(512000, buffering=0) proc = subprocess.Popen( cmdline, shell=False, stdout=terminal_stdout, stderr=terminal_stderr, universal_newlines=True, ) start = time.time() try: # Make sure it actually starts proc.wait(1) except subprocess.TimeoutExpired: pass else: terminate_process(proc.pid, kill_children=True) pytest.fail("The test process failed to start") time.sleep(2) # Send CTRL-C to the process os.kill(proc.pid, signal.SIGINT) with proc: # Wait for the process to terminate, to avoid zombies. # Shouldn't really take the 30 seconds proc.wait(sleep_time * 2) # poll the terminal so the right returncode is set on the popen object proc.poll() # This call shouldn't really be necessary proc.communicate() stop = time.time() terminal_stdout.flush() terminal_stdout.seek(0) if sys.version_info < (3, 6): # pragma: no cover stdout = proc._translate_newlines(terminal_stdout.read(), __salt_system_encoding__) else: stdout = proc._translate_newlines(terminal_stdout.read(), __salt_system_encoding__, sys.stdout.errors) terminal_stdout.close() terminal_stderr.flush() terminal_stderr.seek(0) if sys.version_info < (3, 6): # pragma: no cover stderr = proc._translate_newlines(terminal_stderr.read(), __salt_system_encoding__) else: stderr = proc._translate_newlines(terminal_stderr.read(), __salt_system_encoding__, sys.stderr.errors) terminal_stderr.close() ret = ProcessResult(returncode=proc.returncode, stdout=stdout, stderr=stderr, cmdline=proc.args) log.debug(ret) # If the minion ID is on stdout it means that the command finished and wasn't terminated for id in possible_ids: assert ( id not in ret.stdout ), "The command wasn't actually terminated. Took {} seconds.".format( round(stop - start, 2))
def _terminate(self): """ This method actually terminates the started daemon. """ # We completely override the parent class method because we're not using the # self._terminal property, it's a systemd service if self._process is None: # pragma: no cover if TYPE_CHECKING: # Make mypy happy assert self._terminal_result return self._terminal_result # pylint: disable=access-member-before-definition atexit.unregister(self.terminate) log.info("Stopping %s", self.factory) pid = self.pid # Collect any child processes information before terminating the process with contextlib.suppress(psutil.NoSuchProcess): for child in psutil.Process(pid).children(recursive=True): if child not in self._children: # pylint: disable=access-member-before-definition self._children.append(child) # pylint: disable=access-member-before-definition if self._process.is_running(): # pragma: no cover cmdline = self._process.cmdline() else: # The main pid is not longer alive, try to get the cmdline from systemd ret = self._internal_run("systemctl", "show", "-p", "ExecStart", self.get_service_name()) cmdline = ret.stdout.split("argv[]=")[-1].split( ";")[0].strip().split() # Tell systemd to stop the service self._internal_run("systemctl", "stop", self.get_service_name()) if self._process.is_running(): # pragma: no cover cmdline = self._process.cmdline() try: self._process.wait() except psutil.TimeoutExpired: self._process.terminate() try: self._process.wait() except psutil.TimeoutExpired: pass exitcode = self._process.wait() or 0 # Dereference the internal _process attribute self._process = None # Lets log and kill any child processes left behind, including the main subprocess # if it failed to properly stop terminate_process( pid=pid, kill_children=True, children=self._children, # pylint: disable=access-member-before-definition slow_stop=self.factory.slow_stop, ) if self._terminal_stdout is not None: self._terminal_stdout.close() # pylint: disable=access-member-before-definition if self._terminal_stderr is not None: self._terminal_stderr.close() # pylint: disable=access-member-before-definition stdout = "" ret = self._internal_run("journalctl", "--no-pager", "-u", self.get_service_name()) stderr = ret.stdout try: self._terminal_result = ProcessResult(returncode=exitcode, stdout=stdout, stderr=stderr, cmdline=cmdline) log.info("%s %s", self.factory.__class__.__name__, self._terminal_result) return self._terminal_result finally: self._terminal = None self._terminal_stdout = None self._terminal_stderr = None self._terminal_timeout = None self._children = []
def test_deferred_write_on_atexit(tmp_path): # Python will .flush() and .close() all logging handlers at interpreter shutdown. # This should be enough to flush our deferred messages. pyscript = dedent(r""" import sys import time import logging CODE_DIR = {!r} if CODE_DIR in sys.path: sys.path.remove(CODE_DIR) sys.path.insert(0, CODE_DIR) from salt._logging.handlers import DeferredStreamHandler # Reset any logging handlers we might have already logging.root.handlers[:] = [] handler = DeferredStreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) logging.root.addHandler(handler) log = logging.getLogger(__name__) sys.stdout.write('STARTED\n') sys.stdout.flush() log.debug('Foo') sys.exit(0) """.format(RUNTIME_VARS.CODE_DIR)) script_path = tmp_path / "atexit_deferred_logging_test.py" script_path.write_text(pyscript, encoding="utf-8") proc = NonBlockingPopen( [sys.executable, str(script_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out = b"" err = b"" # This test should never take more than 5 seconds execution_time = 5 max_time = time.time() + execution_time try: # Just loop consuming output while True: if time.time() > max_time: pytest.fail("Script didn't exit after {} second".format( execution_time)) time.sleep(0.125) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if _out is None and _err is None: # The script exited break if proc.poll() is not None: # The script exited break finally: terminate_process(proc.pid, kill_children=True) if b"Foo" not in err: pytest.fail("'Foo' should be in stderr and it's not: {}".format(err))
def test_deferred_write_on_sigint(tmp_path): pyscript = dedent(r""" import sys import time import signal import logging CODE_DIR = {!r} if CODE_DIR in sys.path: sys.path.remove(CODE_DIR) sys.path.insert(0, CODE_DIR) from salt._logging.handlers import DeferredStreamHandler # Reset any logging handlers we might have already logging.root.handlers[:] = [] handler = DeferredStreamHandler(sys.stderr) handler.setLevel(logging.DEBUG) logging.root.addHandler(handler) if signal.getsignal(signal.SIGINT) != signal.default_int_handler: # Looking at you Debian based distros :/ signal.signal(signal.SIGINT, signal.default_int_handler) log = logging.getLogger(__name__) start_printed = False while True: try: log.debug('Foo') if start_printed is False: sys.stdout.write('STARTED\n') sys.stdout.write('SIGINT HANDLER: {{!r}}\n'.format(signal.getsignal(signal.SIGINT))) sys.stdout.flush() start_printed = True time.sleep(0.125) except (KeyboardInterrupt, SystemExit): log.info('KeyboardInterrupt caught') sys.stdout.write('KeyboardInterrupt caught\n') sys.stdout.flush() break log.info('EXITING') sys.stdout.write('EXITING\n') sys.stdout.flush() sys.exit(0) """.format(RUNTIME_VARS.CODE_DIR)) script_path = tmp_path / "sigint_deferred_logging_test.py" script_path.write_text(pyscript, encoding="utf-8") proc = NonBlockingPopen( [sys.executable, str(script_path)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) out = b"" err = b"" # Test should take less than 20 seconds, way less execution_time = 10 start = time.time() max_time = time.time() + execution_time try: signalled = False log.info("Starting Loop") while True: time.sleep(0.125) _out = proc.recv() _err = proc.recv_err() if _out: out += _out if _err: err += _err if b"STARTED" in out and not signalled: # Enough time has passed proc.send_signal(signal.SIGINT) signalled = True log.debug("Sent SIGINT after: %s", time.time() - start) if signalled is False: if out: pytest.fail( "We have stdout output when there should be none: {}". format(out)) if err: pytest.fail( "We have stderr output when there should be none: {}". format(err)) if _out is None and _err is None: log.info("_out and _err are None") if b"Foo" not in err: pytest.fail( "No more output and 'Foo' should be in stderr and it's not: {}" .format(err)) break if proc.poll() is not None: log.debug("poll() is not None") if b"Foo" not in err: pytest.fail( "Process terminated and 'Foo' should be in stderr and it's not: {}" .format(err)) break if time.time() > max_time: log.debug("Reached max time") if b"Foo" not in err: pytest.fail( "'Foo' should be in stderr and it's not:\n{0}\nSTDERR:\n{0}\n{1}\n{0}\nSTDOUT:\n{0}\n{2}\n{0}" .format("-" * 80, err, out)) finally: terminate_process(proc.pid, kill_children=True) log.debug("Test took %s seconds", time.time() - start)
def run_script( self, script, arg_str, catch_stderr=False, with_retcode=False, catch_timeout=False, # FIXME A timeout of zero or disabling timeouts may not return results! timeout=15, raw=False, popen_kwargs=None, log_output=None, config_dir=None, **kwargs): """ Execute a script with the given argument string The ``log_output`` argument is ternary, it can be True, False, or None. If the value is boolean, then it forces the results to either be logged or not logged. If it is None, then the return code of the subprocess determines whether or not to log results. """ import salt.utils.platform script_path = self.get_script_path(script) if not os.path.isfile(script_path): return False popen_kwargs = popen_kwargs or {} python_path_env_var = os.environ.get("PYTHONPATH") or None if python_path_env_var is None: python_path_entries = [RUNTIME_VARS.CODE_DIR] else: python_path_entries = python_path_env_var.split(os.pathsep) if RUNTIME_VARS.CODE_DIR in python_path_entries: python_path_entries.remove(RUNTIME_VARS.CODE_DIR) python_path_entries.insert(0, RUNTIME_VARS.CODE_DIR) python_path_entries.extend(sys.path[0:]) if "env" not in popen_kwargs: popen_kwargs["env"] = os.environ.copy() popen_kwargs["env"]["PYTHONPATH"] = os.pathsep.join( python_path_entries) if "cwd" not in popen_kwargs: popen_kwargs["cwd"] = RUNTIME_VARS.TMP if salt.utils.platform.is_windows(): cmd = "python " else: cmd = "python{}.{} ".format(*sys.version_info) cmd += "{} --config-dir={} {} ".format( script_path, config_dir or RUNTIME_VARS.TMP_CONF_DIR, arg_str) if kwargs: # late import import salt.utils.json for key, value in kwargs.items(): cmd += "'{}={} '".format(key, salt.utils.json.dumps(value)) tmp_file = tempfile.SpooledTemporaryFile() popen_kwargs = dict( { "shell": True, "stdout": tmp_file, "universal_newlines": True }, **popen_kwargs) if catch_stderr is True: popen_kwargs["stderr"] = subprocess.PIPE if salt.utils.platform.is_windows(): # Windows does not support closing FDs close_fds = False elif salt.utils.platform.is_freebsd() and sys.version_info < (3, 9): # Closing FDs in FreeBSD before Py3.9 can be slow # https://bugs.python.org/issue38061 close_fds = False else: close_fds = True popen_kwargs["close_fds"] = close_fds if not salt.utils.platform.is_windows(): def detach_from_parent_group(): # detach from parent group (no more inherited signals!) os.setpgrp() popen_kwargs["preexec_fn"] = detach_from_parent_group def format_return(retcode, stdout, stderr=None, timed_out=False): """ DRY helper to log script result if it failed, and then return the desired output based on whether or not stderr was desired, and wither or not a retcode was desired. """ log_func = log.debug if timed_out: log.error( "run_script timed out after %d seconds (process killed)", timeout) log_func = log.error if log_output is True or timed_out or (log_output is None and retcode != 0): log_func( "run_script results for: %s %s\n" "return code: %s\n" "stdout:\n" "%s\n\n" "stderr:\n" "%s", script, arg_str, retcode, stdout, stderr, ) stdout = stdout or "" stderr = stderr or "" if not raw: stdout = stdout.splitlines() stderr = stderr.splitlines() ret = [stdout] if catch_stderr: ret.append(stderr) if with_retcode: ret.append(retcode) if catch_timeout: ret.append(timed_out) return ret[0] if len(ret) == 1 else tuple(ret) log.debug("Running Popen(%r, %r)", cmd, popen_kwargs) process = subprocess.Popen(cmd, **popen_kwargs) if timeout is not None: stop_at = datetime.now() + timedelta(seconds=timeout) while True: process.poll() time.sleep(0.1) if datetime.now() <= stop_at: # We haven't reached the timeout yet if process.returncode is not None: break else: terminate_process(process.pid, kill_children=True) return format_return(process.returncode, *process.communicate(), timed_out=True) tmp_file.seek(0) try: out = tmp_file.read().decode(__salt_system_encoding__) except (NameError, UnicodeDecodeError): # Let's cross our fingers and hope for the best out = tmp_file.read().decode("utf-8") if catch_stderr: _, err = process.communicate() # Force closing stderr/stdout to release file descriptors if process.stdout is not None: process.stdout.close() if process.stderr is not None: process.stderr.close() # pylint: disable=maybe-no-member try: return format_return(process.returncode, out, err or "") finally: try: if os.path.exists(tmp_file.name): if isinstance(tmp_file.name, str): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) else: # Clean up file handles tmp_file.close() process.terminate() except OSError as err: # process already terminated pass # pylint: enable=maybe-no-member # TODO Remove this? process.communicate() if process.stdout is not None: process.stdout.close() try: return format_return(process.returncode, out) finally: try: if os.path.exists(tmp_file.name): if isinstance(tmp_file.name, str): # tmp_file.name is an int when using SpooledTemporaryFiles # int types cannot be used with os.remove() in Python 3 os.remove(tmp_file.name) else: # Clean up file handles tmp_file.close() process.terminate() except OSError as err: # process already terminated pass