def salt_secondary_master(request, salt_factories):
#
# Enable a secondary Salt master so we can disable follow_symlinks
#
publish_port = ports.get_unused_localhost_port()
ret_port = ports.get_unused_localhost_port()
config_defaults = {
"open_mode": True,
"transport": request.config.getoption("--transport"),
}
config_overrides = {
"interface": "127.0.0.1",
"fileserver_followsymlinks": False,
"publish_port": publish_port,
"ret_port": ret_port,
}
factory = salt_factories.salt_master_daemon(
"secondary-master",
defaults=config_defaults,
overrides=config_overrides,
extra_cli_arguments_after_first_start_failure=["--log-level=debug"],
)
with factory.started(start_timeout=120):
yield factory
def master_config(root_dir, transport):
master_conf = salt.config.master_config("")
master_conf["transport"] = transport
master_conf["id"] = "master"
master_conf["root_dir"] = str(root_dir)
master_conf["sock_dir"] = str(root_dir)
master_conf["interface"] = "127.0.0.1"
master_conf["publish_port"] = ports.get_unused_localhost_port()
master_conf["ret_port"] = ports.get_unused_localhost_port()
master_conf["pki_dir"] = str(root_dir / "pki")
os.makedirs(master_conf["pki_dir"])
salt.crypt.gen_keys(master_conf["pki_dir"], "master", 4096)
minions_keys = os.path.join(master_conf["pki_dir"], "minions")
os.makedirs(minions_keys)
yield master_conf
def apply_pre_start_states(self, salt_call_cli, testclass, username):
# pylint: disable=access-member-before-definition
if self.listen_port in self.check_ports:
self.check_ports.remove(self.listen_port)
if self.listen_port in self.listen_ports:
self.listen_ports.remove(self.listen_port)
# pylint: enable=access-member-before-definition
self.listen_port = ports.get_unused_localhost_port()
self.check_ports.append(self.listen_port)
self.listen_ports.append(self.listen_port)
url = "ssh://{username}@127.0.0.1:{port}/~/repo.git".format(
username=testclass.username, port=self.listen_port)
url_extra_repo = "ssh://{username}@127.0.0.1:{port}/~/extra_repo.git".format(
username=testclass.username, port=self.listen_port)
home = "/root/.ssh"
testclass.ext_opts = {
"url":
url,
"url_extra_repo":
url_extra_repo,
"privkey_nopass":
os.path.join(home, testclass.id_rsa_nopass),
"pubkey_nopass":
os.path.join(home, testclass.id_rsa_nopass + ".pub"),
"privkey_withpass":
os.path.join(home, testclass.id_rsa_withpass),
"pubkey_withpass":
os.path.join(home, testclass.id_rsa_withpass + ".pub"),
"passphrase":
testclass.passphrase,
}
ret = salt_call_cli.run(
"state.apply",
mods="git_pillar.ssh",
pillar={
"git_pillar": {
"git_ssh": testclass.git_ssh,
"id_rsa_nopass": testclass.id_rsa_nopass,
"id_rsa_withpass": testclass.id_rsa_withpass,
"sshd_bin": self.get_script_path(),
"sshd_port": self.listen_port,
"sshd_config_dir": str(self.config_dir),
"master_user": username,
"user": testclass.username,
}
},
_timeout=240,
)
if ret.returncode != 0:
pytest.fail("Failed to apply the 'git_pillar.ssh' state")
if next(iter(ret.data.values()))["result"] is not True:
pytest.fail("Failed to apply the 'git_pillar.ssh' state")
def __attrs_post_init__(self):
"""
Post attrs initialization routines.
"""
if self.authorized_keys is None:
self.authorized_keys = []
if self.sshd_config_dict is None:
self.sshd_config_dict = {}
if self.listen_address is None:
self.listen_address = "127.0.0.1"
if self.listen_port is None:
self.listen_port = ports.get_unused_localhost_port()
self.check_ports = [self.listen_port]
if isinstance(self.config_dir, str):
self.config_dir = pathlib.Path(self.config_dir)
elif not isinstance(self.config_dir, pathlib.Path):
# A py local path?
self.config_dir = pathlib.Path(self.config_dir.strpath)
self.config_dir.chmod(0o0700)
authorized_keys_file = self.config_dir / "authorized_keys"
# Let's generate the client key
self.client_key = self._generate_client_ecdsa_key()
with open("{}.pub".format(self.client_key)) as rfh:
pubkey = rfh.read().strip()
log.debug("SSH client pub key: %r", pubkey)
self.authorized_keys.append(pubkey)
# Write the authorized pub keys to file
with open(str(authorized_keys_file), "w") as wfh:
wfh.write("\n".join(self.authorized_keys))
authorized_keys_file.chmod(0o0600)
with open(str(authorized_keys_file)) as rfh:
log.debug("AuthorizedKeysFile contents:\n%s", rfh.read())
_default_config = {
"ListenAddress": self.listen_address,
"PermitRootLogin": "******" if running_username() == "root" else "no",
"ChallengeResponseAuthentication": "no",
"PasswordAuthentication": "no",
"PubkeyAuthentication": "yes",
"PrintMotd": "no",
"PidFile": self.config_dir / "sshd.pid",
"AuthorizedKeysFile": authorized_keys_file,
}
if self.sshd_config_dict:
_default_config.update(self.sshd_config_dict)
self.sshd_config = _default_config
self._write_config()
super().__attrs_post_init__()
def __attrs_post_init__(self):
"""
Post attrs initialization routines.
"""
self.store = deque(maxlen=10000)
self.address = "tcp://127.0.0.1:{}".format(
ports.get_unused_localhost_port())
self.running_event = threading.Event()
self.running_thread = threading.Thread(target=self._run)
self.cleanup_thread = threading.Thread(target=self._cleanup)
self.sentinel = msgpack.dumps(None)
self.sentinel_event = threading.Event()
self.auth_event_handlers = weakref.WeakValueDictionary()
def apply_pre_start_states(self, salt_call_cli, testclass, root_dir):
if self.listen_port in self.check_ports:
self.check_ports.remove(self.listen_port)
if self.listen_port in self.listen_ports:
self.listen_ports.remove(self.listen_port)
self.listen_port = ports.get_unused_localhost_port()
self.check_ports.append(self.listen_port)
self.listen_ports.append(self.listen_port)
config_dir = os.path.join(root_dir, "config")
git_dir = os.path.join(root_dir, "git")
testclass.repo_dir = repo_dir = os.path.join(git_dir, "repos")
venv_dir = os.path.join(root_dir, "venv")
uwsgi_bin = os.path.join(venv_dir, "bin", "uwsgi")
pillar = {
"git_pillar": {
"config_dir": config_dir,
"git_dir": git_dir,
"venv_dir": venv_dir,
"root_dir": root_dir,
"uwsgi_port": self.listen_port,
}
}
# Different libexec dir for git backend on Debian and FreeBSD-based systems
if salt.utils.platform.is_freebsd():
git_core = "/usr/local/libexec/git-core"
else:
git_core = "/usr/libexec/git-core"
if not os.path.exists(git_core):
git_core = "/usr/lib/git-core"
if not os.path.exists(git_core):
pytest.fail(
"{} not found. Either git is not installed, or the test "
"class needs to be updated.".format(git_core)
)
pillar["git_pillar"]["git-http-backend"] = os.path.join(
git_core, "git-http-backend"
)
ret = salt_call_cli.run(
"state.apply", mods="git_pillar.http.uwsgi", pillar=pillar, _timeout=120
)
if ret.returncode != 0:
pytest.fail("Failed to apply the 'git_pillar.http.uwsgi' state")
if next(iter(ret.data.values()))["result"] is not True:
pytest.fail("Failed to apply the 'git_pillar.http.uwsgi' state")
def datagram_server():
logger = logging.getLogger("test_logstash_logger")
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
port = ports.get_unused_localhost_port()
handler = DatagramLogstashHandler("127.0.0.1", port)
try:
server.bind(("127.0.0.1", port))
server.settimeout(2)
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
yield server
finally:
logger.removeHandler(handler)
server.close()
def setUpClass(cls):
ret_port = ports.get_unused_localhost_port()
publish_port = ports.get_unused_localhost_port()
tcp_master_pub_port = ports.get_unused_localhost_port()
tcp_master_pull_port = ports.get_unused_localhost_port()
tcp_master_publish_pull = ports.get_unused_localhost_port()
tcp_master_workers = ports.get_unused_localhost_port()
cls.master_config = cls.get_temp_config(
"master",
**{
"transport": "tcp",
"auto_accept": True,
"ret_port": ret_port,
"publish_port": publish_port,
"tcp_master_pub_port": tcp_master_pub_port,
"tcp_master_pull_port": tcp_master_pull_port,
"tcp_master_publish_pull": tcp_master_publish_pull,
"tcp_master_workers": tcp_master_workers,
}
)
cls.minion_config = cls.get_temp_config(
"minion",
**{
"transport": "tcp",
"master_ip": "127.0.0.1",
"auth_timeout": 1,
"master_port": ret_port,
"master_uri": "tcp://127.0.0.1:{}".format(ret_port),
}
)
cls.process_manager = salt.utils.process.ProcessManager(
name="ReqServer_ProcessManager"
)
cls.server_channel = salt.channel.server.PubServerChannel.factory(
cls.master_config
)
cls.server_channel.pre_fork(cls.process_manager)
# we also require req server for auth
cls.req_server_channel = salt.channel.server.ReqServerChannel.factory(
cls.master_config
)
cls.req_server_channel.pre_fork(cls.process_manager)
cls.io_loop = salt.ext.tornado.ioloop.IOLoop()
cls.stop = threading.Event()
cls.req_server_channel.post_fork(cls._handle_payload, io_loop=cls.io_loop)
cls.server_thread = threading.Thread(
target=run_loop_in_thread,
args=(
cls.io_loop,
cls.stop,
),
)
cls.server_thread.start()
def salt_message_client():
io_loop_mock = MagicMock(spec=salt.ext.tornado.ioloop.IOLoop)
io_loop_mock.call_later.side_effect = lambda *args, **kwargs: (args, kwargs
)
client = salt.transport.tcp.MessageClient(
{},
"127.0.0.1",
ports.get_unused_localhost_port(),
io_loop=io_loop_mock)
try:
yield client
finally:
client.close()
def setUpClass(cls):
overrides = {
"publish_port": ports.get_unused_localhost_port(),
"ret_port": ports.get_unused_localhost_port(),
"tcp_master_pub_port": ports.get_unused_localhost_port(),
"tcp_master_pull_port": ports.get_unused_localhost_port(),
"tcp_master_publish_pull": ports.get_unused_localhost_port(),
"tcp_master_workers": ports.get_unused_localhost_port(),
"runtests_conn_check_port": ports.get_unused_localhost_port(),
"runtests_log_port": ports.get_unused_localhost_port(),
}
overrides["pytest_engine_port"] = overrides["runtests_conn_check_port"]
temp_config = AdaptedConfigurationTestCaseMixin.get_temp_config(
"master", **overrides)
cls.root_dir = temp_config["root_dir"]
cls.config_dir = os.path.dirname(temp_config["conf_file"])
if temp_config["transport"] == "tcp":
raise SkipTest("Test only applicable to the ZMQ transport")
def zmq_server():
logger = logging.getLogger("test_logstash_logger")
context = zmq.Context()
server = context.socket(zmq.SUB)
port = ports.get_unused_localhost_port()
handler = ZMQLogstashHander("tcp://127.0.0.1:{}".format(port))
try:
server.setsockopt(zmq.SUBSCRIBE, b"")
server.bind("tcp://127.0.0.1:{}".format(port))
logger.setLevel(logging.DEBUG)
logger.addHandler(handler)
yield server
finally:
logger.removeHandler(handler)
server.close()
context.term()
async def test_ipc_connect_sync_wrapped(io_loop, tmp_path):
"""
Ensure IPCMessageSubscriber.connect gets wrapped by
salt.utils.asynchronous.SyncWrapper.
"""
if salt.utils.platform.is_windows():
socket_path = ports.get_unused_localhost_port()
else:
socket_path = str(tmp_path / "noexist.ipc")
subscriber = salt.utils.asynchronous.SyncWrapper(
salt.transport.ipc.IPCMessageSubscriber,
args=(socket_path, ),
kwargs={"io_loop": io_loop},
loop_kwarg="io_loop",
)
with pytest.raises(salt.ext.tornado.iostream.StreamClosedError):
# Don't `await subscriber.connect()`, that's the purpose of the SyncWrapper
subscriber.connect()
def apply_pre_start_states(self, salt_call_cli, testclass, root_dir):
if self.listen_port in self.check_ports:
self.check_ports.remove(self.listen_port)
if self.listen_port in self.listen_ports:
self.listen_ports.remove(self.listen_port)
self.listen_port = ports.get_unused_localhost_port()
self.check_ports.append(self.listen_port)
self.listen_ports.append(self.listen_port)
config_dir = os.path.join(root_dir, "config")
git_dir = os.path.join(root_dir, "git")
url = "http://127.0.0.1:{port}/repo.git".format(port=self.listen_port)
url_extra_repo = "http://127.0.0.1:{port}/extra_repo.git".format(
port=self.listen_port)
ext_opts = {"url": url, "url_extra_repo": url_extra_repo}
# Add auth params if present (if so this will trigger the spawned
# server to turn on HTTP basic auth).
for credential_param in ("user", "password"):
if hasattr(testclass, credential_param):
ext_opts[credential_param] = getattr(testclass,
credential_param)
testclass.ext_opts = ext_opts
testclass.nginx_port = self.listen_port
auth_enabled = hasattr(testclass, "username") and hasattr(
testclass, "password")
pillar = {
"git_pillar": {
"config_dir": config_dir,
"git_dir": git_dir,
"uwsgi_port": self.uwsgi_port,
"nginx_port": self.listen_port,
"auth_enabled": auth_enabled,
}
}
ret = salt_call_cli.run("state.apply",
mods="git_pillar.http.nginx",
pillar=pillar)
if ret.returncode != 0:
pytest.fail("Failed to apply the 'git_pillar.http.nginx' state")
if next(iter(ret.data.values()))["result"] is not True:
pytest.fail("Failed to apply the 'git_pillar.http.nginx' state")
def vault_port():
return ports.get_unused_localhost_port()
def pypi_server_port():
return ports.get_unused_localhost_port()
def sdb_etcd_port():
return ports.get_unused_localhost_port()
def echo_server_port():
return ports.get_unused_localhost_port()
def _default_log_port(self):
return ports.get_unused_localhost_port()
def netapi_port():
return get_unused_localhost_port()
def test_cache_remote_file(self):
"""
cp.cache_file
"""
nginx_port = ports.get_unused_localhost_port()
url_prefix = "http://localhost:{}/".format(nginx_port)
temp_dir = tempfile.mkdtemp(dir=RUNTIME_VARS.TMP)
self.addCleanup(shutil.rmtree, temp_dir, ignore_errors=True)
nginx_root_dir = os.path.join(temp_dir, "root")
nginx_conf_dir = os.path.join(temp_dir, "conf")
nginx_conf = os.path.join(nginx_conf_dir, "nginx.conf")
nginx_pidfile = os.path.join(nginx_conf_dir, "nginx.pid")
file_contents = "Hello world!"
for dirname in (nginx_root_dir, nginx_conf_dir):
os.makedirs(dirname)
# Write the temp file
with salt.utils.files.fopen(
os.path.join(nginx_root_dir, "actual_file"), "w") as fp_:
fp_.write(salt.utils.stringutils.to_str(file_contents))
# Write the nginx config
with salt.utils.files.fopen(nginx_conf, "w") as fp_:
fp_.write(
textwrap.dedent(
salt.utils.stringutils.to_str("""\
user root;
worker_processes 1;
error_log {nginx_conf_dir}/server_error.log;
pid {nginx_pidfile};
events {{
worker_connections 1024;
}}
http {{
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log {nginx_conf_dir}/access.log;
error_log {nginx_conf_dir}/error.log;
server {{
listen {nginx_port} default_server;
server_name cachefile.local;
root {nginx_root_dir};
location ~ ^/301$ {{
return 301 /actual_file;
}}
location ~ ^/302$ {{
return 302 /actual_file;
}}
}}
}}""".format(**locals()))))
self.run_function("cmd.run", [["nginx", "-c", nginx_conf]],
python_shell=False)
with salt.utils.files.fopen(nginx_pidfile) as fp_:
nginx_pid = int(fp_.read().strip())
nginx_proc = psutil.Process(pid=nginx_pid)
self.addCleanup(nginx_proc.send_signal, signal.SIGQUIT)
for code in ("", "301", "302"):
url = url_prefix + (code or "actual_file")
log.debug("attempting to cache %s", url)
ret = self.run_function("cp.cache_file", [url])
self.assertTrue(ret)
with salt.utils.files.fopen(ret) as fp_:
cached_contents = salt.utils.stringutils.to_unicode(fp_.read())
self.assertEqual(cached_contents, file_contents)
def master(salt_factories):
defaults = {"rest_tornado": {"port": ports.get_unused_localhost_port(), "disable_ssl": True}}
factory = salt_factories.salt_master_daemon(random_string("master-"), defaults=defaults)
with factory.started():
yield factory
