def get_kubectl_proxy(node: cluster.BaseNode) -> [str, int]:
LOGGER.debug("Stop any other process listening on kubectl proxy port")
node.remoter.run(f"sudo fuser -v4k {KUBECTL_PROXY_PORT}/tcp",
ignore_status=True)
LOGGER.debug("Start kubectl proxy in detached mode")
node.remoter.run(
"setsid kubectl proxy --disable-filter --accept-hosts '.*' > proxy.log 2>&1 < /dev/null & sleep 1"
)
LOGGER.debug("Start auto_ssh for kubectl proxy")
ContainerManager.run_container(node,
KUBECTL_PROXY_CONTAINER,
local_port=KUBECTL_PROXY_PORT,
remote_port=get_free_port(),
ssh_mode="-L")
host = "127.0.0.1"
port = int(
ContainerManager.get_environ(
node, KUBECTL_PROXY_CONTAINER)["SSH_TUNNEL_REMOTE"])
LOGGER.debug("Waiting for port %s:%s is accepting connections", host,
port)
wait_for_port(host, port)
return host, port
def test_ssh_copy_id(self):
with self.subTest("Copy SSH pub key to non-existent container"):
self.assertRaises(NotFound,
ContainerManager.ssh_copy_id, self.node, "c2", user="******", key_file="/root/.ssh/id_rsa")
with self.subTest("Copy SSH pub key"):
self.container.exec_run = Mock()
self.container.exec_run.return_value.exit_code = 0
with patch("paramiko.rsakey.RSAKey.from_private_key_file") as rsa_key_mock:
rsa_key_mock.return_value.get_base64.return_value = "0123456789"
ContainerManager.ssh_copy_id(self.node, "c1", user=sentinel.ssh_user, key_file="~/lala.key")
rsa_key_mock.assert_called_once_with(os.path.expanduser("~/lala.key"))
rsa_key_mock.return_value.get_base64.assert_called_once_with()
self.container.exec_run.assert_called_once()
# pylint: disable=unsubscriptable-object; disable this message for .call_args[]
self.assertIn(" 0123456789 ", " ".join(self.container.exec_run.call_args[0][0]))
self.assertEqual(self.container.exec_run.call_args[1]["user"], sentinel.ssh_user)
with self.subTest("Check exec_run failure"):
self.container.exec_run.reset_mock()
self.container.exec_run.return_value.exit_code = 126
self.container.exec_run.return_value.output = b"blah"
with patch("paramiko.rsakey.RSAKey.from_private_key_file") as rsa_key_mock:
self.assertRaisesRegex(DockerException, "blah",
ContainerManager.ssh_copy_id,
self.node, "c1", user=sentinel.ssh_user, key_file="lala.key")
def configure_rsyslog(cls, node, enable_ngrok=False):
ContainerManager.run_container(node, "rsyslog", logdir=cls.logdir())
port = node.rsyslog_port
LOGGER.info("rsyslog listen on port %s (config: %s)", port,
node.rsyslog_confpath)
if enable_ngrok:
requests.delete('http://localhost:4040/api/tunnels/rsyslogd')
tunnel = {
"addr": port,
"proto": "tcp",
"name": "rsyslogd",
"bind_tls": False
}
res = requests.post('http://localhost:4040/api/tunnels',
json=tunnel)
assert res.ok, "failed to add a ngrok tunnel [{}, {}]".format(
res, res.text)
ngrok_address = res.json()['public_url'].replace('tcp://', '')
address, port = ngrok_address.split(':')
else:
address = get_my_ip()
cls.RSYSLOG_ADDRESS = (address, port)
def browser(self):
ContainerManager.run_container(self.node, "web_driver")
LOGGER.debug("Waiting for WebDriver container is up")
ContainerManager.wait_for_status(self.node, "web_driver", "running")
port = ContainerManager.get_container_port(self.node, "web_driver", WEB_DRIVER_REMOTE_PORT)
LOGGER.debug("WebDriver port is %s", port)
if self.use_tunnel:
LOGGER.debug("Start auto_ssh for Selenium remote WebDriver")
ContainerManager.run_container(self.node, "auto_ssh:web_driver",
local_port=port,
remote_port=get_free_port(),
ssh_mode="-L")
LOGGER.debug("Waiting for SSH tunnel container is up")
ContainerManager.wait_for_status(self.node, "auto_ssh:web_driver", status="running")
host = "127.0.0.1"
port = int(ContainerManager.get_environ(self.node, "auto_ssh:web_driver")["SSH_TUNNEL_REMOTE"])
else:
host = self.node.external_address
LOGGER.debug("Waiting for port %s:%s is accepting connections", host, port)
wait_for_port(host, port)
time.sleep(WEB_DRIVER_CONTAINER_START_DELAY)
return Remote(command_executor=f"http://{host}:{port}/wd/hub", options=ChromeOptions())
def ldap_ports(self) -> Optional[dict]:
return {
'ldap_port':
ContainerManager.get_container_port(self, "ldap", LDAP_PORT),
'ldap_ssl_port':
ContainerManager.get_container_port(self, "ldap", LDAP_SSL_PORT)
}
def destroy(self) -> None:
ContainerManager.destroy_all_containers(self)
try:
os.remove(self.rsyslog_confpath)
except Exception as exc: # pylint: disable=broad-except
LOGGER.warning("Unable to delete `%s': %s", self.rsyslog_confpath,
exc)
def run(self, force: bool = False) -> str:
if not force and AWS_MOCK_IP_FILE.exists():
LOGGER.warning(
"%s found, don't run a new container and return AWS Mock IP from it",
AWS_MOCK_IP_FILE)
return AWS_MOCK_IP_FILE.read_text(encoding="utf-8")
container = ContainerManager.run_container(self, "aws_mock")
res = container.exec_run([
"bash", "-cxe",
dedent("""\
mkdir -p /src/s3/scylla-qa-keystore
ssh-keygen -q -b 2048 -t rsa -N "" -C aws_mock -f /src/s3/scylla-qa-keystore/scylla-qa-ec2
chown -R nginx:nginx /src/s3/scylla-qa-keystore
useradd ubuntu
mkdir -m 700 -p /home/ubuntu/.ssh
cp /src/s3/scylla-qa-keystore/scylla-qa-ec2.pub /home/ubuntu/.ssh/authorized_keys
chown -R ubuntu:ubuntu /home/ubuntu/.ssh
""")
])
if res.exit_code:
raise DockerException(f"{container}: {res.output.decode('utf-8')}")
aws_mock_ip = ContainerManager.get_ip_address(self, "aws_mock")
AWS_MOCK_IP_FILE.write_text(aws_mock_ip, encoding="utf-8")
return aws_mock_ip
def test_unregister_container(self):
with self.subTest("Unregister non-existent name"):
self.assertRaises(NotFound, ContainerManager.unregister_container, self.node, "c2")
with self.subTest("Unregister container"):
ContainerManager.unregister_container(self.node, "c1")
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
def destroy_container(self, name):
try:
if ContainerManager.get_container(self.node, name, raise_not_found_exc=False) is not None:
LOGGER.debug("Destroy %s (%s) container", name, self)
ContainerManager.destroy_container(self.node, name, ignore_keepalive=True)
LOGGER.info("%s (%s) destroyed", name, self)
except Exception as exc: # pylint: disable=broad-except
LOGGER.error("%s: some exception raised during container '%s' destroying", self, name, exc_info=exc)
def configure_syslogng(cls, node):
ContainerManager.run_container(node, "syslogng", logdir=cls.logdir())
cls._link_running_syslog_logdir(node.syslogng_log_dir)
port = node.syslogng_port
LOGGER.info("syslog-ng listen on port %s (config: %s)", port,
node.syslogng_confpath)
address = get_my_ip()
cls.RSYSLOG_ADDRESS = (address, port)
def _span_container(self):
try:
self._container = self._instance._get_gcloud_container() # pylint: disable=protected-access
except Exception as exc:
try:
ContainerManager.destroy_container(self._instance, self._name)
except Exception: # pylint: disable=broad-except
pass
raise exc from None
def test_get_container(self):
with self.subTest("Get existent container"):
self.assertIs(ContainerManager.get_container(self.node, "c1"), self.container)
with self.subTest("Try to get non-existent container without exception"):
self.assertIs(ContainerManager.get_container(self.node, "c2", raise_not_found_exc=False), None)
with self.subTest("Check NotFound exception"):
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c2")
def _destroy_container(self):
self._span_counter -= 1
if self._span_counter != 0:
return
try:
ContainerManager.destroy_container(self._instance, self._name)
except Exception: # pylint: disable=broad-except
pass
self._container = None
def configure_ldap(cls, node, use_ssl=False):
ContainerManager.run_container(node, "ldap")
if use_ssl:
port = node.ldap_ports['ldap_ssl_port']
else:
port = node.ldap_ports['ldap_port']
address = get_my_ip()
cls.LDAP_ADDRESS = (address, port)
if ContainerManager.get_container(node, 'ldap').exec_run("timeout 30s container/tool/wait-process")[0] != 0:
raise LdapServerNotReady("LDAP server didn't finish its startup yet...")
def test_is_running(self):
with self.subTest("Check exited container"):
self.assertFalse(ContainerManager.is_running(self.node, "c1"))
with self.subTest("Check running container"):
self.container._status = "running"
self.assertTrue(ContainerManager.is_running(self.node, "c1"))
with self.subTest("Try to get status of non-existent container"):
self.assertRaises(NotFound, ContainerManager.is_running, self.node, "c2")
def test_destroy_keep_alive(self):
ContainerManager.set_container_keep_alive(self.node, "c1")
with self.subTest("Try to destroy container with keep-alive tag"):
self.assertFalse(ContainerManager.destroy_container(self.node, "c1"))
self.assertEqual(ContainerManager.get_container(self.node, "c1"), self.container)
with self.subTest("Ignore keep-alive tag"):
self.assertTrue(ContainerManager.destroy_container(self.node, "c1", ignore_keepalive=True))
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
def test_set_container_keep_alive(self):
with self.subTest("Set keep alive to non-existent container"):
self.assertRaises(NotFound, ContainerManager.set_container_keep_alive, self.node, "c2")
with self.subTest("Set keep alive"):
ContainerManager.set_container_keep_alive(self.node, "c1")
self.assertEqual(self.container.name, "dummy---KEEPALIVE")
with self.subTest("Set keep alive again"):
ContainerManager.set_container_keep_alive(self.node, "c1")
self.assertEqual(self.container.name, "dummy---KEEPALIVE")
def test_get_container_port(self):
self.container.ports = {"9999/tcp": [{"HostIp": "0.0.0.0", "HostPort": 1111}, ]}
with self.subTest("Try to get port for non-existent container"):
self.assertRaises(NotFound, ContainerManager.get_container_port, self.node, "c2", 9999)
with self.subTest("No port"):
self.assertIs(ContainerManager.get_container_port(self.node, "c1", 8888), None)
with self.subTest("Open port"):
self.assertEqual(ContainerManager.get_container_port(self.node, "c1", 9999), 1111)
self.assertEqual(ContainerManager.get_container_port(self.node, "c1", "9999"), 1111)
def destroy_container(self, name):
try:
if ContainerManager.get_container(
self.node, name, raise_not_found_exc=False) is not None:
LOGGER.debug(f"Destroy {name} ({self}) container")
ContainerManager.destroy_container(self.node,
name,
ignore_keepalive=True)
LOGGER.info(f"{name} ({self}) destroyed")
except Exception as exc: # pylint: disable=broad-except
LOGGER.error(
f"{self}: some exception raised during container `{name}' destroying",
exc_info=exc)
def _create_node(self, node_index, container=None):
node = DockerNode(parent_cluster=self,
container=container,
ssh_login_info=dict(
hostname=None,
user=self.node_container_user,
key_file=self.node_container_key_file),
base_logdir=self.logdir,
node_prefix=self.node_prefix,
node_index=node_index)
if container is None:
ContainerManager.build_container_image(node, "node")
ContainerManager.run_container(
node,
"node",
seed_ip=self.nodes[0].public_ip_address if node_index else None)
ContainerManager.wait_for_status(node, "node", status="running")
ContainerManager.ssh_copy_id(node, "node", self.node_container_user,
self.node_container_key_file)
node.init()
return node
def test_destroy_all(self):
ContainerManager.run_container(self.node, "c2")
ContainerManager.set_container_keep_alive(self.node, "c1")
with self.subTest("Destroy all containers without keep-alive tag"):
ContainerManager.destroy_all_containers(self.node)
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c2")
self.assertEqual(ContainerManager.get_container(self.node, "c1"), self.container)
with self.subTest("Ignore keep-alive tag"):
ContainerManager.destroy_all_containers(self.node, ignore_keepalive=True)
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
def _get_nodes(self):
containers = ContainerManager.get_containers_by_prefix(self.node_prefix)
for node_index, container in sorted((int(c.labels["NodeIndex"]), c) for c in containers):
LOGGER.debug("Found container %s with name `%s' and index=%d", container, container.name, node_index)
node = self._create_node(node_index, container)
self.nodes.append(node)
return self.nodes
def _get_gcloud_container(self) -> Container:
"""Create Google Cloud SDK container.
Cloud SDK requires to enable some authorization method first. Because of that we start a container which
runs forever using `cat' command (like Jenkins do), put a service account credentials and activate them.
All consequent gcloud commands run using container.exec_run() method.
"""
container = ContainerManager.run_container(self, "gcloud")
credentials = KeyStore().get_gcp_credentials()
credentials[
"client_email"] = f"{credentials['project_id']}@appspot.gserviceaccount.com"
shell_command = f"umask 077 && echo '{json.dumps(credentials)}' > /tmp/gcloud_svc_account.json"
res = container.exec_run(["sh", "-c", shell_command])
if res.exit_code:
raise DockerException(f"{container}: {res.output.decode('utf-8')}")
res = container.exec_run([
"gcloud", "auth", "activate-service-account",
credentials["client_email"], "--key-file",
"/tmp/gcloud_svc_account.json", "--project",
credentials["project_id"]
])
if res.exit_code:
raise DockerException(
f"{container}[]: {res.output.decode('utf-8')}")
return container
def _get_gcloud_container(self) -> Container:
"""Create Google Cloud SDK container.
Cloud SDK requires to enable some authorization method first. Because of that we start a container which
runs forever using `cat' command (like Jenkins do), put a service account credentials and activate them.
All consequent gcloud commands run using container.exec_run() method.
"""
container = ContainerManager.run_container(self, "gcloud")
credentials = KeyStore().get_gcp_credentials()
credentials["client_email"] = f"{credentials['client_email']}"
shell_command = f"umask 077 && echo '{json.dumps(credentials)}' > /tmp/gcloud_svc_account.json"
shell_command += " && echo 'kubeletConfig:\n cpuManagerPolicy: static' > /tmp/system_config.yaml"
# NOTE: use 'bash' in case of non-alpine sdk image and 'sh' when it is 'alpine' one.
res = container.exec_run(["bash", "-c", shell_command])
if res.exit_code:
raise DockerException(f"{container}: {res.output.decode('utf-8')}")
res = container.exec_run([
"gcloud", "auth", "activate-service-account",
credentials["client_email"], "--key-file",
"/tmp/gcloud_svc_account.json", "--project",
credentials["project_id"]
])
if res.exit_code:
raise DockerException(
f"{container}[]: {res.output.decode('utf-8')}")
return container
def test_build_container_image(self):
with self.subTest("Try to build image for existent container"):
self.assertRaises(ContainerAlreadyRegistered, ContainerManager.build_container_image, self.node, "c1")
with self.subTest("Node-wide container image tag"):
self.node.container_image_tag = Mock(return_value="blah")
self.assertRaisesRegex(AssertionError, "image tag", ContainerManager.build_container_image, self.node, "c2")
with self.subTest("Node-wide dockerfile args"):
self.node.c2_container_image_tag = Mock(return_value="hello-world:latest")
self.node.container_image_dockerfile_args = Mock(return_value="blah")
self.assertRaisesRegex(AssertionError, "Dockerfile",
ContainerManager.build_container_image, self.node, "c2")
self.node.c2_container_image_tag.assert_called_once_with()
with self.subTest("No build args"):
self.node.c2_container_image_tag.reset_mock()
self.node.c2_container_image_dockerfile_args = Mock(return_value={"path": "."})
with patch("sdcm.utils.docker_utils.LOGGER.debug") as logger:
image = ContainerManager.build_container_image(self.node, "c2:another", arg1="value1", arg2="value2")
self.assertEqual(image, ((), dict(tag="hello-world:latest",
path=".",
labels=self.node.tags,
pull=True,
rm=True,
arg1="value1",
arg2="value2")))
logger.assert_has_calls([call("blah"), ])
self.node.c2_container_image_tag.assert_called_once_with("another")
self.node.c2_container_image_dockerfile_args.assert_called_once_with("another")
with self.subTest("No build args"):
self.node.c2_container_image_tag.reset_mock()
self.node.c2_container_image_dockerfile_args.reset_mock()
self.node.c2_container_image_build_args = lambda **kw: {v: k for k, v in kw.items()}
with patch("sdcm.utils.docker_utils.LOGGER.debug") as logger:
image = ContainerManager.build_container_image(self.node, "c2:another", arg1="value1", arg2="value2")
self.assertEqual(image, ((), dict(tag="hello-world:latest",
path=".",
labels=self.node.tags,
pull=True,
rm=True,
value1="arg1",
value2="arg2")))
logger.assert_has_calls([call("blah"), ])
self.node.c2_container_image_tag.assert_called_once_with("another")
self.node.c2_container_image_dockerfile_args.assert_called_once_with("another")
def syslogng_log_dir(self) -> Optional[str]:
# This should work after process that had run container was terminated and another one starts
# that is why it is not using variable to store log directory instead
return ContainerManager.get_host_volume_path(
instance=self,
container_name="syslogng",
path_in_container="/var/log",
)
def test_get_environ(self):
self.container.get_attrs = lambda: {"Config": {"Env": ["A", "B=1", ]}}
with self.subTest("Try to get env for non-existent container"):
self.assertRaises(NotFound, ContainerManager.get_environ, self.node, "c2")
with self.subTest("Get env for existent container"):
self.assertEqual(ContainerManager.get_environ(self.node, "c1"), {"A": None, "B": "1"})
def test_destroy(self):
with self.subTest("Try to destroy non-existent container"):
self.assertRaises(NotFound, ContainerManager.destroy_container, self.node, "c2")
with self.subTest("without *_container_logfile hook"):
self.assertTrue(ContainerManager.destroy_container(self.node, "c1"))
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
self.assertRaises(NotFound, ContainerManager.destroy_container, self.node, "c1")
def test_destroy_logfile(self):
self.node.c1_container_logfile = "container.log"
with patch("builtins.open", mock_open()) as mock_file:
self.assertTrue(ContainerManager.destroy_container(self.node, "c1"))
mock_file.assert_called_once_with("container.log", "ab")
mock_file().write.assert_called_once_with("container logs")
self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
self.assertRaises(NotFound, ContainerManager.destroy_container, self.node, "c1")
def test_register_container(self):
c2_container = DummyContainer()
with self.subTest("Try to register registered container"):
self.assertRaisesRegex(ContainerAlreadyRegistered, "container .* registered already",
ContainerManager.register_container,
self.node, "c2", container=self.container)
with self.subTest("Try to register registered container with replace=True"):
self.assertRaisesRegex(ContainerAlreadyRegistered, "container .* registered already",
ContainerManager.register_container,
self.node, "c2", container=self.container, replace=True)
with self.subTest("Try to replace container"):
self.assertRaisesRegex(ContainerAlreadyRegistered, "another container registered",
ContainerManager.register_container,
self.node, "c1", container=c2_container)
with self.subTest("Force container replacement"):
ContainerManager.register_container(self.node, "c1", container=c2_container, replace=True)
self.assertEqual(ContainerManager.get_container(self.node, "c1"), c2_container)
with self.subTest("Register container"):
ContainerManager.register_container(self.node, "c2", container=self.container)
self.assertEqual(ContainerManager.get_container(self.node, "c2"), self.container)
with self.subTest("Try to force container replacement with container registered with another name"):
self.assertRaisesRegex(ContainerAlreadyRegistered, "container .* registered already",
ContainerManager.register_container,
self.node, "c1", container=self.container, replace=True)