def get_kubectl_proxy(node: cluster.BaseNode) -> [str, int]: LOGGER.debug("Stop any other process listening on kubectl proxy port") node.remoter.run(f"sudo fuser -v4k {KUBECTL_PROXY_PORT}/tcp", ignore_status=True) LOGGER.debug("Start kubectl proxy in detached mode") node.remoter.run( "setsid kubectl proxy --disable-filter --accept-hosts '.*' > proxy.log 2>&1 < /dev/null & sleep 1" ) LOGGER.debug("Start auto_ssh for kubectl proxy") ContainerManager.run_container(node, KUBECTL_PROXY_CONTAINER, local_port=KUBECTL_PROXY_PORT, remote_port=get_free_port(), ssh_mode="-L") host = "127.0.0.1" port = int( ContainerManager.get_environ( node, KUBECTL_PROXY_CONTAINER)["SSH_TUNNEL_REMOTE"]) LOGGER.debug("Waiting for port %s:%s is accepting connections", host, port) wait_for_port(host, port) return host, port
def test_ssh_copy_id(self): with self.subTest("Copy SSH pub key to non-existent container"): self.assertRaises(NotFound, ContainerManager.ssh_copy_id, self.node, "c2", user="******", key_file="/root/.ssh/id_rsa") with self.subTest("Copy SSH pub key"): self.container.exec_run = Mock() self.container.exec_run.return_value.exit_code = 0 with patch("paramiko.rsakey.RSAKey.from_private_key_file") as rsa_key_mock: rsa_key_mock.return_value.get_base64.return_value = "0123456789" ContainerManager.ssh_copy_id(self.node, "c1", user=sentinel.ssh_user, key_file="~/lala.key") rsa_key_mock.assert_called_once_with(os.path.expanduser("~/lala.key")) rsa_key_mock.return_value.get_base64.assert_called_once_with() self.container.exec_run.assert_called_once() # pylint: disable=unsubscriptable-object; disable this message for .call_args[] self.assertIn(" 0123456789 ", " ".join(self.container.exec_run.call_args[0][0])) self.assertEqual(self.container.exec_run.call_args[1]["user"], sentinel.ssh_user) with self.subTest("Check exec_run failure"): self.container.exec_run.reset_mock() self.container.exec_run.return_value.exit_code = 126 self.container.exec_run.return_value.output = b"blah" with patch("paramiko.rsakey.RSAKey.from_private_key_file") as rsa_key_mock: self.assertRaisesRegex(DockerException, "blah", ContainerManager.ssh_copy_id, self.node, "c1", user=sentinel.ssh_user, key_file="lala.key")
def configure_rsyslog(cls, node, enable_ngrok=False): ContainerManager.run_container(node, "rsyslog", logdir=cls.logdir()) port = node.rsyslog_port LOGGER.info("rsyslog listen on port %s (config: %s)", port, node.rsyslog_confpath) if enable_ngrok: requests.delete('http://localhost:4040/api/tunnels/rsyslogd') tunnel = { "addr": port, "proto": "tcp", "name": "rsyslogd", "bind_tls": False } res = requests.post('http://localhost:4040/api/tunnels', json=tunnel) assert res.ok, "failed to add a ngrok tunnel [{}, {}]".format( res, res.text) ngrok_address = res.json()['public_url'].replace('tcp://', '') address, port = ngrok_address.split(':') else: address = get_my_ip() cls.RSYSLOG_ADDRESS = (address, port)
def browser(self): ContainerManager.run_container(self.node, "web_driver") LOGGER.debug("Waiting for WebDriver container is up") ContainerManager.wait_for_status(self.node, "web_driver", "running") port = ContainerManager.get_container_port(self.node, "web_driver", WEB_DRIVER_REMOTE_PORT) LOGGER.debug("WebDriver port is %s", port) if self.use_tunnel: LOGGER.debug("Start auto_ssh for Selenium remote WebDriver") ContainerManager.run_container(self.node, "auto_ssh:web_driver", local_port=port, remote_port=get_free_port(), ssh_mode="-L") LOGGER.debug("Waiting for SSH tunnel container is up") ContainerManager.wait_for_status(self.node, "auto_ssh:web_driver", status="running") host = "127.0.0.1" port = int(ContainerManager.get_environ(self.node, "auto_ssh:web_driver")["SSH_TUNNEL_REMOTE"]) else: host = self.node.external_address LOGGER.debug("Waiting for port %s:%s is accepting connections", host, port) wait_for_port(host, port) time.sleep(WEB_DRIVER_CONTAINER_START_DELAY) return Remote(command_executor=f"http://{host}:{port}/wd/hub", options=ChromeOptions())
def ldap_ports(self) -> Optional[dict]: return { 'ldap_port': ContainerManager.get_container_port(self, "ldap", LDAP_PORT), 'ldap_ssl_port': ContainerManager.get_container_port(self, "ldap", LDAP_SSL_PORT) }
def destroy(self) -> None: ContainerManager.destroy_all_containers(self) try: os.remove(self.rsyslog_confpath) except Exception as exc: # pylint: disable=broad-except LOGGER.warning("Unable to delete `%s': %s", self.rsyslog_confpath, exc)
def run(self, force: bool = False) -> str: if not force and AWS_MOCK_IP_FILE.exists(): LOGGER.warning( "%s found, don't run a new container and return AWS Mock IP from it", AWS_MOCK_IP_FILE) return AWS_MOCK_IP_FILE.read_text(encoding="utf-8") container = ContainerManager.run_container(self, "aws_mock") res = container.exec_run([ "bash", "-cxe", dedent("""\ mkdir -p /src/s3/scylla-qa-keystore ssh-keygen -q -b 2048 -t rsa -N "" -C aws_mock -f /src/s3/scylla-qa-keystore/scylla-qa-ec2 chown -R nginx:nginx /src/s3/scylla-qa-keystore useradd ubuntu mkdir -m 700 -p /home/ubuntu/.ssh cp /src/s3/scylla-qa-keystore/scylla-qa-ec2.pub /home/ubuntu/.ssh/authorized_keys chown -R ubuntu:ubuntu /home/ubuntu/.ssh """) ]) if res.exit_code: raise DockerException(f"{container}: {res.output.decode('utf-8')}") aws_mock_ip = ContainerManager.get_ip_address(self, "aws_mock") AWS_MOCK_IP_FILE.write_text(aws_mock_ip, encoding="utf-8") return aws_mock_ip
def test_unregister_container(self): with self.subTest("Unregister non-existent name"): self.assertRaises(NotFound, ContainerManager.unregister_container, self.node, "c2") with self.subTest("Unregister container"): ContainerManager.unregister_container(self.node, "c1") self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
def destroy_container(self, name): try: if ContainerManager.get_container(self.node, name, raise_not_found_exc=False) is not None: LOGGER.debug("Destroy %s (%s) container", name, self) ContainerManager.destroy_container(self.node, name, ignore_keepalive=True) LOGGER.info("%s (%s) destroyed", name, self) except Exception as exc: # pylint: disable=broad-except LOGGER.error("%s: some exception raised during container '%s' destroying", self, name, exc_info=exc)
def configure_syslogng(cls, node): ContainerManager.run_container(node, "syslogng", logdir=cls.logdir()) cls._link_running_syslog_logdir(node.syslogng_log_dir) port = node.syslogng_port LOGGER.info("syslog-ng listen on port %s (config: %s)", port, node.syslogng_confpath) address = get_my_ip() cls.RSYSLOG_ADDRESS = (address, port)
def _span_container(self): try: self._container = self._instance._get_gcloud_container() # pylint: disable=protected-access except Exception as exc: try: ContainerManager.destroy_container(self._instance, self._name) except Exception: # pylint: disable=broad-except pass raise exc from None
def test_get_container(self): with self.subTest("Get existent container"): self.assertIs(ContainerManager.get_container(self.node, "c1"), self.container) with self.subTest("Try to get non-existent container without exception"): self.assertIs(ContainerManager.get_container(self.node, "c2", raise_not_found_exc=False), None) with self.subTest("Check NotFound exception"): self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c2")
def _destroy_container(self): self._span_counter -= 1 if self._span_counter != 0: return try: ContainerManager.destroy_container(self._instance, self._name) except Exception: # pylint: disable=broad-except pass self._container = None
def configure_ldap(cls, node, use_ssl=False): ContainerManager.run_container(node, "ldap") if use_ssl: port = node.ldap_ports['ldap_ssl_port'] else: port = node.ldap_ports['ldap_port'] address = get_my_ip() cls.LDAP_ADDRESS = (address, port) if ContainerManager.get_container(node, 'ldap').exec_run("timeout 30s container/tool/wait-process")[0] != 0: raise LdapServerNotReady("LDAP server didn't finish its startup yet...")
def test_is_running(self): with self.subTest("Check exited container"): self.assertFalse(ContainerManager.is_running(self.node, "c1")) with self.subTest("Check running container"): self.container._status = "running" self.assertTrue(ContainerManager.is_running(self.node, "c1")) with self.subTest("Try to get status of non-existent container"): self.assertRaises(NotFound, ContainerManager.is_running, self.node, "c2")
def test_destroy_keep_alive(self): ContainerManager.set_container_keep_alive(self.node, "c1") with self.subTest("Try to destroy container with keep-alive tag"): self.assertFalse(ContainerManager.destroy_container(self.node, "c1")) self.assertEqual(ContainerManager.get_container(self.node, "c1"), self.container) with self.subTest("Ignore keep-alive tag"): self.assertTrue(ContainerManager.destroy_container(self.node, "c1", ignore_keepalive=True)) self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
def test_set_container_keep_alive(self): with self.subTest("Set keep alive to non-existent container"): self.assertRaises(NotFound, ContainerManager.set_container_keep_alive, self.node, "c2") with self.subTest("Set keep alive"): ContainerManager.set_container_keep_alive(self.node, "c1") self.assertEqual(self.container.name, "dummy---KEEPALIVE") with self.subTest("Set keep alive again"): ContainerManager.set_container_keep_alive(self.node, "c1") self.assertEqual(self.container.name, "dummy---KEEPALIVE")
def test_get_container_port(self): self.container.ports = {"9999/tcp": [{"HostIp": "0.0.0.0", "HostPort": 1111}, ]} with self.subTest("Try to get port for non-existent container"): self.assertRaises(NotFound, ContainerManager.get_container_port, self.node, "c2", 9999) with self.subTest("No port"): self.assertIs(ContainerManager.get_container_port(self.node, "c1", 8888), None) with self.subTest("Open port"): self.assertEqual(ContainerManager.get_container_port(self.node, "c1", 9999), 1111) self.assertEqual(ContainerManager.get_container_port(self.node, "c1", "9999"), 1111)
def destroy_container(self, name): try: if ContainerManager.get_container( self.node, name, raise_not_found_exc=False) is not None: LOGGER.debug(f"Destroy {name} ({self}) container") ContainerManager.destroy_container(self.node, name, ignore_keepalive=True) LOGGER.info(f"{name} ({self}) destroyed") except Exception as exc: # pylint: disable=broad-except LOGGER.error( f"{self}: some exception raised during container `{name}' destroying", exc_info=exc)
def _create_node(self, node_index, container=None): node = DockerNode(parent_cluster=self, container=container, ssh_login_info=dict( hostname=None, user=self.node_container_user, key_file=self.node_container_key_file), base_logdir=self.logdir, node_prefix=self.node_prefix, node_index=node_index) if container is None: ContainerManager.build_container_image(node, "node") ContainerManager.run_container( node, "node", seed_ip=self.nodes[0].public_ip_address if node_index else None) ContainerManager.wait_for_status(node, "node", status="running") ContainerManager.ssh_copy_id(node, "node", self.node_container_user, self.node_container_key_file) node.init() return node
def test_destroy_all(self): ContainerManager.run_container(self.node, "c2") ContainerManager.set_container_keep_alive(self.node, "c1") with self.subTest("Destroy all containers without keep-alive tag"): ContainerManager.destroy_all_containers(self.node) self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c2") self.assertEqual(ContainerManager.get_container(self.node, "c1"), self.container) with self.subTest("Ignore keep-alive tag"): ContainerManager.destroy_all_containers(self.node, ignore_keepalive=True) self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1")
def _get_nodes(self): containers = ContainerManager.get_containers_by_prefix(self.node_prefix) for node_index, container in sorted((int(c.labels["NodeIndex"]), c) for c in containers): LOGGER.debug("Found container %s with name `%s' and index=%d", container, container.name, node_index) node = self._create_node(node_index, container) self.nodes.append(node) return self.nodes
def _get_gcloud_container(self) -> Container: """Create Google Cloud SDK container. Cloud SDK requires to enable some authorization method first. Because of that we start a container which runs forever using `cat' command (like Jenkins do), put a service account credentials and activate them. All consequent gcloud commands run using container.exec_run() method. """ container = ContainerManager.run_container(self, "gcloud") credentials = KeyStore().get_gcp_credentials() credentials[ "client_email"] = f"{credentials['project_id']}@appspot.gserviceaccount.com" shell_command = f"umask 077 && echo '{json.dumps(credentials)}' > /tmp/gcloud_svc_account.json" res = container.exec_run(["sh", "-c", shell_command]) if res.exit_code: raise DockerException(f"{container}: {res.output.decode('utf-8')}") res = container.exec_run([ "gcloud", "auth", "activate-service-account", credentials["client_email"], "--key-file", "/tmp/gcloud_svc_account.json", "--project", credentials["project_id"] ]) if res.exit_code: raise DockerException( f"{container}[]: {res.output.decode('utf-8')}") return container
def _get_gcloud_container(self) -> Container: """Create Google Cloud SDK container. Cloud SDK requires to enable some authorization method first. Because of that we start a container which runs forever using `cat' command (like Jenkins do), put a service account credentials and activate them. All consequent gcloud commands run using container.exec_run() method. """ container = ContainerManager.run_container(self, "gcloud") credentials = KeyStore().get_gcp_credentials() credentials["client_email"] = f"{credentials['client_email']}" shell_command = f"umask 077 && echo '{json.dumps(credentials)}' > /tmp/gcloud_svc_account.json" shell_command += " && echo 'kubeletConfig:\n cpuManagerPolicy: static' > /tmp/system_config.yaml" # NOTE: use 'bash' in case of non-alpine sdk image and 'sh' when it is 'alpine' one. res = container.exec_run(["bash", "-c", shell_command]) if res.exit_code: raise DockerException(f"{container}: {res.output.decode('utf-8')}") res = container.exec_run([ "gcloud", "auth", "activate-service-account", credentials["client_email"], "--key-file", "/tmp/gcloud_svc_account.json", "--project", credentials["project_id"] ]) if res.exit_code: raise DockerException( f"{container}[]: {res.output.decode('utf-8')}") return container
def test_build_container_image(self): with self.subTest("Try to build image for existent container"): self.assertRaises(ContainerAlreadyRegistered, ContainerManager.build_container_image, self.node, "c1") with self.subTest("Node-wide container image tag"): self.node.container_image_tag = Mock(return_value="blah") self.assertRaisesRegex(AssertionError, "image tag", ContainerManager.build_container_image, self.node, "c2") with self.subTest("Node-wide dockerfile args"): self.node.c2_container_image_tag = Mock(return_value="hello-world:latest") self.node.container_image_dockerfile_args = Mock(return_value="blah") self.assertRaisesRegex(AssertionError, "Dockerfile", ContainerManager.build_container_image, self.node, "c2") self.node.c2_container_image_tag.assert_called_once_with() with self.subTest("No build args"): self.node.c2_container_image_tag.reset_mock() self.node.c2_container_image_dockerfile_args = Mock(return_value={"path": "."}) with patch("sdcm.utils.docker_utils.LOGGER.debug") as logger: image = ContainerManager.build_container_image(self.node, "c2:another", arg1="value1", arg2="value2") self.assertEqual(image, ((), dict(tag="hello-world:latest", path=".", labels=self.node.tags, pull=True, rm=True, arg1="value1", arg2="value2"))) logger.assert_has_calls([call("blah"), ]) self.node.c2_container_image_tag.assert_called_once_with("another") self.node.c2_container_image_dockerfile_args.assert_called_once_with("another") with self.subTest("No build args"): self.node.c2_container_image_tag.reset_mock() self.node.c2_container_image_dockerfile_args.reset_mock() self.node.c2_container_image_build_args = lambda **kw: {v: k for k, v in kw.items()} with patch("sdcm.utils.docker_utils.LOGGER.debug") as logger: image = ContainerManager.build_container_image(self.node, "c2:another", arg1="value1", arg2="value2") self.assertEqual(image, ((), dict(tag="hello-world:latest", path=".", labels=self.node.tags, pull=True, rm=True, value1="arg1", value2="arg2"))) logger.assert_has_calls([call("blah"), ]) self.node.c2_container_image_tag.assert_called_once_with("another") self.node.c2_container_image_dockerfile_args.assert_called_once_with("another")
def syslogng_log_dir(self) -> Optional[str]: # This should work after process that had run container was terminated and another one starts # that is why it is not using variable to store log directory instead return ContainerManager.get_host_volume_path( instance=self, container_name="syslogng", path_in_container="/var/log", )
def test_get_environ(self): self.container.get_attrs = lambda: {"Config": {"Env": ["A", "B=1", ]}} with self.subTest("Try to get env for non-existent container"): self.assertRaises(NotFound, ContainerManager.get_environ, self.node, "c2") with self.subTest("Get env for existent container"): self.assertEqual(ContainerManager.get_environ(self.node, "c1"), {"A": None, "B": "1"})
def test_destroy(self): with self.subTest("Try to destroy non-existent container"): self.assertRaises(NotFound, ContainerManager.destroy_container, self.node, "c2") with self.subTest("without *_container_logfile hook"): self.assertTrue(ContainerManager.destroy_container(self.node, "c1")) self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1") self.assertRaises(NotFound, ContainerManager.destroy_container, self.node, "c1")
def test_destroy_logfile(self): self.node.c1_container_logfile = "container.log" with patch("builtins.open", mock_open()) as mock_file: self.assertTrue(ContainerManager.destroy_container(self.node, "c1")) mock_file.assert_called_once_with("container.log", "ab") mock_file().write.assert_called_once_with("container logs") self.assertRaises(NotFound, ContainerManager.get_container, self.node, "c1") self.assertRaises(NotFound, ContainerManager.destroy_container, self.node, "c1")
def test_register_container(self): c2_container = DummyContainer() with self.subTest("Try to register registered container"): self.assertRaisesRegex(ContainerAlreadyRegistered, "container .* registered already", ContainerManager.register_container, self.node, "c2", container=self.container) with self.subTest("Try to register registered container with replace=True"): self.assertRaisesRegex(ContainerAlreadyRegistered, "container .* registered already", ContainerManager.register_container, self.node, "c2", container=self.container, replace=True) with self.subTest("Try to replace container"): self.assertRaisesRegex(ContainerAlreadyRegistered, "another container registered", ContainerManager.register_container, self.node, "c1", container=c2_container) with self.subTest("Force container replacement"): ContainerManager.register_container(self.node, "c1", container=c2_container, replace=True) self.assertEqual(ContainerManager.get_container(self.node, "c1"), c2_container) with self.subTest("Register container"): ContainerManager.register_container(self.node, "c2", container=self.container) self.assertEqual(ContainerManager.get_container(self.node, "c2"), self.container) with self.subTest("Try to force container replacement with container registered with another name"): self.assertRaisesRegex(ContainerAlreadyRegistered, "container .* registered already", ContainerManager.register_container, self.node, "c1", container=self.container, replace=True)