def add_nodes(
self,
count: int,
ec2_user_data: str = "",
dc_idx: int = 0,
enable_auto_bootstrap: bool = False) -> List[BasePodContainer]:
new_nodes = super().add_nodes(
count=count,
ec2_user_data=ec2_user_data,
dc_idx=dc_idx,
enable_auto_bootstrap=enable_auto_bootstrap)
for node in new_nodes:
KubernetesOps.expose_pod_ports(
self.k8s_cluster,
node.name,
ports=SCYLLA_POD_EXPOSED_PORTS,
labels=f"statefulset.kubernetes.io/pod-name={node.name}",
selector=f"statefulset.kubernetes.io/pod-name={node.name}",
namespace=self.namespace)
LOCALRUNNER.run_shell_script(cmd="\n".join(
self.hydra_iptables_redirect_rules(nodes=new_nodes)),
sudo=True)
atexit.register(LOCALRUNNER.run_shell_script,
cmd="\n".join(
self.hydra_iptables_redirect_rules(
command="D", nodes=new_nodes)),
sudo=True)
self.update_nodes_iptables_redirect_rules(nodes=new_nodes)
return new_nodes
def add_keys(cls, paths: Iterable[str], verbose: bool = True) -> None:
if paths:
if not cls.is_running():
cls.start(verbose=verbose)
LOCALRUNNER.run(
f"""ssh-add '{"' '".join(os.path.expanduser(path) for path in paths)}'""",
verbose=verbose)
def stop(verbose: bool = True) -> None:
if "SSH_AGENT_PID" in os.environ:
LOCALRUNNER.run("ssh-agent -k",
ignore_status=True,
verbose=verbose)
try:
del os.environ["SSH_AUTH_SOCK"]
del os.environ["SSH_AGENT_PID"]
except KeyError:
pass
def apply_file(cls, kluster, config_path, namespace=None, # pylint: disable=too-many-locals
timeout=KUBECTL_TIMEOUT, environ=None, envsubst=True,
modifiers: List[Callable] = None):
if environ:
environ_str = (' '.join([f'{name}="{value}"' for name, value in environ.items()])) + ' '
else:
environ_str = ''
with NamedTemporaryFile(mode='tw') as temp_file:
resulted_content = []
if envsubst:
data = LOCALRUNNER.run(f'{environ_str}envsubst<{config_path}', verbose=False).stdout
else:
with open(config_path, 'r') as config_file_stream:
data = config_file_stream.read()
file_content = yaml.load_all(data)
for doc in file_content:
if modifiers:
for modifier in modifiers:
modifier(doc)
resulted_content.append(doc)
temp_file.write(yaml.safe_dump_all(resulted_content))
temp_file.flush()
@retrying(n=0, sleep_time=5, timeout=timeout, allowed_exceptions=RuntimeError)
def run_kubectl():
try:
cls.kubectl(kluster, "apply", "-f", temp_file.name, namespace=namespace, timeout=timeout)
except invoke.exceptions.UnexpectedExit as exc:
if 'did you specify the right host or port' in exc.result.stderr:
raise RuntimeError(str(exc)) from None
raise
run_kubectl()
def get_local_kubectl_proxy() -> [str, int]:
LOGGER.info("Stop any other process listening on kubectl proxy port")
LOCALRUNNER.sudo(f"fuser -v4k {KUBECTL_PROXY_PORT}/tcp",
ignore_status=True)
LOGGER.info("Start kubectl proxy in detached mode")
proxy_port = get_free_port(address='127.0.0.1')
LOCALRUNNER.run(
f"setsid kubectl proxy --disable-filter --address '127.0.0.1' --port {proxy_port} "
"--accept-hosts '.*' > proxy.log 2>&1 < /dev/null & sleep 1")
def get_proxy_ip_port():
return LOCALRUNNER.run(
"grep -P '^Starting' proxy.log | grep -oP '127.0.0.1:[0-9]+'"
).stdout
ip_port = wait_for(get_proxy_ip_port, timeout=15, throw_exc=True)
return ip_port.strip().split(':')
def __call__(self, cmd, timeout=10):
deprecation("consider to use Docker Python module instead of using Docker CLI commands")
res = LOCALRUNNER.run('docker {}'.format(cmd), ignore_status=True, timeout=timeout)
if res.exit_status:
if 'No such container:' in res.stderr:
raise NotFound(res.stderr)
raise DockerException('command: {}, error: {}, output: {}'.format(cmd, res.stderr, res.stdout))
return res
def start(cls, verbose: bool = True) -> None:
if cls.is_running():
LOGGER.warning(
"ssh-agent started already:\n\t\tSSH_AUTH_SOCK=%s\n\t\tSSH_AGENT_PID=%s",
os.environ["SSH_AUTH_SOCK"], os.environ["SSH_AGENT_PID"])
return
res = LOCALRUNNER.run(
r"""eval $(ssh-agent -s) && """
r"""eval 'echo "{\"SSH_AUTH_SOCK\": \"$SSH_AUTH_SOCK\", """
r""" \"SSH_AGENT_PID\": \"$SSH_AGENT_PID\"}" >&2'""",
verbose=verbose)
if not res.ok:
raise RuntimeError()
os.environ.update(json.loads(res.stderr))
if verbose:
LOGGER.info(
"ssh-agent started successfully:\n\t\tSSH_AUTH_SOCK=%s\n\t\tSSH_AGENT_PID=%s",
os.environ["SSH_AUTH_SOCK"], os.environ["SSH_AGENT_PID"])
atexit.register(cls.stop, verbose)
def aws_cli(self, cmd) -> str:
return LOCALRUNNER.run(cmd).stdout
def create_kubectl_config(self):
LOCALRUNNER.run(
f'aws eks --region {self.region_name} update-kubeconfig --name {self.short_cluster_name}'
)
def get_token(self) -> str:
return LOCALRUNNER.run(self._aws_cmd).stdout
def add_hydra_iptables_rules(self, nodes: Optional[list] = None) -> None:
add_rules_commands = self.hydra_iptables_redirect_rules(nodes=nodes)
del_rules_commands = self.hydra_iptables_redirect_rules(command="D", nodes=nodes)
LOCALRUNNER.sudo(shell_script_cmd("\n".join(add_rules_commands)))
atexit.register(LOCALRUNNER.sudo, shell_script_cmd("\n".join(del_rules_commands)))
def local_kubectl_version(self): # pylint: disable=no-self-use
# Example of kubectl command output:
# $ kubectl version --client --short
# Client Version: v1.18.5
return LOCALRUNNER.run(
"kubectl version --client --short").stdout.rsplit(None, 1)[-1][1:]
def get_proxy_ip_port():
return LOCALRUNNER.run(
"grep -P '^Starting' proxy.log | grep -oP '127.0.0.1:[0-9]+'"
).stdout
