async def reset_allocation(namespace: str, pod_name: str, has_ssl: bool) -> None:
"""
Reset all temporary node deallocations to none.
.. note::
Ideally, we'd be using the system user to reset the allocation
exclusions. However, `due to a bug
<https://github.com/crate/crate/pull/10083>`_, this isn't possible in
CrateDB <= 4.1.6. We therefore fall back to the "exec-in-container"
approach that we also use during cluster bootstrapping.
:param namespace: The Kubernetes namespace for the CrateDB cluster.
:param pod_name: The pod name of one of the eligible master nodes in
the cluster. Used to ``exec`` into.
:param has_ssl: When ``True``, ``crash`` will establish a connection to
the CrateDB cluster from inside the ``crate`` container using SSL/TLS.
This must match how the cluster is configured, otherwise ``crash``
won't be able to connect, since non-encrypted connections are forbidden
when SSL/TLS is enabled, and encrypted connections aren't possible when
no SSL/TLS is configured.
"""
# async with conn_factory() as conn:
# async with conn.cursor() as cursor:
# await cursor.execute(
# """
# RESET GLOBAL "cluster.routing.allocation.exclude._name"
# """,
# )
scheme = "https" if has_ssl else "http"
command_grant = [
"crash",
"--verify-ssl=false",
f"--host={scheme}://localhost:4200",
"-c",
'RESET GLOBAL "cluster.routing.allocation.exclude._name";',
]
async with WsApiClient() as ws_api_client:
core_ws = CoreV1Api(ws_api_client)
await core_ws.connect_get_namespaced_pod_exec(
namespace=namespace,
name=pod_name,
command=command_grant,
container="crate",
stderr=True,
stdin=False,
stdout=True,
tty=False,
)
async def test_exec_ws(self):
class WsMock:
def __init__(self):
self.iter = 0
def __aiter__(self):
return self
async def __aexit__(self, exc_type, exc, tb):
return self
async def __aenter__(self):
return self
async def __anext__(self):
self.iter += 1
if self.iter > 5:
raise StopAsyncIteration
return WsResponse((chr(1) + 'mock').encode('utf-8'))
mock = CoroutineMock()
mock.RESTClientObject.return_value.pool_manager = mock
mock.ws_connect.return_value = WsMock()
with patch('kubernetes_asyncio.client.api_client.rest', mock):
api_client = WsApiClient()
api_client.configuration.host = 'https://localhost'
ws = client.CoreV1Api(api_client=api_client)
resp = ws.connect_get_namespaced_pod_exec('pod',
'namespace',
command="mock-command",
stderr=True,
stdin=False,
stdout=True,
tty=False)
ret = await resp
self.assertEqual(ret, 'mock' * 5)
mock.ws_connect.assert_called_once_with(
'wss://localhost/api/v1/namespaces/namespace/pods/pod/exec?'
'command=mock-command&stderr=True&stdin=False&stdout=True&tty=False',
headers={
'sec-websocket-protocol': 'v4.channel.k8s.io',
'Accept': '*/*',
'User-Agent': api_client.user_agent
})
async def do_exec(self):
to_exec = await self.list_pods(lambda conds: 'Ready' in conds)
if not self.args.cmd:
self.args.cmd = input('Enter a command to execut (or use --exec):')
if not self.args.cmd:
return
else:
print('Execute command {}'.format(self.args.cmd))
v1_ws = client.CoreV1Api(api_client=WsApiClient())
batch = []
for i, pod in enumerate(to_exec):
pod_namespace = pod.metadata.namespace
pod_name = pod.metadata.name
pod_container = pod.status.container_statuses[0].name
print('{}/{} {}/{} sending command'.format(i + 1, len(to_exec),
pod_namespace,
pod_name))
async def __exec_and_print(pod_name, pod_namespace, pod_container):
call = await v1_ws.connect_get_namespaced_pod_exec(
pod_name,
pod_namespace,
container=pod_container,
command=self.args.cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
print('Response from {}/{}\n{}\n'.format(
pod_namespace, pod_name, call))
batch.append(
__exec_and_print(pod_name, pod_namespace, pod_container))
if self.args.batch_size is not None and len(
batch) >= self.args.batch_size:
await asyncio.wait(batch)
batch = []
if batch:
await asyncio.wait(batch)
async def main():
# Configs can be set in Configuration class directly or using helper
# utility. If no argument provided, the config will be loaded from
# default location.
config.load_kube_config()
v1 = client.CoreV1Api()
print("Try to find a pod with busybox (name busybox*) ...")
ret = await v1.list_pod_for_all_namespaces()
for i in ret.items:
if i.metadata.name.startswith('busybox'):
pod = i.metadata.name
namespace = i.metadata.namespace
print('Buxy box', pod, 'namespace', namespace)
break
else:
print('Busybox not found !')
return
v1_ws = client.CoreV1Api(api_client=WsApiClient())
exec_command = [
'/bin/sh', '-c',
'echo This message goes to stderr >&2; echo This message goes to stdout'
]
resp = v1_ws.connect_get_namespaced_pod_exec(pod,
namespace,
command=exec_command,
stderr=True,
stdin=False,
stdout=True,
tty=False)
ret = await resp
print("Response: ", ret)
async def test_pod_apis(self):
client = api_client.ApiClient(configuration=self.config)
client_ws = WsApiClient(configuration=self.config)
api = core_v1_api.CoreV1Api(client)
api_ws = core_v1_api.CoreV1Api(client_ws)
name = 'busybox-test-' + short_uuid()
pod_manifest = {
'apiVersion': 'v1',
'kind': 'Pod',
'metadata': {
'name': name
},
'spec': {
'containers': [{
'image':
'busybox',
'name':
'sleep',
"args":
["/bin/sh", "-c", "while true;do date;sleep 5; done"]
}]
}
}
resp = await api.create_namespaced_pod(body=pod_manifest,
namespace='default')
self.assertEqual(name, resp.metadata.name)
self.assertTrue(resp.status.phase)
while True:
resp = await api.read_namespaced_pod(name=name,
namespace='default')
self.assertEqual(name, resp.metadata.name)
self.assertTrue(resp.status.phase)
if resp.status.phase != 'Pending':
break
time.sleep(1)
exec_command = ['/bin/sh', '-c', 'for i in $(seq 1 3); do date; done']
resp = await api_ws.connect_get_namespaced_pod_exec(
name,
'default',
command=exec_command,
stderr=False,
stdin=False,
stdout=True,
tty=False)
print('EXEC response : %s' % resp)
self.assertEqual(3, len(resp.splitlines()))
exec_command = 'uptime'
resp = await api_ws.connect_post_namespaced_pod_exec(
name,
'default',
command=exec_command,
stderr=False,
stdin=False,
stdout=True,
tty=False)
print('EXEC response : %s' % resp)
self.assertEqual(1, len(resp.splitlines()))
resp = await api.list_pod_for_all_namespaces()
number_of_pods = len(resp.items)
self.assertTrue(number_of_pods > 0)
resp = await api.delete_namespaced_pod(name=name,
body={},
namespace='default')
async def bootstrap_license(
core: CoreV1Api,
namespace: str,
master_node_pod: str,
has_ssl: bool,
license: SecretKeyRefContainer,
logger: logging.Logger,
) -> None:
"""
Set a license key on a CrateDB cluster.
When starting up a cluster, the operator doesn't have a system user yet
that it could use. The operator will therefore ``exec`` into the ``crate``
container in the ``master_node_pod`` and attempt to set a license key.
:param core: An instance of the Kubernetes Core V1 API.
:param namespace: The Kubernetes namespace for the CrateDB cluster.
:param master_node_pod: The pod name of one of the eligible master nodes in
the cluster. Used to ``exec`` into.
:param has_ssl: When ``True``, ``crash`` will establish a connection to
the CrateDB cluster from inside the ``crate`` container using SSL/TLS.
This must match how the cluster is configured, otherwise ``crash``
won't be able to connect, since non-encrypted connections are forbidden
when SSL/TLS is enabled, and encrypted connections aren't possible when
no SSL/TLS is configured.
:param license: A ``secretKeyRef`` to the Kubernetes secret that holds the
CrateDB license key.
"""
scheme = "https" if has_ssl else "http"
license_key = await resolve_secret_key_ref(namespace,
license["secretKeyRef"], core)
license_key_quoted = QuotedString(license_key).getquoted().decode()
command = [
"crash",
"--verify-ssl=false",
f"--host={scheme}://localhost:4200",
"-c",
f"SET LICENSE {license_key_quoted};",
]
core_ws = CoreV1Api(api_client=WsApiClient())
exception_logger = logger.exception if config.TESTING else logger.error
while True:
try:
logger.info("Trying to set license ...")
result = await core_ws.connect_get_namespaced_pod_exec(
namespace=namespace,
name=master_node_pod,
command=command,
container="crate",
stderr=True,
stdin=False,
stdout=True,
tty=False,
)
except ApiException as e:
# We don't use `logger.exception()` to not accidentally include the
# license key in the log messages which might be part of the string
# representation of the exception.
exception_logger("... failed. Status: %s Reason: %s", e.status,
e.reason)
await asyncio.sleep(BACKOFF_TIME / 2.0)
except WSServerHandshakeError as e:
# We don't use `logger.exception()` to not accidentally include the
# license key in the log messages which might be part of the string
# representation of the exception.
exception_logger("... failed. Status: %s Message: %s", e.status,
e.message)
await asyncio.sleep(BACKOFF_TIME / 2.0)
else:
if "SET OK" in result:
logger.info("... success")
break
else:
logger.info("... error. %s", result)
await asyncio.sleep(BACKOFF_TIME / 2.0)
async def bootstrap_system_user(
core: CoreV1Api,
namespace: str,
name: str,
master_node_pod: str,
has_ssl: bool,
logger: logging.Logger,
) -> None:
"""
Exec into to a CrateDB container and create the system user.
When starting up a cluster, the operator doesn't have a system user yet
that it could use. The operator will therefore ``exec`` into the ``crate``
container in the ``master_node_pod`` and attempt to create a user and grant
it all privileges.
:param core: An instance of the Kubernetes Core V1 API.
:param namespace: The Kubernetes namespace for the CrateDB cluster.
:param name: The name for the ``CrateDB`` custom resource. Used to lookup
the password for the system user created during deployment.
:param master_node_pod: The pod name of one of the eligible master nodes in
the cluster. Used to ``exec`` into.
:param has_ssl: When ``True``, ``crash`` will establish a connection to
the CrateDB cluster from inside the ``crate`` container using SSL/TLS.
This must match how the cluster is configured, otherwise ``crash``
won't be able to connect, since non-encrypted connections are forbidden
when SSL/TLS is enabled, and encrypted connections aren't possible when
no SSL/TLS is configured.
"""
scheme = "https" if has_ssl else "http"
password = await get_system_user_password(namespace, name, core)
password_quoted = QuotedString(password).getquoted().decode()
# Yes, we're constructing the SQL for the system user manually. But that's
# fine in this case, because we have full control of the formatting of
# the username and it's only `[a-z]+`.
command_create_user = [
"crash",
"--verify-ssl=false",
f"--host={scheme}://localhost:4200",
"-c",
f'CREATE USER "{SYSTEM_USERNAME}" WITH (password={password_quoted});',
]
command_alter_user = [
"crash",
"--verify-ssl=false",
f"--host={scheme}://localhost:4200",
"-c",
f'ALTER USER "{SYSTEM_USERNAME}" SET (password={password_quoted});',
]
command_grant = [
"crash",
"--verify-ssl=false",
f"--host={scheme}://localhost:4200",
"-c",
f'GRANT ALL PRIVILEGES TO "{SYSTEM_USERNAME}";',
]
core_ws = CoreV1Api(api_client=WsApiClient())
exception_logger = logger.exception if config.TESTING else logger.error
needs_update = False
while True:
try:
logger.info("Trying to create system user ...")
result = await core_ws.connect_get_namespaced_pod_exec(
namespace=namespace,
name=master_node_pod,
command=command_create_user,
container="crate",
stderr=True,
stdin=False,
stdout=True,
tty=False,
)
except ApiException as e:
# We don't use `logger.exception()` to not accidentally include the
# password in the log messages which might be part of the string
# representation of the exception.
exception_logger("... failed. Status: %s Reason: %s", e.status,
e.reason)
await asyncio.sleep(BACKOFF_TIME / 2.0)
except WSServerHandshakeError as e:
# We don't use `logger.exception()` to not accidentally include the
# password in the log messages which might be part of the string
# representation of the exception.
exception_logger("... failed. Status: %s Message: %s", e.status,
e.message)
await asyncio.sleep(BACKOFF_TIME / 2.0)
else:
if "CREATE OK" in result:
logger.info("... success")
break
elif "UserAlreadyExistsException" in result:
needs_update = True
logger.info("... success. Already present")
break
else:
logger.info("... error. %s", result)
await asyncio.sleep(BACKOFF_TIME / 2.0)
if needs_update:
while True:
try:
logger.info("Trying to update system user password ...")
result = await core_ws.connect_get_namespaced_pod_exec(
namespace=namespace,
name=master_node_pod,
command=command_alter_user,
container="crate",
stderr=True,
stdin=False,
stdout=True,
tty=False,
)
except ApiException as e:
# We don't use `logger.exception()` to not accidentally include the
# password in the log messages which might be part of the string
# representation of the exception.
exception_logger("... failed. Status: %s Reason: %s", e.status,
e.reason)
await asyncio.sleep(BACKOFF_TIME / 2.0)
except WSServerHandshakeError as e:
# We don't use `logger.exception()` to not accidentally include the
# password in the log messages which might be part of the string
# representation of the exception.
exception_logger("... failed. Status: %s Message: %s",
e.status, e.message)
await asyncio.sleep(BACKOFF_TIME / 2.0)
else:
if "ALTER OK" in result:
logger.info("... success")
break
else:
logger.info("... error. %s", result)
await asyncio.sleep(BACKOFF_TIME / 2.0)
while True:
try:
logger.info("Trying to grant system user all privileges ...")
result = await core_ws.connect_get_namespaced_pod_exec(
namespace=namespace,
name=master_node_pod,
command=command_grant,
container="crate",
stderr=True,
stdin=False,
stdout=True,
tty=False,
)
except (ApiException, WSServerHandshakeError):
logger.exception("... failed")
await asyncio.sleep(BACKOFF_TIME / 2.0)
else:
if "GRANT OK" in result:
logger.info("... success")
break
else:
logger.info("... error. %s", result)
await asyncio.sleep(BACKOFF_TIME / 2.0)