async def secret_update(
namespace: str,
name: str,
diff: kopf.Diff,
logger: logging.Logger,
**kwargs,
):
async with ApiClient() as api_client:
coapi = CustomObjectsApi(api_client)
core = CoreV1Api(api_client)
for operation, field_path, old_value, new_value in diff:
custom_objects = await coapi.list_namespaced_custom_object(
namespace=namespace,
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
)
for crate_custom_object in custom_objects["items"]:
host = await get_host(
core, namespace, crate_custom_object["metadata"]["name"]
)
for user_spec in crate_custom_object["spec"]["users"]:
expected_field_path = (
"data",
user_spec["password"]["secretKeyRef"]["key"],
)
if (
user_spec["password"]["secretKeyRef"]["name"] == name
and field_path == expected_field_path
):
kopf.register(
fn=subhandler_partial(
update_user_password,
host,
user_spec["name"],
old_value,
new_value,
logger,
),
id=f"update-{crate_custom_object['metadata']['name']}-{user_spec['name']}", # noqa
timeout=config.BOOTSTRAP_TIMEOUT,
)
async def restart_cluster(namespace: str, name: str, total_nodes: int,
logger: logging.Logger) -> None:
"""
Perform a rolling restart of the CrateDB cluster ``name`` in ``namespace``.
One node at a time, this function will terminate first the master nodes and
then the data nodes in the cluster. After triggering a pod's termination,
the operator will wait for that pod to be terminated and gone. It will then
wait for the cluster to have the desired number of nodes again and for the
cluster to be in a ``GREEN`` state.
:param namespace: The Kubernetes namespace where to look up CrateDB cluster.
:param name: The CrateDB custom resource name defining the CrateDB cluster.
:param total_nodes: The total number of nodes that the cluster should
consist of, per the CrateDB cluster spec.
"""
coapi = CustomObjectsApi()
core = CoreV1Api()
cluster = await coapi.get_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace,
name=name,
)
password = await get_system_user_password(namespace, name, core)
host = await get_host(core, namespace, name)
conn_factory = connection_factory(host, password)
if "master" in cluster["spec"]["nodes"]:
await restart_statefulset(core, conn_factory, namespace, name,
"master", total_nodes, logger)
for node_spec in cluster["spec"]["nodes"]["data"]:
await restart_statefulset(core, conn_factory, namespace, name,
node_spec["name"], total_nodes, logger)
async def test_update_cluster_password(faker, namespace, cleanup_handler,
kopf_runner, api_client):
coapi = CustomObjectsApi(api_client)
core = CoreV1Api(api_client)
name = faker.domain_word()
password = faker.password(length=40)
new_password = faker.password(length=40)
username = faker.user_name()
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"))
await asyncio.gather(
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password)},
metadata=V1ObjectMeta(name=f"user-{name}",
labels={LABEL_USER_PASSWORD: "******"}),
type="Opaque",
),
), )
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": CRATE_VERSION,
},
"nodes": {
"data": [{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}]
},
"users": [
{
"name": username,
"password": {
"secretKeyRef": {
"key": "password",
"name": f"user-{name}",
}
},
},
],
},
},
)
host = await asyncio.wait_for(
get_public_host(core, namespace.metadata.name, name),
# It takes a while to retrieve an external IP on AKS.
timeout=DEFAULT_TIMEOUT * 5,
)
await core.patch_namespaced_secret(
namespace=namespace.metadata.name,
name=f"user-{name}",
body=V1Secret(data={"password": b64encode(new_password)}, ),
)
await assert_wait_for(
True,
is_password_set,
host,
new_password,
username,
timeout=DEFAULT_TIMEOUT * 5,
)
async def test_bootstrap_license(
bootstrap_system_user: mock.AsyncMock,
bootstrap_license_mock: mock.AsyncMock,
faker,
namespace,
cleanup_handler,
kopf_runner,
api_client,
):
coapi = CustomObjectsApi(api_client)
core = CoreV1Api(api_client)
name = faker.domain_word()
license = base64.b64encode(faker.binary(64)).decode()
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"), )
await core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"license": b64encode(license)},
metadata=V1ObjectMeta(name=f"license-{name}"),
type="Opaque",
),
)
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"license": {
"secretKeyRef": {
"key": "license",
"name": f"license-{name}"
},
},
"name": "my-crate-cluster",
"version": CRATE_VERSION,
},
"nodes": {
"data": [{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}]
},
},
},
)
await assert_wait_for(
True,
was_license_set,
bootstrap_license_mock,
mock.ANY,
namespace.metadata.name,
f"crate-data-data-{name}-0",
False,
{"secretKeyRef": {
"key": "license",
"name": f"license-{name}"
}},
timeout=DEFAULT_TIMEOUT * 3,
)
async def test_scale_cluster(
repl_master_from,
repl_master_to,
repl_hot_from,
repl_hot_to,
repl_cold_from,
repl_cold_to,
faker,
namespace,
cleanup_handler,
cratedb_crd,
kopf_runner,
):
coapi = CustomObjectsApi()
core = CoreV1Api()
name = faker.domain_word()
# Clean up persistent volume after the test
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"))
body = {
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": "4.1.5",
},
"nodes": {
"data": []
},
},
}
if repl_master_from:
body["spec"]["nodes"]["master"] = {
"replicas": repl_master_from,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1
},
},
}
body["spec"]["nodes"]["data"].append(
{
"name": "hot",
"replicas": repl_hot_from,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1
},
},
}, )
if repl_cold_from:
body["spec"]["nodes"]["data"].append(
{
"name": "cold",
"replicas": repl_cold_from,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1
},
},
}, )
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body=body,
)
host = await asyncio.wait_for(
get_public_host(core, namespace.metadata.name, name),
timeout=BACKOFF_TIME *
5, # It takes a while to retrieve an external IP on AKS.
)
password = await get_system_user_password(namespace.metadata.name, name,
core)
await assert_wait_for(
True,
is_cluster_healthy,
connection_factory(host, password),
repl_master_from + repl_hot_from + repl_cold_from,
err_msg="Cluster wasn't healthy after 5 minutes.",
timeout=BACKOFF_TIME * 5,
)
patch_body = []
if repl_master_from != repl_master_to:
patch_body.append({
"op": "replace",
"path": "/spec/nodes/master/replicas",
"value": repl_master_to,
})
if repl_hot_from != repl_hot_to:
patch_body.append({
"op": "replace",
"path": "/spec/nodes/data/0/replicas",
"value": repl_hot_to,
})
if repl_cold_from != repl_cold_to:
patch_body.append({
"op": "replace",
"path": "/spec/nodes/data/1/replicas",
"value": repl_cold_to,
})
await coapi.patch_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
name=name,
body=patch_body,
)
await assert_wait_for(
True,
is_cluster_healthy,
connection_factory(host, password),
repl_master_to + repl_hot_to + repl_cold_to,
err_msg="Cluster wasn't healthy after 5 minutes.",
timeout=BACKOFF_TIME * 5,
)
async def test_restart_cluster(faker, namespace, cleanup_handler, cratedb_crd,
kopf_runner):
coapi = CustomObjectsApi()
core = CoreV1Api()
name = faker.domain_word()
# Clean up persistent volume after the test
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"))
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": "4.1.5",
},
"nodes": {
"data": [
{
"name": "hot",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
},
{
"name": "cold",
"replicas": 2,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
},
],
},
},
},
)
host = await asyncio.wait_for(
get_public_host(core, namespace.metadata.name, name),
timeout=BACKOFF_TIME *
5, # It takes a while to retrieve an external IP on AKS.
)
password = await get_system_user_password(namespace.metadata.name, name,
core)
await assert_wait_for(
True,
do_pods_exist,
core,
namespace.metadata.name,
{
f"crate-data-hot-{name}-0",
f"crate-data-cold-{name}-0",
f"crate-data-cold-{name}-1",
},
)
await assert_wait_for(
True,
is_cluster_healthy,
connection_factory(host, password),
err_msg="Cluster wasn't healthy after 5 minutes.",
timeout=BACKOFF_TIME * 5,
)
pods = await core.list_namespaced_pod(namespace=namespace.metadata.name)
original_pods = {p.metadata.uid for p in pods.items}
await asyncio.wait_for(
restart_cluster(namespace.metadata.name, name, 3,
logging.getLogger(__name__)),
BACKOFF_TIME * 15,
)
pods = await core.list_namespaced_pod(namespace=namespace.metadata.name)
new_pods = {p.metadata.uid for p in pods.items}
assert original_pods.intersection(new_pods) == set()
async def test_bootstrap_users(
bootstrap_license_mock: mock.AsyncMock,
faker,
namespace,
cleanup_handler,
kopf_runner,
):
coapi = CustomObjectsApi()
core = CoreV1Api()
name = faker.domain_word()
password1 = faker.password(length=40)
password2 = faker.password(length=30)
username1 = faker.user_name()
username2 = faker.user_name()
cleanup_handler.append(
core.delete_persistent_volume(name=f"temp-pv-{namespace.metadata.name}-{name}")
)
await asyncio.gather(
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password1)},
metadata=V1ObjectMeta(name=f"user-{name}-1"),
type="Opaque",
),
),
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password2)},
metadata=V1ObjectMeta(name=f"user-{name}-2"),
type="Opaque",
),
),
)
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {"name": name},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": "4.1.5",
},
"nodes": {
"data": [
{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}
]
},
"users": [
{
"name": username1,
"password": {
"secretKeyRef": {
"key": "password",
"name": f"user-{name}-1",
}
},
},
{
"name": username2,
"password": {
"secretKeyRef": {
"key": "password",
"name": f"user-{name}-2",
}
},
},
],
},
},
)
host = await asyncio.wait_for(
get_public_host(core, namespace.metadata.name, name),
timeout=BACKOFF_TIME * 5, # It takes a while to retrieve an external IP on AKS.
)
password_system = await get_system_user_password(
namespace.metadata.name, name, core
)
await assert_wait_for(
True,
does_user_exist,
host,
password_system,
SYSTEM_USERNAME,
timeout=BACKOFF_TIME * 5,
)
await assert_wait_for(
True, does_user_exist, host, password1, username1, timeout=BACKOFF_TIME * 3,
)
await assert_wait_for(
True, does_user_exist, host, password2, username2, timeout=BACKOFF_TIME * 3,
)
async def test_upgrade_cluster(
faker, namespace, cleanup_handler, kopf_runner, api_client
):
version_from = "4.4.1"
version_to = "4.4.2"
coapi = CustomObjectsApi(api_client)
core = CoreV1Api(api_client)
name = faker.domain_word()
host, password = await start_cluster(
name, namespace, cleanup_handler, core, coapi, 3, version_from
)
await assert_wait_for(
True,
do_pods_exist,
core,
namespace.metadata.name,
{
f"crate-data-hot-{name}-0",
f"crate-data-hot-{name}-1",
f"crate-data-hot-{name}-2",
},
)
conn_factory = connection_factory(host, password)
await assert_wait_for(
True,
is_cluster_healthy,
conn_factory,
3,
err_msg="Cluster wasn't healthy",
timeout=DEFAULT_TIMEOUT,
)
await create_test_sys_jobs_table(conn_factory)
pods = await core.list_namespaced_pod(namespace=namespace.metadata.name)
original_pods = {p.metadata.uid for p in pods.items}
await coapi.patch_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
name=name,
body=[
{
"op": "replace",
"path": "/spec/cluster/version",
"value": version_to,
},
],
)
await assert_wait_for(
False,
do_pod_ids_exist,
core,
namespace.metadata.name,
original_pods,
timeout=DEFAULT_TIMEOUT * 15,
)
await assert_wait_for(
True,
is_kopf_handler_finished,
coapi,
name,
namespace.metadata.name,
"operator.cloud.crate.io/cluster_update.upgrade",
err_msg="Upgrade has not finished",
timeout=DEFAULT_TIMEOUT * 5,
)
await assert_wait_for(
True,
is_kopf_handler_finished,
coapi,
name,
namespace.metadata.name,
"operator.cloud.crate.io/cluster_update.restart",
err_msg="Restart has not finished",
timeout=DEFAULT_TIMEOUT,
)
await assert_wait_for(
True,
is_cluster_healthy,
connection_factory(host, password),
3,
err_msg="Cluster wasn't healthy",
timeout=DEFAULT_TIMEOUT,
)
async def test_create_minimal(self, faker, namespace, cleanup_handler,
kopf_runner):
apps = AppsV1Api()
coapi = CustomObjectsApi()
core = CoreV1Api()
name = faker.domain_word()
# Clean up persistent volume after the test
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"))
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": "4.1.5",
},
"nodes": {
"data": [{
"name": "data",
"replicas": 3,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}]
},
},
},
)
await assert_wait_for(
True,
self.does_statefulset_exist,
apps,
namespace.metadata.name,
f"crate-data-data-{name}",
)
await assert_wait_for(
True,
self.do_pods_exist,
core,
namespace.metadata.name,
{f"crate-data-data-{name}-{i}"
for i in range(3)},
)