async def test_conflict_raises(faker, namespace, api_client):
core = CoreV1Api(api_client)
name = faker.domain_word()
ns = namespace.metadata.name
password1 = faker.password(length=12)
password2 = faker.password(length=12)
await core.create_namespaced_secret(
namespace=ns,
body=V1Secret(
data={"password": b64encode(password1)},
metadata=V1ObjectMeta(name=name),
type="Opaque",
),
)
with pytest.raises(ApiException):
await call_kubeapi(
core.create_namespaced_secret,
logger,
namespace=ns,
body=V1Secret(
data={"password": b64encode(password2)},
metadata=V1ObjectMeta(name=name),
type="Opaque",
),
)
secret = await core.read_namespaced_secret(name=name, namespace=ns)
assert b64decode(secret.data["password"]) == password1
async def test_conflict_logs(faker, namespace, caplog, api_client):
caplog.set_level(logging.DEBUG, logger=__name__)
core = CoreV1Api(api_client)
name = faker.domain_word()
ns = namespace.metadata.name
password1 = faker.password(length=12)
password2 = faker.password(length=12)
await core.create_namespaced_secret(
namespace=ns,
body=V1Secret(
data={"password": b64encode(password1)},
metadata=V1ObjectMeta(name=name),
type="Opaque",
),
)
await call_kubeapi(
core.create_namespaced_secret,
logger,
continue_on_conflict=True,
namespace=ns,
body=V1Secret(
data={"password": b64encode(password2)},
metadata=V1ObjectMeta(name=name),
type="Opaque",
),
)
secret = await core.read_namespaced_secret(name=name, namespace=ns)
assert b64decode(secret.data["password"]) == password1
assert (
f"Failed creating V1Secret '{ns}/{name}' because it already exists. Continuing."
in caplog.messages
)
def _build_secret_for_service_token(self, parent: GafaelfawrServiceToken,
token: Token) -> V1Secret:
"""Construct a new Secret object.
Parameters
----------
parent : `GafaelfawrServiceSecret`
The parent GafaelfawrServiceSecret object.
token : `gafaelfawr.models.token.Token`
The Gafaelfawr token to store in the secret.
"""
return V1Secret(
api_version="v1",
kind="Secret",
data={"token": self._encode_token(token)},
metadata=V1ObjectMeta(
name=parent.name,
namespace=parent.namespace,
annotations=parent.annotations,
labels=parent.labels,
owner_references=[
V1OwnerReference(
api_version="gafaelfawr.lsst.io/v1alpha1",
block_owner_deletion=True,
controller=True,
kind="GafaelfawrServiceToken",
name=parent.name,
uid=parent.uid,
),
],
),
type="Opaque",
)
async def assert_kubernetes_secrets_are_correct(
factory: ComponentFactory, mock: MockKubernetesApi, is_fresh: bool = True
) -> None:
token_service = factory.create_token_service()
# Get all of the GafaelfawrServiceToken custom objects.
service_tokens = mock.get_all_objects_for_test("GafaelfawrServiceToken")
# Calculate the expected secrets.
expected = [
V1Secret(
api_version="v1",
kind="Secret",
data={"token": ANY},
metadata=V1ObjectMeta(
name=t["metadata"]["name"],
namespace=t["metadata"]["namespace"],
annotations=t["metadata"].get("annotations", {}),
labels=t["metadata"].get("labels", {}),
owner_references=[
V1OwnerReference(
api_version="gafaelfawr.lsst.io/v1alpha1",
block_owner_deletion=True,
controller=True,
kind="GafaelfawrServiceToken",
name=t["metadata"]["name"],
uid=t["metadata"]["uid"],
),
],
),
type="Opaque",
)
for t in service_tokens
]
expected = sorted(
expected, key=lambda o: (o.metadata.namespace, o.metadata.name)
)
assert mock.get_all_objects_for_test("Secret") == expected
# Now check that every token in those secrets is correct.
for service_token in service_tokens:
name = service_token["metadata"]["name"]
namespace = service_token["metadata"]["namespace"]
secret = await mock.read_namespaced_secret(name, namespace)
data = await token_data_from_secret(token_service, secret)
assert data == TokenData(
token=data.token,
username=service_token["spec"]["service"],
token_type=TokenType.service,
scopes=service_token["spec"]["scopes"],
created=data.created,
expires=None,
name=None,
uid=None,
groups=None,
)
if is_fresh:
now = current_datetime()
assert now - timedelta(seconds=5) <= data.created <= now
async def test_mock(mock_kubernetes: MockKubernetesApi) -> None:
custom: Dict[str, Any] = {
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["admin:token"],
},
}
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": "bogus"},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
custom["metadata"]["namespace"],
"gafaelfawrservicetokens",
custom,
)
await mock_kubernetes.create_namespaced_secret(secret.metadata.namespace,
secret)
assert await mock_kubernetes.list_cluster_custom_object(
"gafaelfawr.lsst.io", "v1alpha1", "gafaelfawrservicetokens") == {
"items": [{
**custom, "metadata": {
**custom["metadata"], "uid": ANY
}
}]
}
assert mock_kubernetes.get_all_objects_for_test("Secret") == [secret]
def error(method: str, *args: Any) -> None:
assert method == "replace_namespaced_custom_object"
raise ValueError("some exception")
mock_kubernetes.error_callback = error
with pytest.raises(ValueError) as excinfo:
await mock_kubernetes.replace_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
custom["metadata"]["namespace"],
"gafaelfawrservicetokens",
"gafaelfawr-secret",
custom,
)
assert str(excinfo.value) == "some exception"
def get_system_user_secret(owner_references: Optional[List[V1OwnerReference]],
name: str, labels: LabelType) -> V1Secret:
return V1Secret(
data={"password": b64encode(gen_password(50))},
metadata=V1ObjectMeta(
name=f"user-system-{name}",
labels=labels,
owner_references=owner_references,
),
type="Opaque",
)
async def test_success(faker, namespace, api_client):
core = CoreV1Api(api_client)
name = faker.domain_word()
password = faker.password(length=12)
await call_kubeapi(
core.create_namespaced_secret,
logger,
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password)},
metadata=V1ObjectMeta(name=name),
type="Opaque",
),
)
secret = await core.read_namespaced_secret(
name=name, namespace=namespace.metadata.name
)
assert b64decode(secret.data["password"]) == password
async def ensure_user_password_label(core: CoreV1Api, namespace: str,
secret_name: str):
"""
Add the LABEL_USER_PASSWORD label to a namespaced secret.
During testing, the function returns the public IP address, because the
operator doesn't run inside Kubernetes during tests but outside. And the
only way to connect to the CrateDB cluster is to go through the public
interface.
:param core: An instance of the Kubernetes Core V1 API.
:param namespace: The namespace where the Kubernetes Secret is deployed.
:param secret_name: The name of the Kubernetes Secret.
"""
await core.patch_namespaced_secret(
namespace=namespace,
name=secret_name,
body=V1Secret(metadata=V1ObjectMeta(
labels={LABEL_USER_PASSWORD: "******"}, ), ),
)
async def test_update_cluster_password(faker, namespace, cleanup_handler,
kopf_runner, api_client):
coapi = CustomObjectsApi(api_client)
core = CoreV1Api(api_client)
name = faker.domain_word()
password = faker.password(length=40)
new_password = faker.password(length=40)
username = faker.user_name()
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"))
await asyncio.gather(
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password)},
metadata=V1ObjectMeta(name=f"user-{name}",
labels={LABEL_USER_PASSWORD: "******"}),
type="Opaque",
),
), )
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": CRATE_VERSION,
},
"nodes": {
"data": [{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}]
},
"users": [
{
"name": username,
"password": {
"secretKeyRef": {
"key": "password",
"name": f"user-{name}",
}
},
},
],
},
},
)
host = await asyncio.wait_for(
get_public_host(core, namespace.metadata.name, name),
# It takes a while to retrieve an external IP on AKS.
timeout=DEFAULT_TIMEOUT * 5,
)
await core.patch_namespaced_secret(
namespace=namespace.metadata.name,
name=f"user-{name}",
body=V1Secret(data={"password": b64encode(new_password)}, ),
)
await assert_wait_for(
True,
is_password_set,
host,
new_password,
username,
timeout=DEFAULT_TIMEOUT * 5,
)
async def test_bootstrap_license(
bootstrap_system_user: mock.AsyncMock,
bootstrap_license_mock: mock.AsyncMock,
faker,
namespace,
cleanup_handler,
kopf_runner,
api_client,
):
coapi = CustomObjectsApi(api_client)
core = CoreV1Api(api_client)
name = faker.domain_word()
license = base64.b64encode(faker.binary(64)).decode()
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"), )
await core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"license": b64encode(license)},
metadata=V1ObjectMeta(name=f"license-{name}"),
type="Opaque",
),
)
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {
"name": name
},
"spec": {
"cluster": {
"imageRegistry": "crate",
"license": {
"secretKeyRef": {
"key": "license",
"name": f"license-{name}"
},
},
"name": "my-crate-cluster",
"version": CRATE_VERSION,
},
"nodes": {
"data": [{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}]
},
},
},
)
await assert_wait_for(
True,
was_license_set,
bootstrap_license_mock,
mock.ANY,
namespace.metadata.name,
f"crate-data-data-{name}-0",
False,
{"secretKeyRef": {
"key": "license",
"name": f"license-{name}"
}},
timeout=DEFAULT_TIMEOUT * 3,
)
async def test_bootstrap_users(
bootstrap_license_mock: mock.AsyncMock,
faker,
namespace,
cleanup_handler,
kopf_runner,
):
coapi = CustomObjectsApi()
core = CoreV1Api()
name = faker.domain_word()
password1 = faker.password(length=40)
password2 = faker.password(length=30)
username1 = faker.user_name()
username2 = faker.user_name()
cleanup_handler.append(
core.delete_persistent_volume(name=f"temp-pv-{namespace.metadata.name}-{name}")
)
await asyncio.gather(
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password1)},
metadata=V1ObjectMeta(name=f"user-{name}-1"),
type="Opaque",
),
),
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password2)},
metadata=V1ObjectMeta(name=f"user-{name}-2"),
type="Opaque",
),
),
)
await coapi.create_namespaced_custom_object(
group=API_GROUP,
version="v1",
plural=RESOURCE_CRATEDB,
namespace=namespace.metadata.name,
body={
"apiVersion": "cloud.crate.io/v1",
"kind": "CrateDB",
"metadata": {"name": name},
"spec": {
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": "4.1.5",
},
"nodes": {
"data": [
{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"memory": "1Gi",
"heapRatio": 0.25,
"disk": {
"storageClass": "default",
"size": "16GiB",
"count": 1,
},
},
}
]
},
"users": [
{
"name": username1,
"password": {
"secretKeyRef": {
"key": "password",
"name": f"user-{name}-1",
}
},
},
{
"name": username2,
"password": {
"secretKeyRef": {
"key": "password",
"name": f"user-{name}-2",
}
},
},
],
},
},
)
host = await asyncio.wait_for(
get_public_host(core, namespace.metadata.name, name),
timeout=BACKOFF_TIME * 5, # It takes a while to retrieve an external IP on AKS.
)
password_system = await get_system_user_password(
namespace.metadata.name, name, core
)
await assert_wait_for(
True,
does_user_exist,
host,
password_system,
SYSTEM_USERNAME,
timeout=BACKOFF_TIME * 5,
)
await assert_wait_for(
True, does_user_exist, host, password1, username1, timeout=BACKOFF_TIME * 3,
)
await assert_wait_for(
True, does_user_exist, host, password2, username2, timeout=BACKOFF_TIME * 3,
)
async def test_resume_set_secret_labels(
faker,
namespace,
cleanup_handler,
kopf_runner,
api_client,
):
core = CoreV1Api(api_client)
name = faker.domain_word()
password1 = faker.password(length=40)
password2 = faker.password(length=40)
cleanup_handler.append(
core.delete_persistent_volume(
name=f"temp-pv-{namespace.metadata.name}-{name}"))
await asyncio.gather(
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password1)},
metadata=V1ObjectMeta(name=f"user-{name}-1"),
type="Opaque",
),
),
core.create_namespaced_secret(
namespace=namespace.metadata.name,
body=V1Secret(
data={"password": b64encode(password2)},
metadata=V1ObjectMeta(name=f"user-{name}-2",
labels={LABEL_USER_PASSWORD: "******"}),
type="Opaque",
),
),
)
secret_1 = await core.read_namespaced_secret(
name=f"user-{name}-1", namespace=namespace.metadata.name)
secret_2 = await core.read_namespaced_secret(
name=f"user-{name}-2", namespace=namespace.metadata.name)
assert secret_1.metadata.labels is None
assert LABEL_USER_PASSWORD in secret_2.metadata.labels
await update_cratedb_resource(
namespace=namespace.metadata.name,
name=f"user-{name}",
spec={
"cluster": {
"imageRegistry": "crate",
"name": "my-crate-cluster",
"version": CRATE_VERSION,
},
"nodes": {
"data": [{
"name": "data",
"replicas": 1,
"resources": {
"cpus": 0.5,
"disk": {
"count": 1,
"size": "16GiB",
"storageClass": "default",
},
"heapRatio": 0.25,
"memory": "1Gi",
},
}]
},
"users": [
{
"name": name,
"password": {
"secretKeyRef": {
"key": b64encode(password1),
"name": f"user-{name}-1",
}
},
},
{
"name": name,
"password": {
"secretKeyRef": {
"key": b64encode(password1),
"name": f"user-{name}-2",
}
},
},
],
},
)
secret_1_after_resume = await core.read_namespaced_secret(
name=f"user-{name}-1", namespace=namespace.metadata.name)
secret_2_after_resume = await core.read_namespaced_secret(
name=f"user-{name}-2", namespace=namespace.metadata.name)
assert LABEL_USER_PASSWORD in secret_1_after_resume.metadata.labels
assert LABEL_USER_PASSWORD in secret_2_after_resume.metadata.labels
async def test_errors_replace_read(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
token_service = factory.create_token_service()
# Create a secret that should exist but has an invalid token.
secret = V1Secret(
api_version="v1",
data={"token": token_as_base64(Token())},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("mobu", secret)
# Simulate some errors. The callback function takes the operation and the
# secret name.
def error_callback_replace(method: str, *args: Any) -> None:
if method in ("replace_namespaced_secret"):
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback_replace
# Now run the synchronization. The secret should be left unchanged, but
# we should still create the missing nublado2 secret.
await kubernetes_service.update_service_tokens()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
assert secret in objects
good_secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)
assert await token_data_from_secret(token_service, good_secret)
# We should have also updated the status of the parent custom object.
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Kubernetes API error: (500)\nReason: Some error\n",
"observedGeneration": 1,
"reason": StatusReason.Failed.value,
"status": "False",
"type": "SecretCreated",
}
]
# Try again, but simulating an error in retrieving a secret.
def error_callback_read(method: str, *args: Any) -> None:
if method == "read_namespaced_secret":
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback_read
# Now run the synchronization. As before, the secret should be left
# unchanged, and the good secret should also be left unchanged.
await kubernetes_service.update_service_tokens()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
assert secret in objects
async def test_modify(
factory: ComponentFactory,
mock_kubernetes: MockKubernetesApi,
caplog: LogCaptureFixture,
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
token_service = factory.create_token_service()
# Valid secret but with a bogus token.
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": "bogus"},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("mobu", secret)
# Valid secret but with a nonexistent token.
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": token_as_base64(Token())},
metadata=V1ObjectMeta(
name="gafaelfawr",
namespace="nublado2",
labels={
"foo": "bar",
"other": "blah",
},
annotations={
"argocd.argoproj.io/compare-options": "IgnoreExtraneous",
"argocd.argoproj.io/sync-options": "Prune=false",
},
),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("nublado2", secret)
# Update the secrets. This should replace both with fresh secrets.
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
# Check the logging.
assert parse_log(caplog) == [
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "admin:token",
"token_username": "******",
},
{
"event": "Updated mobu/gafaelfawr-secret secret",
"scopes": ["admin:token"],
"severity": "info",
"service": "mobu",
},
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "",
"token_username": "******",
},
{
"event": "Updated nublado2/gafaelfawr secret",
"scopes": [],
"severity": "info",
"service": "nublado-hub",
},
]
# Replace one secret with a valid token for the wrong service.
async with factory.session.begin():
token = await token_service.create_token_from_admin_request(
AdminTokenRequest(
username="******",
token_type=TokenType.service,
scopes=["admin:token"],
),
TokenData.internal_token(),
ip_address=None,
)
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": token_as_base64(token)},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.replace_namespaced_secret(
"gafaelfawr-secret", "mobu", secret
)
# Replace the other token with a valid token with the wrong scopes.
async with factory.session.begin():
token = await token_service.create_token_from_admin_request(
AdminTokenRequest(
username="******",
token_type=TokenType.service,
scopes=["read:all"],
),
TokenData.internal_token(),
ip_address=None,
)
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": token_as_base64(token)},
metadata=V1ObjectMeta(name="gafaelfawr", namespace="nublado2"),
type="Opaque",
)
await mock_kubernetes.replace_namespaced_secret(
"gafaelfawr", "nublado2", secret
)
# Update the secrets. This should create new tokens for both.
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
nublado_secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)
# Finally, replace a secret with one with no token.
secret = V1Secret(
api_version="v1",
data={},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.replace_namespaced_secret(
"gafaelfawr-secret", "mobu", secret
)
# Update the secrets. This should create a new token for the first secret
# but not for the second.
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(
factory, mock_kubernetes, is_fresh=False
)
assert nublado_secret == await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)