async def test_errors_scope(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
await mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
{
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["invalid:scope"],
},
},
)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
await kubernetes_service.update_service_tokens()
with pytest.raises(ApiException):
await mock_kubernetes.read_namespaced_secret(
"gafaelfawr-secret", "mobu"
)
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Unknown scopes requested",
"observedGeneration": 1,
"reason": StatusReason.Failed.value,
"status": "False",
"type": "SecretCreated",
}
]
async def test_errors_replace_read(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
token_service = factory.create_token_service()
# Create a secret that should exist but has an invalid token.
secret = V1Secret(
api_version="v1",
data={"token": token_as_base64(Token())},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("mobu", secret)
# Simulate some errors. The callback function takes the operation and the
# secret name.
def error_callback_replace(method: str, *args: Any) -> None:
if method in ("replace_namespaced_secret"):
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback_replace
# Now run the synchronization. The secret should be left unchanged, but
# we should still create the missing nublado2 secret.
await kubernetes_service.update_service_tokens()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
assert secret in objects
good_secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)
assert await token_data_from_secret(token_service, good_secret)
# We should have also updated the status of the parent custom object.
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Kubernetes API error: (500)\nReason: Some error\n",
"observedGeneration": 1,
"reason": StatusReason.Failed.value,
"status": "False",
"type": "SecretCreated",
}
]
# Try again, but simulating an error in retrieving a secret.
def error_callback_read(method: str, *args: Any) -> None:
if method == "read_namespaced_secret":
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback_read
# Now run the synchronization. As before, the secret should be left
# unchanged, and the good secret should also be left unchanged.
await kubernetes_service.update_service_tokens()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
assert secret in objects
async def test_update_metadata(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
"""Test that Secret metadata is updated even if generation doesn't change.
Updates to metadata doesn't trigger a generation bump (since generation is
in metadata itself), so propagating metadata to secrets has to be handled
specially.
"""
service_token: Dict[str, Any] = {
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["admin:token"],
},
}
await mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
service_token,
)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr-secret", "mobu"
)
assert secret
# Sending a modified event does nothing.
queue: Queue[WatchEvent] = Queue()
await queue.put(
WatchEvent(
event_type=WatchEventType.MODIFIED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
assert queue.empty()
assert secret == await mock_kubernetes.read_namespaced_secret(
"gafaelfawr-secret", "mobu"
)
# Now add some labels and annotations.
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
service_token["metadata"]["labels"] = {"foo": "bar"}
service_token["metadata"]["annotations"] = {"one": "1", "two": "2"}
await mock_kubernetes.replace_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
service_token,
)
await queue.put(
WatchEvent(
event_type=WatchEventType.MODIFIED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
assert queue.empty()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
async def test_update_generation(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
"""Test that GafaelfawrServiceToken status changes don't trigger updates.
We always modify the GafaelfawrServiceToken on successful or failed
changes to its associated Secret, but that in turn triggers another MODIFY
watch message. We don't want to act on that MODIFY because, if the Secret
creation is failing, we could get into an infinite loop.
This test verifies that we observe the generation for which we last
processed an update and decline to attempt another update unless the
generation changes.
"""
service_token: Dict[str, Any] = {
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["admin:token"],
},
}
await mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
service_token,
)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
queue: Queue[WatchEvent] = Queue()
await queue.put(
WatchEvent(
event_type=WatchEventType.ADDED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
assert queue.empty()
secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr-secret", "mobu"
)
assert secret
# Modify the GafaelfawrServiceToken without changing the generation. The
# modify event should then be ignored.
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
service_token["spec"]["service"] = "other-mobu"
await mock_kubernetes.replace_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
service_token,
)
await queue.put(
WatchEvent(
event_type=WatchEventType.ADDED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
assert queue.empty()
assert secret == await mock_kubernetes.read_namespaced_secret(
"gafaelfawr-secret", "mobu"
)
# But if we send a delete and then an add with the same generation, it
# should be processed.
await queue.put(
WatchEvent(
event_type=WatchEventType.DELETED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await queue.put(
WatchEvent(
event_type=WatchEventType.ADDED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
assert queue.empty()
assert secret != await mock_kubernetes.read_namespaced_secret(
"gafaelfawr-secret", "mobu"
)
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
async def test_update_from_queue(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
service_token: Dict[str, Any] = {
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["admin:token"],
},
}
await mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
service_token,
)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
queue: Queue[WatchEvent] = Queue()
await queue.put(
WatchEvent(
event_type=WatchEventType.ADDED,
name="gafaelfawr-secret",
namespace="mobu",
generation=1,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
assert queue.empty()
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
service_token["metadata"]["generation"] = 2
service_token["spec"]["service"] = "other-mobu"
await mock_kubernetes.replace_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
service_token,
)
await queue.put(
WatchEvent(
event_type=WatchEventType.MODIFIED,
name="gafaelfawr-secret",
namespace="mobu",
generation=2,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
assert queue.empty()
# Deletion does nothing, but shouldn't prompt an error.
await queue.put(
WatchEvent(
event_type=WatchEventType.MODIFIED,
name="gafaelfawr-secret",
namespace="mobu",
generation=2,
)
)
await kubernetes_service.update_service_tokens_from_queue(
queue, exit_on_empty=True
)
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
assert queue.empty()
async def test_modify(
factory: ComponentFactory,
mock_kubernetes: MockKubernetesApi,
caplog: LogCaptureFixture,
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
token_service = factory.create_token_service()
# Valid secret but with a bogus token.
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": "bogus"},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("mobu", secret)
# Valid secret but with a nonexistent token.
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": token_as_base64(Token())},
metadata=V1ObjectMeta(
name="gafaelfawr",
namespace="nublado2",
labels={
"foo": "bar",
"other": "blah",
},
annotations={
"argocd.argoproj.io/compare-options": "IgnoreExtraneous",
"argocd.argoproj.io/sync-options": "Prune=false",
},
),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("nublado2", secret)
# Update the secrets. This should replace both with fresh secrets.
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
# Check the logging.
assert parse_log(caplog) == [
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "admin:token",
"token_username": "******",
},
{
"event": "Updated mobu/gafaelfawr-secret secret",
"scopes": ["admin:token"],
"severity": "info",
"service": "mobu",
},
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "",
"token_username": "******",
},
{
"event": "Updated nublado2/gafaelfawr secret",
"scopes": [],
"severity": "info",
"service": "nublado-hub",
},
]
# Replace one secret with a valid token for the wrong service.
async with factory.session.begin():
token = await token_service.create_token_from_admin_request(
AdminTokenRequest(
username="******",
token_type=TokenType.service,
scopes=["admin:token"],
),
TokenData.internal_token(),
ip_address=None,
)
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": token_as_base64(token)},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.replace_namespaced_secret(
"gafaelfawr-secret", "mobu", secret
)
# Replace the other token with a valid token with the wrong scopes.
async with factory.session.begin():
token = await token_service.create_token_from_admin_request(
AdminTokenRequest(
username="******",
token_type=TokenType.service,
scopes=["read:all"],
),
TokenData.internal_token(),
ip_address=None,
)
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": token_as_base64(token)},
metadata=V1ObjectMeta(name="gafaelfawr", namespace="nublado2"),
type="Opaque",
)
await mock_kubernetes.replace_namespaced_secret(
"gafaelfawr", "nublado2", secret
)
# Update the secrets. This should create new tokens for both.
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
nublado_secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)
# Finally, replace a secret with one with no token.
secret = V1Secret(
api_version="v1",
data={},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.replace_namespaced_secret(
"gafaelfawr-secret", "mobu", secret
)
# Update the secrets. This should create a new token for the first secret
# but not for the second.
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(
factory, mock_kubernetes, is_fresh=False
)
assert nublado_secret == await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)
async def test_create(
factory: ComponentFactory,
mock_kubernetes: MockKubernetesApi,
caplog: LogCaptureFixture,
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Secret was created",
"observedGeneration": 1,
"reason": StatusReason.Created.value,
"status": "True",
"type": "SecretCreated",
}
]
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"nublado2",
"gafaelfawrservicetokens",
"gafaelfawr",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Secret was created",
"observedGeneration": 45,
"reason": StatusReason.Created.value,
"status": "True",
"type": "SecretCreated",
}
]
assert parse_log(caplog) == [
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "admin:token",
"token_username": "******",
},
{
"event": "Created mobu/gafaelfawr-secret secret",
"scopes": ["admin:token"],
"severity": "info",
"service": "mobu",
},
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "",
"token_username": "******",
},
{
"event": "Created nublado2/gafaelfawr secret",
"scopes": [],
"severity": "info",
"service": "nublado-hub",
},
]
# Running creation again should not change anything.
caplog.clear()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
await kubernetes_service.update_service_tokens()
assert mock_kubernetes.get_all_objects_for_test("Secret") == objects
assert caplog.record_tuples == []
