def test_update_service_tokens(tmp_path: Path, config: Config,
mock_kubernetes: MockKubernetesApi) -> None:
asyncio.run(initialize_database(config, reset=True))
asyncio.run(
mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
{
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["admin:token"],
},
},
))
runner = CliRunner()
result = runner.invoke(main, ["update-service-tokens"])
assert result.exit_code == 0
assert mock_kubernetes.get_all_objects_for_test("Secret")
async def test_route_retire(
client: AsyncClient,
dossier: Dossier,
mock_kubernetes: MockKubernetesApi,
mock_kubernetes_watch: MockKubernetesWatch,
) -> None:
"""Retire is configured to not have any containers."""
r = await client.post("/moneypenny/users", json=dossier.dict())
assert r.status_code == 303
await wait_for_completion(client, dossier.username, mock_kubernetes,
mock_kubernetes_watch)
r = await client.get(f"/moneypenny/users/{dossier.username}")
assert r.status_code == 200
data = r.json()
assert data["status"] == "active"
r = await client.post(f"/moneypenny/users/{dossier.username}/retire",
json=dossier.dict())
assert r.status_code == 204
r = await client.get(f"/moneypenny/users/{dossier.username}")
assert r.status_code == 404
assert mock_kubernetes.get_all_objects_for_test("ConfigMap") == []
assert mock_kubernetes.get_all_objects_for_test("Pod") == []
async def assert_kubernetes_secrets_are_correct(
factory: ComponentFactory, mock: MockKubernetesApi, is_fresh: bool = True
) -> None:
token_service = factory.create_token_service()
# Get all of the GafaelfawrServiceToken custom objects.
service_tokens = mock.get_all_objects_for_test("GafaelfawrServiceToken")
# Calculate the expected secrets.
expected = [
V1Secret(
api_version="v1",
kind="Secret",
data={"token": ANY},
metadata=V1ObjectMeta(
name=t["metadata"]["name"],
namespace=t["metadata"]["namespace"],
annotations=t["metadata"].get("annotations", {}),
labels=t["metadata"].get("labels", {}),
owner_references=[
V1OwnerReference(
api_version="gafaelfawr.lsst.io/v1alpha1",
block_owner_deletion=True,
controller=True,
kind="GafaelfawrServiceToken",
name=t["metadata"]["name"],
uid=t["metadata"]["uid"],
),
],
),
type="Opaque",
)
for t in service_tokens
]
expected = sorted(
expected, key=lambda o: (o.metadata.namespace, o.metadata.name)
)
assert mock.get_all_objects_for_test("Secret") == expected
# Now check that every token in those secrets is correct.
for service_token in service_tokens:
name = service_token["metadata"]["name"]
namespace = service_token["metadata"]["namespace"]
secret = await mock.read_namespaced_secret(name, namespace)
data = await token_data_from_secret(token_service, secret)
assert data == TokenData(
token=data.token,
username=service_token["spec"]["service"],
token_type=TokenType.service,
scopes=service_token["spec"]["scopes"],
created=data.created,
expires=None,
name=None,
uid=None,
groups=None,
)
if is_fresh:
now = current_datetime()
assert now - timedelta(seconds=5) <= data.created <= now
async def test_mock(mock_kubernetes: MockKubernetesApi) -> None:
custom: Dict[str, Any] = {
"apiVersion": "gafaelfawr.lsst.io/v1alpha1",
"kind": "GafaelfawrServiceToken",
"metadata": {
"name": "gafaelfawr-secret",
"namespace": "mobu",
"generation": 1,
},
"spec": {
"service": "mobu",
"scopes": ["admin:token"],
},
}
secret = V1Secret(
api_version="v1",
kind="Secret",
data={"token": "bogus"},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
custom["metadata"]["namespace"],
"gafaelfawrservicetokens",
custom,
)
await mock_kubernetes.create_namespaced_secret(secret.metadata.namespace,
secret)
assert await mock_kubernetes.list_cluster_custom_object(
"gafaelfawr.lsst.io", "v1alpha1", "gafaelfawrservicetokens") == {
"items": [{
**custom, "metadata": {
**custom["metadata"], "uid": ANY
}
}]
}
assert mock_kubernetes.get_all_objects_for_test("Secret") == [secret]
def error(method: str, *args: Any) -> None:
assert method == "replace_namespaced_custom_object"
raise ValueError("some exception")
mock_kubernetes.error_callback = error
with pytest.raises(ValueError) as excinfo:
await mock_kubernetes.replace_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
custom["metadata"]["namespace"],
"gafaelfawrservicetokens",
"gafaelfawr-secret",
custom,
)
assert str(excinfo.value) == "some exception"
def test_update_service_tokens_error(
tmp_path: Path,
config: Config,
mock_kubernetes: MockKubernetesApi,
caplog: LogCaptureFixture,
) -> None:
asyncio.run(initialize_database(config, reset=True))
caplog.clear()
def error_callback(method: str, *args: Any) -> None:
if method == "list_cluster_custom_object":
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback
runner = CliRunner()
result = runner.invoke(main, ["update-service-tokens"])
assert result.exit_code == 1
assert parse_log(caplog) == [
{
"event": "Unable to list GafaelfawrServiceToken objects",
"error": "Kubernetes API error: (500)\nReason: Some error\n",
"severity": "error",
},
{
"error": "Kubernetes API error: (500)\nReason: Some error\n",
"event": "Failed to update service token secrets",
"severity": "error",
},
]
async def test_errors(
client: AsyncClient,
dossier: Dossier,
mock_kubernetes: MockKubernetesApi,
mock_kubernetes_watch: MockKubernetesWatch,
) -> None:
"""Test resource creation when Kubernetes calls fail."""
count = 0
def error_callback(method: str, *args: Any, **kwargs: Any) -> None:
nonlocal count
count += 1
if method == "create_namespaced_config_map" and count <= 1:
raise ApiException(status=409, reason="Resoure conflict")
elif method == "create_namespaced_config_map" and count <= 6:
raise ApiException(status=500, reason="Other error")
elif method == "create_namespaced_pod" and count <= 3:
raise ApiException(status=409, reason="Resource conflict")
elif method == "create_namespaced_pod" and count <= 4:
raise ApiException(status=500, reason="Other error")
mock_kubernetes.error_callback = error_callback
r = await client.post("/moneypenny/users", json=dossier.dict())
assert r.status_code == 303
r = await client.get(f"/moneypenny/users/{dossier.username}")
assert r.status_code == 200
data = r.json()
assert data["status"] == "commissioning"
await wait_for_completion(client, dossier.username, mock_kubernetes,
mock_kubernetes_watch)
async def test_errors_replace_read(
factory: ComponentFactory, mock_kubernetes: MockKubernetesApi
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
token_service = factory.create_token_service()
# Create a secret that should exist but has an invalid token.
secret = V1Secret(
api_version="v1",
data={"token": token_as_base64(Token())},
metadata=V1ObjectMeta(name="gafaelfawr-secret", namespace="mobu"),
type="Opaque",
)
await mock_kubernetes.create_namespaced_secret("mobu", secret)
# Simulate some errors. The callback function takes the operation and the
# secret name.
def error_callback_replace(method: str, *args: Any) -> None:
if method in ("replace_namespaced_secret"):
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback_replace
# Now run the synchronization. The secret should be left unchanged, but
# we should still create the missing nublado2 secret.
await kubernetes_service.update_service_tokens()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
assert secret in objects
good_secret = await mock_kubernetes.read_namespaced_secret(
"gafaelfawr", "nublado2"
)
assert await token_data_from_secret(token_service, good_secret)
# We should have also updated the status of the parent custom object.
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Kubernetes API error: (500)\nReason: Some error\n",
"observedGeneration": 1,
"reason": StatusReason.Failed.value,
"status": "False",
"type": "SecretCreated",
}
]
# Try again, but simulating an error in retrieving a secret.
def error_callback_read(method: str, *args: Any) -> None:
if method == "read_namespaced_secret":
raise ApiException(status=500, reason="Some error")
mock_kubernetes.error_callback = error_callback_read
# Now run the synchronization. As before, the secret should be left
# unchanged, and the good secret should also be left unchanged.
await kubernetes_service.update_service_tokens()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
assert secret in objects
async def test_create(
factory: ComponentFactory,
mock_kubernetes: MockKubernetesApi,
caplog: LogCaptureFixture,
) -> None:
await create_test_service_tokens(mock_kubernetes)
kubernetes_service = factory.create_kubernetes_service(MagicMock())
await kubernetes_service.update_service_tokens()
await assert_kubernetes_secrets_are_correct(factory, mock_kubernetes)
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"mobu",
"gafaelfawrservicetokens",
"gafaelfawr-secret",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Secret was created",
"observedGeneration": 1,
"reason": StatusReason.Created.value,
"status": "True",
"type": "SecretCreated",
}
]
service_token = await mock_kubernetes.get_namespaced_custom_object(
"gafaelfawr.lsst.io",
"v1alpha1",
"nublado2",
"gafaelfawrservicetokens",
"gafaelfawr",
)
assert service_token["status"]["conditions"] == [
{
"lastTransitionTime": ANY,
"message": "Secret was created",
"observedGeneration": 45,
"reason": StatusReason.Created.value,
"status": "True",
"type": "SecretCreated",
}
]
assert parse_log(caplog) == [
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "admin:token",
"token_username": "******",
},
{
"event": "Created mobu/gafaelfawr-secret secret",
"scopes": ["admin:token"],
"severity": "info",
"service": "mobu",
},
{
"event": "Created new service token",
"key": ANY,
"severity": "info",
"token_scope": "",
"token_username": "******",
},
{
"event": "Created nublado2/gafaelfawr secret",
"scopes": [],
"severity": "info",
"service": "nublado-hub",
},
]
# Running creation again should not change anything.
caplog.clear()
objects = mock_kubernetes.get_all_objects_for_test("Secret")
await kubernetes_service.update_service_tokens()
assert mock_kubernetes.get_all_objects_for_test("Secret") == objects
assert caplog.record_tuples == []
async def test_route_commission(
client: AsyncClient,
dossier: Dossier,
mock_kubernetes: MockKubernetesApi,
mock_kubernetes_watch: MockKubernetesWatch,
) -> None:
r = await client.post("/moneypenny/users", json=dossier.dict())
assert r.status_code == 303
assert r.headers["Location"] == url_for(f"users/{dossier.username}")
r = await client.get(f"/moneypenny/users/{dossier.username}")
assert r.status_code == 200
data = r.json()
assert data == {
"username": dossier.username,
"status": "commissioning",
"last_changed": ANY,
"uid": dossier.uid,
"groups": [g.dict() for g in dossier.groups],
}
# Requesting the exact same thing again even though it's not complete is
# fine and produces the same redirect.
r = await client.post("/moneypenny/users", json=dossier.dict())
assert r.status_code == 303
assert r.headers["Location"] == url_for(f"users/{dossier.username}")
assert mock_kubernetes.get_all_objects_for_test("ConfigMap") == [
V1ConfigMap(
metadata=V1ObjectMeta(
name=f"{dossier.username}-cm",
namespace="default",
owner_references=[
V1OwnerReference(
api_version="v1",
kind="Pod",
name="moneypenny-78547dcf97-9xqq8",
uid="00386592-214f-40c5-88e1-b9657d53a7c6",
)
],
),
data={
"dossier.json":
json.dumps(dossier.dict(), sort_keys=True, indent=4)
},
)
]
assert mock_kubernetes.get_all_objects_for_test("Pod") == [
V1Pod(
metadata=V1ObjectMeta(
name=f"{dossier.username}-pod",
namespace="default",
owner_references=[
V1OwnerReference(
api_version="v1",
kind="Pod",
name="moneypenny-78547dcf97-9xqq8",
uid="00386592-214f-40c5-88e1-b9657d53a7c6",
)
],
),
spec=V1PodSpec(
automount_service_account_token=False,
containers=[{
"name":
"farthing",
"image":
"lsstsqre/farthing",
"securityContext": {
"runAsUser": 1000,
"runAsNonRootUser": True,
"allowPrivilegeEscalation": False,
},
"volumeMounts": [
{
"mountPath": "/homedirs",
"name": "homedirs",
},
{
"mountPath": "/opt/dossier",
"name": f"dossier-{dossier.username}-vol",
"readOnly": True,
},
],
}],
image_pull_secrets=[],
init_containers=[],
restart_policy="OnFailure",
security_context=V1PodSecurityContext(run_as_group=1000,
run_as_user=1000),
volumes=[
{
"name": "homedirs",
"nfs": {
"server": "10.10.10.10",
"path": "/homedirs",
},
},
V1Volume(
name=f"dossier-{dossier.username}-vol",
config_map=V1ConfigMapVolumeSource(
default_mode=0o644,
name=f"{dossier.username}-cm",
),
),
],
),
status=V1PodStatus(phase="Running"),
)
]
await wait_for_completion(client, dossier.username, mock_kubernetes,
mock_kubernetes_watch)