async def _wait_for_service_account_token(self, spawner: KubeSpawner,
name: str,
namespace: str) -> bool:
"""Waits for a service account to spawn an associated token.
Returns
-------
done : `bool`
`True` once the secret exists, `False` otherwise (so it can be
called from ``exponential_backoff``)
"""
api = shared_client("CoreV1Api")
try:
service_account = await asyncio.wait_for(
api.read_namespaced_service_account(name, namespace),
spawner.k8s_api_request_timeout,
)
if not service_account.secrets:
return False
secret_name = service_account.secrets[0].name
secret = await asyncio.wait_for(
api.read_namespaced_secret(secret_name, namespace),
spawner.k8s_api_request_timeout,
)
return secret.metadata.name == secret_name
except asyncio.TimeoutError:
return False
except ApiException as e:
if e.status == 404:
self.log.debug("Waiting for secret for service account {name}")
return False
raise
def kube_client(request, kube_ns):
"""fixture for the Kubernetes client object.
skips test that require kubernetes if kubernetes cannot be contacted
"""
load_kube_config()
client = shared_client("CoreV1Api")
try:
namespaces = client.list_namespace(_request_timeout=3)
except Exception as e:
pytest.skip("Kubernetes not found: %s" % e)
if not any(ns.metadata.name == kube_ns for ns in namespaces.items):
print("Creating namespace %s" % kube_ns)
client.create_namespace(V1Namespace(metadata=dict(name=kube_ns)))
else:
print("Using existing namespace %s" % kube_ns)
# delete the test namespace when we finish
def cleanup_namespace():
client.delete_namespace(kube_ns, body={}, grace_period_seconds=0)
for i in range(3):
try:
ns = client.read_namespace(kube_ns)
except ApiException as e:
if e.status == 404:
return
else:
raise
else:
print("waiting for %s to delete" % kube_ns)
time.sleep(1)
# request.addfinalizer(cleanup_namespace)
return client
def get_custom_resources(self, namespace, plural):
api_instance = shared_client('CustomObjectsApi')
group = 'primehub.io'
version = 'v1alpha1'
api_response = api_instance.list_namespaced_custom_object(
group, version, namespace, plural)
return api_response['items']
async def test_multi_namespace_spawn():
# We cannot use the fixtures, because they assume the standard
# namespace and client for that namespace.
spawner = KubeSpawner(
hub=Hub(),
user=MockUser(),
config=Config(),
enable_user_namespaces=True,
)
# empty spawner isn't running
status = await spawner.poll()
assert isinstance(status, int)
# get a client
kube_ns = spawner.namespace
load_kube_config()
client = shared_client('CoreV1Api')
# the spawner will create the namespace on its own.
# Wrap in a try block so we clean up the namespace.
saved_exception = None
try:
# start the spawner
await spawner.start()
# verify the pod exists
pods = client.list_namespaced_pod(kube_ns).items
pod_names = [p.metadata.name for p in pods]
assert "jupyter-%s" % spawner.user.name in pod_names
# verify poll while running
status = await spawner.poll()
assert status is None
# stop the pod
await spawner.stop()
# verify pod is gone
pods = client.list_namespaced_pod(kube_ns).items
pod_names = [p.metadata.name for p in pods]
assert "jupyter-%s" % spawner.user.name not in pod_names
# verify exit status
status = await spawner.poll()
assert isinstance(status, int)
except Exception as saved_exception:
pass # We will raise after namespace removal
# remove namespace
client.delete_namespace(kube_ns, body={})
if saved_exception is not None:
raise saved_exception
async def _wait_for_namespace_deletion(self, spawner: KubeSpawner) -> bool:
"""Waits for the user's namespace to be deleted.
If the namespace exists but has not been marked for deletion, try to
delete it. If we're spawning a new lab while the namespace still
exists, that means something has gone wrong with the user's lab and
there's nothing salvagable.
Returns
-------
done : `bool`
`True` if the namespace has been deleted, `False` if it still
exists
"""
api = shared_client("CoreV1Api")
try:
ns_name = spawner.namespace
# Note that this is not safe to run if you aren't using
# user namespaces. Check that:
assert spawner.enable_user_namespaces
# Let's do a further check for paranoia. We will assume that
# a user namespace will be of the form <hub-namespace>-<something>
# This is true for Rubin user namespaces, but might not be
# universally.
#
# If opening the ns_path file fails, it means we are not running
# in a namespace with service accounts enabled, in which case
# we definitely want to let the exception crash the process.
ns_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
with open(ns_path) as f:
hub_ns = f.read().strip()
assert ns_name.startswith(f"{hub_ns}-")
namespace = await asyncio.wait_for(
api.read_namespace(ns_name),
spawner.k8s_api_request_timeout,
)
if namespace.status.phase != "Terminating":
self.log.info(f"Deleting namespace {ns_name}")
await asyncio.wait_for(
api.delete_namespace(ns_name),
spawner.k8s_api_request_timeout,
)
return False
except asyncio.TimeoutError:
return False
except ApiException as e:
if e.status == 404:
return True
raise
def _ensure_namespaced_service_account(self):
"""Create a service account with role and rolebinding to allow it
to manipulate pods in the namespace."""
namespace = self.get_user_namespace()
account = self.service_account
svcacct, role, rolebinding = self._create_namespaced_account_objects()
if not svcacct:
self.log.info("Service account not defined.")
return
try:
self.api.create_namespaced_service_account(namespace=namespace,
body=svcacct)
except ApiException as e:
if e.status != 409:
self.log.exception("Create service account '%s' " % account +
"in namespace '%s' " % namespace +
"failed: %s" % str(e))
raise
else:
self.log.info("Service account '%s' " % account +
"in namespace '%s' already exists." % namespace)
if not self.rbacapi:
self.log.info("Creating RBAC API Client.")
self.rbacapi = shared_client('RbacAuthorizationV1Api')
try:
self.rbacapi.create_namespaced_role(namespace, role)
except ApiException as e:
if e.status != 409:
self.log.exception("Create role '%s' " % account +
"in namespace '%s' " % namespace +
"failed: %s" % str(e))
raise
else:
self.log.info("Role '%s' " % account +
"already exists in namespace '%s'." % namespace)
try:
self.rbacapi.create_namespaced_role_binding(namespace, rolebinding)
except ApiException as e:
if e.status != 409:
self.log.exception(
"Create rolebinding '%s'" % account +
"in namespace '%s' " % namespace + "failed: %s", str(e))
raise
else:
self.log.info("Rolebinding '%s' " % account +
"already exists in '%s'." % namespace)
def kube_client(request, kube_ns):
"""fixture for the Kubernetes client object.
skips test that require kubernetes if kubernetes cannot be contacted
- Ensures kube_ns namespace exists
- Hooks up kubernetes events and logs to pytest capture
- Cleans up kubernetes namespace on exit
"""
load_kube_config()
client = shared_client("CoreV1Api")
try:
namespaces = client.list_namespace(_request_timeout=3)
except Exception as e:
pytest.skip("Kubernetes not found: %s" % e)
if not any(ns.metadata.name == kube_ns for ns in namespaces.items):
print("Creating namespace %s" % kube_ns)
client.create_namespace(V1Namespace(metadata=dict(name=kube_ns)))
else:
print("Using existing namespace %s" % kube_ns)
# begin streaming all logs and events in our test namespace
t = Thread(target=watch_kubernetes, args=(client, kube_ns), daemon=True)
t.start()
# delete the test namespace when we finish
def cleanup_namespace():
client.delete_namespace(kube_ns, body={}, grace_period_seconds=0)
for i in range(3):
try:
ns = client.read_namespace(kube_ns)
except ApiException as e:
if e.status == 404:
return
else:
raise
else:
print("waiting for %s to delete" % kube_ns)
time.sleep(1)
# allow opting out of namespace cleanup, for post-mortem debugging
if not os.environ.get("KUBESPAWNER_DEBUG_NAMESPACE"):
request.addfinalizer(cleanup_namespace)
return client
def kube_client(request, kube_ns):
"""fixture for the Kubernetes client object.
skips test that require kubernetes if kubernetes cannot be contacted
"""
load_kube_config()
client = shared_client('CoreV1Api')
try:
namespaces = client.list_namespace(_request_timeout=3)
except Exception as e:
pytest.skip("Kubernetes not found: %s" % e)
if not any(ns.metadata.name == kube_ns for ns in namespaces.items):
print("Creating namespace %s" % kube_ns)
client.create_namespace(V1Namespace(metadata=dict(name=kube_ns)))
else:
print("Using existing namespace %s" % kube_ns)
# delete the test namespace when we finish
request.addfinalizer(lambda: client.delete_namespace(kube_ns, {}))
return client
async def test_multi_namespace_spawn():
# We cannot use the fixtures, because they assume the standard
# namespace and client for that namespace.
spawner = MultiNamespaceKubeSpawner(hub=Hub(),
user=MockUser(),
config=Config())
# empty spawner isn't running
status = await spawner.poll()
assert isinstance(status, int)
# get a client
kube_ns = spawner.namespace
load_kube_config()
client = shared_client('CoreV1Api')
# the spawner will create the namespace on its own.
# start the spawner
await spawner.start()
# verify the pod exists
pods = client.list_namespaced_pod(kube_ns).items
pod_names = [p.metadata.name for p in pods]
assert "jupyter-%s" % spawner.user.name in pod_names
# verify poll while running
status = await spawner.poll()
assert status is None
# stop the pod
await spawner.stop()
# verify pod is gone
pods = client.list_namespaced_pod(kube_ns).items
pod_names = [p.metadata.name for p in pods]
assert "jupyter-%s" % spawner.user.name not in pod_names
# verify exit status
status = await spawner.poll()
assert isinstance(status, int)
# remove namespace
client.delete_namespace(kube_ns, body={})
async def _create_kubernetes_resources(self, spawner: KubeSpawner,
options: SelectedOptions) -> None:
custom_api = shared_client("CustomObjectsApi")
template_values = await self._build_template_values(spawner, options)
# Generate the list of additional user resources from the template.
t = Template(self.nublado_config.user_resources_template)
templated_user_resources = t.render(template_values)
self.log.debug("Generated user resources:")
self.log.debug(templated_user_resources)
resources = self.yaml.load(templated_user_resources)
# Add in the standard labels and annotations common to every resource
# and create the resources.
service_account = None
for resource in resources:
if "metadata" not in resource:
resource["metadata"] = {}
resource["metadata"]["annotations"] = spawner.extra_annotations
resource["metadata"]["labels"] = spawner.extra_labels
# Custom resources cannot be created by create_from_dict:
# https://github.com/kubernetes-client/python/issues/740
#
# Detect those from the apiVersion field and handle them
# specially.
api_version = resource["apiVersion"]
if "." in api_version and ".k8s.io/" not in api_version:
crd_parser = CRDParser.from_crd_body(resource)
await asyncio.wait_for(
custom_api.create_namespaced_custom_object(
body=resource,
group=crd_parser.group,
version=crd_parser.version,
namespace=spawner.namespace,
plural=crd_parser.plural,
),
spawner.k8s_api_request_timeout,
)
else:
await asyncio.wait_for(
create_from_dict(spawner.api.api_client, resource),
spawner.k8s_api_request_timeout,
)
# If this was a service account, note its name.
if resource["kind"] == "ServiceAccount":
service_account = resource["metadata"]["name"]
# Construct the lab environment ConfigMap. This is constructed from
# configuration settings and doesn't use a resource template like
# other resources. This has to be done last, becuase the namespace is
# created from the user resources template.
await self._create_lab_environment_configmap(spawner, template_values)
# Wait for the service account to generate a token before proceeding.
# Otherwise, we may try to create the pod before the service account
# token exists and Kubernetes will object.
if service_account:
await exponential_backoff(
partial(
self._wait_for_service_account_token,
spawner,
service_account,
spawner.namespace,
),
f"Service account {service_account} has no token",
timeout=spawner.k8s_api_request_retry_timeout,
)