def get_backup_metrics_exporter(
owner_references: Optional[List[V1OwnerReference]],
name: str,
labels: LabelType,
http_port: int,
prometheus_port: int,
backup_aws: Dict[str, Any],
image_pull_secrets: Optional[List[V1LocalObjectReference]],
has_ssl: bool,
) -> V1Deployment:
env = [
V1EnvVar(name="EXPORTER_PORT", value=str(prometheus_port)),
V1EnvVar(name="PYTHONWARNINGS",
value="ignore:Unverified HTTPS request"),
V1EnvVar(name="REPOSITORY_PREFIX", value="system_backup"),
] + get_backup_env(name, http_port, backup_aws, has_ssl)
return V1Deployment(
metadata=V1ObjectMeta(
name=f"backup-metrics-{name}",
labels=labels,
owner_references=owner_references,
),
spec=V1DeploymentSpec(
replicas=1,
selector=V1LabelSelector(match_labels={
LABEL_COMPONENT: "backup",
LABEL_NAME: name
}),
template=V1PodTemplateSpec(
metadata=V1ObjectMeta(
annotations={
"prometheus.io/port": str(prometheus_port),
"prometheus.io/scrape": "true",
},
labels=labels,
name=f"backup-metrics-{name}",
),
spec=V1PodSpec(
containers=[
V1Container(
command=["metrics-exporter", "-vv"],
env=env,
image=config.CLUSTER_BACKUP_IMAGE,
name="metrics-exporter",
ports=[
V1ContainerPort(
container_port=prometheus_port,
name="backup-metrics",
)
],
)
],
image_pull_secrets=image_pull_secrets,
restart_policy="Always",
),
),
),
)
def get_webhook_env():
if (config.WEBHOOK_URL is not None and config.WEBHOOK_USERNAME is not None
and config.WEBHOOK_PASSWORD is not None):
return [
V1EnvVar(name="WEBHOOK_URL", value=config.WEBHOOK_URL),
V1EnvVar(name="AUTH_USER", value=config.WEBHOOK_USERNAME),
V1EnvVar(name="AUTH_PASS", value=config.WEBHOOK_PASSWORD),
]
return []
def get_statefulset_crate_env(
node_spec: Dict[str, Any],
jmx_port: int,
prometheus_port: int,
ssl: Optional[Dict[str, Any]],
) -> List[V1EnvVar]:
crate_env = [
V1EnvVar(
name="CRATE_HEAP_SIZE",
value=str(
int(
bitmath.parse_string_unsafe(
node_spec["resources"]["memory"]).bytes *
node_spec["resources"]["heapRatio"])),
),
V1EnvVar(
name="CRATE_JAVA_OPTS",
value=" ".join(
get_statefulset_crate_env_java_opts(jmx_port,
prometheus_port)),
),
]
if ssl is not None:
crate_env.extend([
V1EnvVar(
name="KEYSTORE_KEY_PASSWORD",
value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(
key=ssl["keystoreKeyPassword"]["secretKeyRef"]["key"],
name=ssl["keystoreKeyPassword"]["secretKeyRef"]["name"],
)),
),
V1EnvVar(
name="KEYSTORE_PASSWORD",
value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(
key=ssl["keystorePassword"]["secretKeyRef"]["key"],
name=ssl["keystorePassword"]["secretKeyRef"]["name"],
)),
),
])
return crate_env
def get_backup_env(
name: str, http_port: int, backup_aws: Dict[str, Any], has_ssl: bool,
) -> List[V1EnvVar]:
schema = "https" if has_ssl else "http"
return [
V1EnvVar(name="BASE_PATH", value=backup_aws.get("basePath", "")),
V1EnvVar(
name="BUCKET",
value_from=V1EnvVarSource(
secret_key_ref=V1SecretKeySelector(
key=backup_aws["bucket"]["secretKeyRef"]["key"],
name=backup_aws["bucket"]["secretKeyRef"]["name"],
),
),
),
V1EnvVar(name="HOSTS", value=f"{schema}://crate-{name}:{http_port}"),
V1EnvVar(
name="PASSWORD",
value_from=V1EnvVarSource(
secret_key_ref=V1SecretKeySelector(
key="password", name=f"user-system-{name}",
),
),
),
V1EnvVar(
name="REGION",
value_from=V1EnvVarSource(
secret_key_ref=V1SecretKeySelector(
key=backup_aws["region"]["secretKeyRef"]["key"],
name=backup_aws["region"]["secretKeyRef"]["name"],
),
),
),
V1EnvVar(name="USERNAME", value=SYSTEM_USERNAME),
]
def get_backup_env(name: str, http_port: int, backup_aws: Dict[str, Any],
has_ssl: bool) -> List[V1EnvVar]:
schema = "https" if has_ssl else "http"
return [
# The base path is here for backwards-compatibility and
# is not used by newer versions of the backup CronJob,
# in favour of the namespace.
V1EnvVar(name="BASE_PATH", value=backup_aws.get("basePath", "")),
V1EnvVar(
name="NAMESPACE",
value_from=V1EnvVarSource(field_ref=V1ObjectFieldSelector(
api_version="v1",
field_path="metadata.namespace",
)),
),
V1EnvVar(
name="BUCKET",
value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(
key=backup_aws["bucket"]["secretKeyRef"]["key"],
name=backup_aws["bucket"]["secretKeyRef"]["name"],
), ),
),
V1EnvVar(name="HOSTS", value=f"{schema}://crate-{name}:{http_port}"),
V1EnvVar(
name="PASSWORD",
value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(
key="password", name=f"user-system-{name}"), ),
),
V1EnvVar(
name="REGION",
value_from=V1EnvVarSource(secret_key_ref=V1SecretKeySelector(
key=backup_aws["region"]["secretKeyRef"]["key"],
name=backup_aws["region"]["secretKeyRef"]["name"],
), ),
),
V1EnvVar(name="USERNAME", value=SYSTEM_USERNAME),
]
def get_backup_cronjob(
owner_references: Optional[List[V1OwnerReference]],
name: str,
labels: LabelType,
http_port: int,
backup_aws: Dict[str, Any],
image_pull_secrets: Optional[List[V1LocalObjectReference]],
has_ssl: bool,
) -> V1beta1CronJob:
env = [
V1EnvVar(
name="AWS_ACCESS_KEY_ID",
value_from=V1EnvVarSource(
secret_key_ref=V1SecretKeySelector(
key=backup_aws["accessKeyId"]["secretKeyRef"]["key"],
name=backup_aws["accessKeyId"]["secretKeyRef"]["name"],
),
),
),
V1EnvVar(
name="AWS_SECRET_ACCESS_KEY",
value_from=V1EnvVarSource(
secret_key_ref=V1SecretKeySelector(
key=backup_aws["secretAccessKey"]["secretKeyRef"]["key"],
name=backup_aws["secretAccessKey"]["secretKeyRef"]["name"],
),
),
),
V1EnvVar(name="CLUSTER_ID", value=name),
V1EnvVar(name="PYTHONWARNINGS", value="ignore:Unverified HTTPS request"),
V1EnvVar(name="REPOSITORY_PREFIX", value="system_backup"),
] + get_backup_env(name, http_port, backup_aws, has_ssl)
return V1beta1CronJob(
metadata=V1ObjectMeta(
name=f"create-snapshot-{name}",
labels=labels,
owner_references=owner_references,
),
spec=V1beta1CronJobSpec(
concurrency_policy="Forbid",
failed_jobs_history_limit=1,
job_template=V1beta1JobTemplateSpec(
metadata=V1ObjectMeta(labels=labels, name=f"create-snapshot-{name}"),
spec=V1JobSpec(
template=V1PodTemplateSpec(
metadata=V1ObjectMeta(
labels=labels, name=f"create-snapshot-{name}",
),
spec=V1PodSpec(
containers=[
V1Container(
command=["backup", "-vv"],
env=env,
image=config.CLUSTER_BACKUP_IMAGE,
name="backup",
)
],
image_pull_secrets=image_pull_secrets,
restart_policy="Never",
),
),
),
),
schedule=backup_aws["cron"],
successful_jobs_history_limit=1,
),
)
async def test_create_kubernetes_resources(
kubernetes_api_mock: KubernetesApiMock, ) -> None:
spawner = Mock(spec=KubeSpawner)
spawner.k8s_api_request_timeout = 3
spawner.k8s_api_request_retry_timeout = 30
spawner.namespace = "nublado2-someuser"
spawner.extra_annotations = {
"argocd.argoproj.io/compare-options": "IgnoreExtraneous",
"argocd.argoproj.io/sync-options": "Prune=false",
}
spawner.extra_labels = {
"hub.jupyter.org/network-access-hub": "true",
"argocd.argoproj.io/instance": "nublado-users",
}
spawner._make_create_resource_request = kubernetes_api_mock.create_object
spawner.hub = Mock()
spawner.hub.base_url = "/nb/hub/"
spawner.user = Mock(spec=User)
spawner.user.name = "someuser"
spawner.api = kubernetes_api_mock
auth_state = {
"token": "user-token",
"uid": 1234,
"groups": [{
"name": "foo",
"id": 1235
}, {
"name": "bar",
"id": 4567
}],
}
pod_manifest = V1Pod(
api_version="v1",
kind="Pod",
metadata=V1ObjectMeta(
name="user-pod",
namespace=spawner.namespace,
),
spec=V1PodSpec(containers=[
V1Container(
name="container",
command=["run-something"],
env=[V1EnvVar(name="FOO", value="BAR")],
image="blah:latest",
)
], ),
)
if sys.version_info < (3, 8):
spawner.get_pod_manifest.return_value = asyncio.Future()
spawner.get_pod_manifest.return_value.set_result(pod_manifest)
spawner.user.get_auth_state.return_value = asyncio.Future()
spawner.user.get_auth_state.return_value.set_result(auth_state)
else:
spawner.get_pod_manifest.return_value = pod_manifest
spawner.user.get_auth_state.return_value = auth_state
options = Mock(spec=SelectedOptions)
options.debug = "true"
options.clear_dotlocal = "true"
options.image_info = ImageInfo(
reference="registry.hub.docker.com/lsstsqre/sciplat-lab:w_2021_13",
display_name="blah blah blah",
digest="sha256:123456789abcdef",
)
resource_manager = ResourceManager()
await resource_manager._create_kubernetes_resources(spawner, options)
assert sorted(
kubernetes_api_mock.objects,
key=lambda o: (o["kind"], o["metadata"]["name"]),
) == [
{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "dask",
"namespace": spawner.namespace,
"annotations": spawner.extra_annotations,
"labels": spawner.extra_labels,
},
"data": {
"dask_worker.yml":
f"""\
apiVersion: v1
kind: Pod
metadata:
namespace: {spawner.namespace}
spec:
containers:
- command:
- run-something
env:
- name: FOO
value: BAR
- name: DASK_WORKER
value: 'TRUE'
image: blah:latest
name: container
"""
},
},
{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "group",
"namespace": spawner.namespace,
"annotations": spawner.extra_annotations,
"labels": spawner.extra_labels,
},
"data": {
"group": ("someuser:x:1234:\n"
"foo:x:1235:someuser\n"
"bar:x:4567:someuser\n")
},
},
{
"apiVersion": "v1",
"kind": "ConfigMap",
"metadata": {
"name": "lab-environment",
"namespace": spawner.namespace,
"annotations": spawner.extra_annotations,
"labels": spawner.extra_labels,
},
"data": {
"EXTERNAL_INSTANCE_URL": "https://data.example.com/",
"FIREFLY_ROUTE": "/portal/app",
"HUB_ROUTE": "/nb/hub/",
"EXTERNAL_GROUPS": "foo:1235,bar:4567",
"EXTERNAL_UID": "1234",
"ACCESS_TOKEN": "user-token",
"IMAGE_DIGEST": "sha256:123456789abcdef",
"IMAGE_DESCRIPTION": "blah blah blah",
"CLEAR_DOTLOCAL": "true",
"DEBUG": "true",
},
},
]
assert sorted(
kubernetes_api_mock.custom,
key=lambda o: (o["kind"], o["metadata"]["name"]),
) == [{
"apiVersion": "ricoberger.de/v1alpha1",
"kind": "VaultSecret",
"metadata": {
"name": "butler-secret",
"namespace": spawner.namespace,
"annotations": spawner.extra_annotations,
"labels": spawner.extra_labels,
},
"spec": {
"path": "k8s_operator/data/butler",
"type": "Opaque",
},
}]