def test_construct_dagster_k8s_job_with_user_defined_env():
@graph
def user_defined_k8s_env_tags_graph():
pass
user_defined_k8s_config = get_user_defined_k8s_config(
user_defined_k8s_env_tags_graph.to_job(
tags={
USER_DEFINED_K8S_CONFIG_KEY: {
"container_config": {
"env": [
{
"name": "ENV_VAR_1",
"value": "one"
},
{
"name": "ENV_VAR_2",
"value": "two"
},
{
"name": "DD_AGENT_HOST",
"valueFrom": {
"fieldRef": {
"fieldPath": "status.hostIP"
}
},
},
]
}
}
}).tags)
cfg = DagsterK8sJobConfig(
job_image="test/foo:latest",
dagster_home="/opt/dagster/dagster_home",
instance_config_map="some-instance-configmap",
)
job = construct_dagster_k8s_job(
cfg, ["foo", "bar"],
"job",
user_defined_k8s_config=user_defined_k8s_config).to_dict()
env = job["spec"]["template"]["spec"]["containers"][0]["env"]
env_mapping = remove_none_recursively(
{env_var["name"]: env_var
for env_var in env})
# Has DAGSTER_HOME and three additional env vars
assert len(env_mapping) == 4
assert env_mapping["ENV_VAR_1"]["value"] == "one"
assert env_mapping["ENV_VAR_2"]["value"] == "two"
assert env_mapping["DD_AGENT_HOST"]["value_from"] == {
"field_ref": {
"field_path": "status.hostIP"
}
}
def test_celery_queue_volumes(deployment_template: HelmTemplate):
volume_mounts = [
{
"name": "test-volume",
"mountPath":
"/opt/dagster/test_mount_path/volume_mounted_file.yaml",
"subPath": "volume_mounted_file.yaml",
},
]
volumes = [
{
"name": "test-volume",
"configMap": {
"name": "test-volume-configmap"
}
},
{
"name": "test-pvc",
"persistentVolumeClaim": {
"claimName": "my_claim",
"readOnly": False
}
},
]
repository = "repository"
tag = "tag"
helm_values = DagsterHelmValues.construct(runLauncher=RunLauncher(
type=RunLauncherType.CELERY,
config=RunLauncherConfig(
celeryK8sRunLauncher=CeleryK8sRunLauncherConfig.construct(
image=kubernetes.Image.construct(repository=repository,
tag=tag),
volumeMounts=volume_mounts,
volumes=volumes,
)),
))
celery_queue_deployments = deployment_template.render(helm_values)
assert len(celery_queue_deployments) == 1
mounts = celery_queue_deployments[0].spec.template.spec.containers[
0].volume_mounts
assert [remove_none_recursively(mount.to_dict()) for mount in mounts] == [
{
"mount_path": "/opt/dagster/dagster_home/dagster.yaml",
"name": "dagster-instance",
"sub_path": "dagster.yaml",
},
{
"mount_path": "/opt/dagster/dagster_home/celery-config.yaml",
"name": "dagster-celery",
"sub_path": "celery.yaml",
},
{
"mount_path":
"/opt/dagster/test_mount_path/volume_mounted_file.yaml",
"name": "test-volume",
"sub_path": "volume_mounted_file.yaml",
},
]
rendered_volumes = celery_queue_deployments[0].spec.template.spec.volumes
assert [
remove_none_recursively(volume.to_dict())
for volume in rendered_volumes
] == [
{
"config_map": {
"name": "release-name-dagster-instance"
},
"name": "dagster-instance"
},
{
"config_map": {
"name": "release-name-dagster-celery-dagster"
},
"name": "dagster-celery"
},
{
"name": "test-volume",
"config_map": {
"name": "test-volume-configmap"
}
},
{
"name": "test-pvc",
"persistent_volume_claim": {
"claim_name": "my_claim",
"read_only": False
},
},
]