def create_simple_user_deployment(name: str) -> UserDeployment:
return UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}", tag="tag1", pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
)
def create_complex_user_deployment(name: str) -> UserDeployment:
return UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}",
tag="tag1",
pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
annotations=kubernetes.Annotations.parse_obj({
"annotation_1":
"an_annotation_1",
"annotation_2":
"an_annotation_2"
}),
nodeSelector=kubernetes.NodeSelector.parse_obj({
"node_selector_1":
"node_type_1",
"node_selector_2":
"node_type_2"
}),
affinity=kubernetes.Affinity.parse_obj({
"nodeAffinity": {
"requiredDuringSchedulingIgnoredDuringExecution": {
"nodeSelectorTerms": [{
"matchExpressions": [{
"key": "kubernetes.io/e2e-az-name",
"operator": "In",
"values": ["e2e-az1", "e2e-az2"],
}]
}]
}
}
}),
tolerations=kubernetes.Tolerations.parse_obj([{
"key": "key1",
"operator": "Exists",
"effect": "NoSchedule"
}]),
podSecurityContext=kubernetes.PodSecurityContext.parse_obj({
"runAsUser":
1000,
"runAsGroup":
3000
}),
securityContext=kubernetes.SecurityContext.parse_obj({
"runAsUser": 1000,
"runAsGroup": 3000
}),
resources=kubernetes.Resources.parse_obj({
"requests": {
"memory": "64Mi",
"cpu": "250m"
},
"limits": {
"memory": "128Mi",
"cpu": "500m"
},
}),
)
def test_user_deployment_secrets_and_configmaps(
template: HelmTemplate, include_config_in_launched_runs: bool
):
name = "foo"
secrets = [{"name": "my-secret"}, {"name": "my-other-secret"}]
configmaps = [{"name": "my-configmap"}, {"name": "my-other-configmap"}]
deployment = UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}", tag="tag1", pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
envConfigMaps=[
kubernetes.ConfigMapEnvSource.construct(None, **configmap) for configmap in configmaps
],
envSecrets=[kubernetes.SecretEnvSource.construct(None, **secret) for secret in secrets],
includeConfigInLaunchedRuns=UserDeploymentIncludeConfigInLaunchedRuns(
enabled=include_config_in_launched_runs
),
)
helm_values = DagsterHelmValues.construct(
dagsterUserDeployments=UserDeployments(
enabled=True,
enableSubchart=True,
deployments=[deployment],
)
)
user_deployments = template.render(helm_values)
assert len(user_deployments) == 1
if include_config_in_launched_runs:
container_context = user_deployments[0].spec.template.spec.containers[0].env[2]
assert container_context.name == "DAGSTER_CLI_API_GRPC_CONTAINER_CONTEXT"
assert json.loads(container_context.value) == {
"k8s": {
"image_pull_policy": "Always",
"env_secrets": ["my-secret", "my-other-secret"],
"env_config_maps": [
"release-name-dagster-user-deployments-foo-user-env",
"my-configmap",
"my-other-configmap",
],
}
}
else:
_assert_no_container_context(user_deployments[0])
def create_simple_user_deployment(
name: str,
include_config_in_launched_runs: Optional[bool] = False
) -> UserDeployment:
return UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}",
tag="tag1",
pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
includeConfigInLaunchedRuns=(UserDeploymentIncludeConfigInLaunchedRuns(
enabled=include_config_in_launched_runs)
if include_config_in_launched_runs != None
else None),
)
def test_user_deployment_labels(template: HelmTemplate, include_config_in_launched_runs: bool):
name = "foo"
labels = {"my-label-key": "my-label-val", "my-other-label-key": "my-other-label-val"}
deployment = UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}", tag="tag1", pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
labels=labels,
includeConfigInLaunchedRuns=UserDeploymentIncludeConfigInLaunchedRuns(
enabled=include_config_in_launched_runs
),
)
helm_values = DagsterHelmValues.construct(
dagsterUserDeployments=UserDeployments(
enabled=True,
enableSubchart=True,
deployments=[deployment],
)
)
user_deployments = template.render(helm_values)
assert len(user_deployments) == 1
if include_config_in_launched_runs:
container_context = user_deployments[0].spec.template.spec.containers[0].env[2]
assert container_context.name == "DAGSTER_CLI_API_GRPC_CONTAINER_CONTEXT"
assert json.loads(container_context.value) == {
"k8s": {
"image_pull_policy": "Always",
"env_config_maps": [
"release-name-dagster-user-deployments-foo-user-env",
],
"labels": labels,
}
}
else:
_assert_no_container_context(user_deployments[0])
def test_user_deployment_volumes(template: HelmTemplate):
name = "foo"
volumes = [
{
"name": "test-volume",
"configMap": {
"name": "test-volume-configmap"
}
},
{
"name": "test-pvc",
"persistentVolumeClaim": {
"claimName": "my_claim",
"readOnly": False
}
},
]
volume_mounts = [{
"name": "test-volume",
"mountPath": "/opt/dagster/test_mount_path/volume_mounted_file.yaml",
"subPath": "volume_mounted_file.yaml",
}]
deployment = UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}",
tag="tag1",
pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
volumes=[
kubernetes.Volume.construct(None, **volume) for volume in volumes
],
volumeMounts=[
kubernetes.VolumeMount.construct(None, **volume_mount)
for volume_mount in volume_mounts
],
)
helm_values = DagsterHelmValues.construct(
dagsterUserDeployments=UserDeployments(
enabled=True,
enableSubchart=True,
deployments=[deployment],
))
user_deployments = template.render(helm_values)
assert len(user_deployments) == 1
image = user_deployments[0].spec.template.spec.containers[0].image
image_name, image_tag = image.split(":")
deployed_volume_mounts = user_deployments[0].spec.template.spec.containers[
0].volume_mounts
assert deployed_volume_mounts == [
k8s_model_from_dict(k8s_client.models.V1VolumeMount, volume_mount)
for volume_mount in volume_mounts
]
deployed_volumes = user_deployments[0].spec.template.spec.volumes
assert deployed_volumes == [
k8s_model_from_dict(k8s_client.models.V1Volume, volume)
for volume in volumes
]
assert image_name == deployment.image.repository
assert image_tag == deployment.image.tag
def test_user_deployment_volumes(template: HelmTemplate, include_config_in_launched_runs: bool):
name = "foo"
volumes = [
{"name": "test-volume", "configMap": {"name": "test-volume-configmap"}},
{"name": "test-pvc", "persistentVolumeClaim": {"claimName": "my_claim", "readOnly": False}},
]
volume_mounts = [
{
"name": "test-volume",
"mountPath": "/opt/dagster/test_mount_path/volume_mounted_file.yaml",
"subPath": "volume_mounted_file.yaml",
}
]
deployment = UserDeployment(
name=name,
image=kubernetes.Image(repository=f"repo/{name}", tag="tag1", pullPolicy="Always"),
dagsterApiGrpcArgs=["-m", name],
port=3030,
volumes=[kubernetes.Volume.construct(None, **volume) for volume in volumes],
volumeMounts=[
kubernetes.VolumeMount.construct(None, **volume_mount) for volume_mount in volume_mounts
],
includeConfigInLaunchedRuns=UserDeploymentIncludeConfigInLaunchedRuns(
enabled=include_config_in_launched_runs
),
)
helm_values = DagsterHelmValues.construct(
dagsterUserDeployments=UserDeployments(
enabled=True,
enableSubchart=True,
deployments=[deployment],
)
)
user_deployments = template.render(helm_values)
assert len(user_deployments) == 1
image = user_deployments[0].spec.template.spec.containers[0].image
image_name, image_tag = image.split(":")
deployed_volume_mounts = user_deployments[0].spec.template.spec.containers[0].volume_mounts
assert deployed_volume_mounts == [
k8s_model_from_dict(
k8s_client.models.V1VolumeMount,
k8s_snake_case_dict(k8s_client.models.V1VolumeMount, volume_mount),
)
for volume_mount in volume_mounts
]
deployed_volumes = user_deployments[0].spec.template.spec.volumes
assert deployed_volumes == [
k8s_model_from_dict(
k8s_client.models.V1Volume, k8s_snake_case_dict(k8s_client.models.V1Volume, volume)
)
for volume in volumes
]
assert image_name == deployment.image.repository
assert image_tag == deployment.image.tag
if include_config_in_launched_runs:
container_context = user_deployments[0].spec.template.spec.containers[0].env[2]
assert container_context.name == "DAGSTER_CLI_API_GRPC_CONTAINER_CONTEXT"
assert json.loads(container_context.value) == {
"k8s": {
"env_config_maps": [
"release-name-dagster-user-deployments-foo-user-env",
],
"image_pull_policy": "Always",
"volume_mounts": volume_mounts,
"volumes": volumes,
}
}
else:
_assert_no_container_context(user_deployments[0])