from dagster import check
from dagster.config.field_utils import Permissive
from .config_schema import input_hydration_config
from .runtime_type import RuntimeType, define_python_dagster_type, resolve_to_runtime_type
@input_hydration_config(Permissive())
def _dict_input(_context, value):
return value
PythonDict = define_python_dagster_type(
dict,
'PythonDict',
input_hydration_config=_dict_input,
description='''Represents a python dictionary to pass between solids''',
)
class _TypedPythonDict(RuntimeType):
def __init__(self, key_type, value_type):
self.key_type = check.inst_param(key_type, 'key_type', RuntimeType)
self.value_type = check.inst_param(value_type, 'value_type',
RuntimeType)
super(_TypedPythonDict, self).__init__(
key='TypedPythonDict.{}.{}'.format(key_type.key, value_type.key),
name=None,
type_check_fn=self.type_check_method,
)
def reset(cls):
cls.processes = []
cls.launch_step_count = 0
cls.check_step_health_count = 0
cls.terminate_step_count = 0
@classmethod
def wait_for_processes(cls):
for p in cls.processes:
p.wait(timeout=5)
@executor(
name="test_step_delegating_executor",
requirements=multiple_process_executor_requirements(),
config_schema=Permissive(),
)
def test_step_delegating_executor(exc_init):
return StepDelegatingExecutor(
TestStepHandler(),
sleep_seconds=exc_init.executor_config.get("sleep_seconds"),
check_step_health_interval_seconds=exc_init.executor_config.get(
"check_step_health_interval_seconds"),
)
@solid
def bar_solid(_):
return "bar"
# container.
DAGSTER_PG_PASSWORD_ENV_VAR = "DAGSTER_PG_PASSWORD"
# We expect the PG secret to have this key.
#
# For an example, see:
# python_modules/libraries/dagster-k8s/helm/dagster/templates/secret-postgres.yaml
DAGSTER_PG_PASSWORD_SECRET_KEY = "postgresql-password"
# Kubernetes Job object names cannot be longer than 63 characters
MAX_K8S_NAME_LEN = 63
# TODO: Deprecate this tag
K8S_RESOURCE_REQUIREMENTS_KEY = "dagster-k8s/resource_requirements"
K8S_RESOURCE_REQUIREMENTS_SCHEMA = Shape({
"limits": Permissive(),
"requests": Permissive()
})
USER_DEFINED_K8S_CONFIG_KEY = "dagster-k8s/config"
USER_DEFINED_K8S_CONFIG_SCHEMA = Shape({
"container_config":
Permissive(),
"pod_template_spec_metadata":
Permissive(),
"pod_spec_config":
Permissive(),
"job_config":
Permissive(),
"job_metadata":
Permissive(),
from dagster import check
from dagster.config.field_utils import Permissive
from dagster.core.types.dagster_type import String
from .config_schema import DagsterTypeLoader, dagster_type_loader
from .dagster_type import DagsterType, PythonObjectDagsterType, resolve_dagster_type
@dagster_type_loader(Permissive())
def _dict_input(_context, value):
return value
PythonDict = PythonObjectDagsterType(
dict,
"PythonDict",
loader=_dict_input,
description="""Represents a python dictionary to pass between solids""",
)
class TypedDictLoader(DagsterTypeLoader):
def __init__(self, value_dagster_type):
self._value_dagster_type = check.inst_param(
value_dagster_type, "value_dagster_type", DagsterType
)
@property
def schema_type(self):
return Permissive()
def schema_type(self):
return Permissive()
def config_type_pipeline_run(cls, default_image_pull_policy=None):
"""Configuration intended to be set at pipeline execution time."""
return {
"job_image": Field(
Noneable(StringSource),
is_required=False,
description="Docker image to use for launched task Jobs. If the repository is not "
"loaded from a GRPC server, then this field is required. If the repository is "
"loaded from a GRPC server, then leave this field empty."
'(Ex: "mycompany.com/dagster-k8s-image:latest").',
),
"image_pull_policy": Field(
Noneable(StringSource),
is_required=False,
description="Image pull policy to set on the launched task Job Pods. Defaults to "
'"IfNotPresent".',
default_value=default_image_pull_policy,
),
"image_pull_secrets": Field(
Noneable(Array(Shape({"name": StringSource}))),
is_required=False,
description="(Advanced) Specifies that Kubernetes should get the credentials from "
"the Secrets named in this list.",
),
"service_account_name": Field(
Noneable(StringSource),
is_required=False,
description="(Advanced) Override the name of the Kubernetes service account under "
"which to run the Job.",
),
"env_config_maps": Field(
Noneable(Array(StringSource)),
is_required=False,
description="A list of custom ConfigMapEnvSource names from which to draw "
"environment variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container",
),
"env_secrets": Field(
Noneable(Array(StringSource)),
is_required=False,
description="A list of custom Secret names from which to draw environment "
"variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables",
),
"env_vars": Field(
Noneable(Array(str)),
is_required=False,
description="A list of environment variables to inject into the Job. "
"Default: ``[]``. See: "
"https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables",
),
"volume_mounts": Field(
Array(
Shape(
{
"name": StringSource,
"mountPath": StringSource,
"mountPropagation": Field(StringSource, is_required=False),
"readOnly": Field(BoolSource, is_required=False),
"subPath": Field(StringSource, is_required=False),
"subPathExpr": Field(StringSource, is_required=False),
}
)
),
is_required=False,
default_value=[],
description="A list of volume mounts to include in the job's container. Default: ``[]``. See: "
"https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core",
),
"volumes": Field(
Array(
Permissive(
{
"name": str,
}
)
),
is_required=False,
default_value=[],
description="A list of volumes to include in the Job's Pod. Default: ``[]``. For the many "
"possible volume source types that can be included, see: "
"https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core",
),
}
def config_type_run_launcher(cls):
"""Configuration intended to be set on the Dagster instance."""
return {
"instance_config_map": Field(
StringSource,
is_required=True,
description="The ``name`` of an existing Volume to mount into the pod in order to "
"provide a ConfigMap for the Dagster instance. This Volume should contain a "
"``dagster.yaml`` with appropriate values for run storage, event log storage, etc.",
),
"postgres_password_secret": Field(
StringSource,
is_required=False,
description="The name of the Kubernetes Secret where the postgres password can be "
"retrieved. Will be mounted and supplied as an environment variable to the Job Pod."
'Secret must contain the key ``"postgresql-password"`` which will be exposed in '
"the Job environment as the environment variable ``DAGSTER_PG_PASSWORD``.",
),
"dagster_home": Field(
StringSource,
is_required=False,
default_value=DAGSTER_HOME_DEFAULT,
description="The location of DAGSTER_HOME in the Job container; this is where the "
"``dagster.yaml`` file will be mounted from the instance ConfigMap specified here. "
"Defaults to /opt/dagster/dagster_home.",
),
"env_config_maps": Field(
Noneable(Array(StringSource)),
is_required=False,
description="A list of custom ConfigMapEnvSource names from which to draw "
"environment variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container",
),
"env_secrets": Field(
Noneable(Array(StringSource)),
is_required=False,
description="A list of custom Secret names from which to draw environment "
"variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables",
),
"volume_mounts": Field(
Array(
Shape(
{
"name": StringSource,
"mountPath": StringSource,
"mountPropagation": Field(StringSource, is_required=False),
"readOnly": Field(BoolSource, is_required=False),
"subPath": Field(StringSource, is_required=False),
"subPathExpr": Field(StringSource, is_required=False),
}
)
),
is_required=False,
default_value=[],
description="A list of volume mounts to include in the job's container. Default: ``[]``. See: "
"https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core",
),
"volumes": Field(
Array(
Permissive(
{
"name": str,
}
)
),
is_required=False,
default_value=[],
description="A list of volumes to include in the Job's Pod. Default: ``[]``. For the many "
"possible volume source types that can be included, see: "
"https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core",
),
}
# The Kubernetes Secret containing the PG password will be exposed as this env var in the job
# container.
DAGSTER_PG_PASSWORD_ENV_VAR = "DAGSTER_PG_PASSWORD"
# We expect the PG secret to have this key.
#
# For an example, see:
# helm/dagster/templates/secret-postgres.yaml
DAGSTER_PG_PASSWORD_SECRET_KEY = "postgresql-password"
# Kubernetes Job object names cannot be longer than 63 characters
MAX_K8S_NAME_LEN = 63
# TODO: Deprecate this tag
K8S_RESOURCE_REQUIREMENTS_KEY = "dagster-k8s/resource_requirements"
K8S_RESOURCE_REQUIREMENTS_SCHEMA = Shape({"limits": Permissive(), "requests": Permissive()})
USER_DEFINED_K8S_CONFIG_KEY = "dagster-k8s/config"
USER_DEFINED_K8S_CONFIG_SCHEMA = Shape(
{
"container_config": Permissive(),
"pod_template_spec_metadata": Permissive(),
"pod_spec_config": Permissive(),
"job_config": Permissive(),
"job_metadata": Permissive(),
"job_spec_config": Permissive(),
}
)
DEFAULT_JOB_SPEC_CONFIG = {
"ttl_seconds_after_finished": DEFAULT_K8S_JOB_TTL_SECONDS_AFTER_FINISHED,
remapped.
'''
from dagster.core.types.dagster_type import resolve_dagster_type
check.param_invariant(is_supported_runtime_python_builtin(ttype), 'ttype')
return resolve_dagster_type(SUPPORTED_RUNTIME_BUILTINS[ttype])
SUPPORTED_CONFIG_BUILTINS = {
int: Int,
float: Float,
bool: Bool,
str: String,
list: Array(ConfigAny),
dict: Permissive(),
}
def is_supported_config_python_builtin(ttype):
return ttype in SUPPORTED_CONFIG_BUILTINS
def remap_python_builtin_for_config(ttype):
'''This function remaps a python type to a Dagster type, or passes it through if it cannot be
remapped.
'''
from dagster.config.field import resolve_to_config_type
check.param_invariant(is_supported_config_python_builtin(ttype), 'ttype')
def config_type_container(cls):
return {
"image_pull_policy":
Field(
Noneable(StringSource),
is_required=False,
description="Image pull policy to set on launched Pods.",
),
"image_pull_secrets":
Field(
Noneable(Array(Shape({"name": StringSource}))),
is_required=False,
description=
"Specifies that Kubernetes should get the credentials from "
"the Secrets named in this list.",
),
"service_account_name":
Field(
Noneable(StringSource),
is_required=False,
description=
"The name of the Kubernetes service account under which to run.",
),
"env_config_maps":
Field(
Noneable(Array(StringSource)),
is_required=False,
description=
"A list of custom ConfigMapEnvSource names from which to draw "
"environment variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/define-environment-variable-container/#define-an-environment-variable-for-a-container",
),
"env_secrets":
Field(
Noneable(Array(StringSource)),
is_required=False,
description=
"A list of custom Secret names from which to draw environment "
"variables (using ``envFrom``) for the Job. Default: ``[]``. See:"
"https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables",
),
"env_vars":
Field(
Noneable(Array(str)),
is_required=False,
description=
"A list of environment variables to inject into the Job. "
"Default: ``[]``. See: "
"https://kubernetes.io/docs/tasks/inject-data-application/distribute-credentials-secure/#configure-all-key-value-pairs-in-a-secret-as-container-environment-variables",
),
"volume_mounts":
Field(
Array(
# Can supply either snake_case or camelCase, but in typeaheads based on the
# schema we assume snake_case
Permissive({
"name":
StringSource,
"mount_path":
Field(StringSource, is_required=False),
"mount_propagation":
Field(StringSource, is_required=False),
"read_only":
Field(BoolSource, is_required=False),
"sub_path":
Field(StringSource, is_required=False),
"sub_path_expr":
Field(StringSource, is_required=False),
})),
is_required=False,
default_value=[],
description=
"A list of volume mounts to include in the job's container. Default: ``[]``. See: "
"https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core",
),
"volumes":
Field(
Array(Permissive({
"name": str,
})),
is_required=False,
default_value=[],
description=
"A list of volumes to include in the Job's Pod. Default: ``[]``. For the many "
"possible volume source types that can be included, see: "
"https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core",
),
"labels":
Field(
dict,
is_required=False,
description="Labels to apply to all created pods. See: "
"https://kubernetes.io/docs/concepts/overview/working-with-objects/labels",
),
}