def celery_k8s_config():
from dagster_k8s import DagsterK8sJobConfig
# DagsterK8sJobConfig provides config schema for specifying Dagster K8s Jobs
job_config = DagsterK8sJobConfig.config_type_pipeline_run()
additional_config = {
'load_incluster_config': Field(
bool,
is_required=False,
default_value=True,
description='''Set this value if you are running the launcher within a k8s cluster. If
``True``, we assume the launcher is running within the target cluster and load config
using ``kubernetes.config.load_incluster_config``. Otherwise, we will use the k8s config
specified in ``kubeconfig_file`` (using ``kubernetes.config.load_kube_config``) or fall
back to the default kubeconfig. Default: ``True``.''',
),
'kubeconfig_file': Field(
Noneable(str),
is_required=False,
description='Path to a kubeconfig file to use, if not using default kubeconfig.',
),
'job_namespace': Field(
StringSource,
is_required=False,
default_value='default',
description='The namespace into which to launch new jobs. Note that any '
'other Kubernetes resources the Job requires (such as the service account) must be '
'present in this namespace. Default: ``"default"``',
),
}
cfg = merge_dicts(CELERY_CONFIG, job_config)
cfg = merge_dicts(cfg, additional_config)
return cfg
def celery_k8s_config():
# DagsterK8sJobConfig provides config schema for specifying Dagster K8s Jobs
job_config = DagsterK8sJobConfig.config_type_pipeline_run()
additional_config = {
"load_incluster_config":
Field(
bool,
is_required=False,
default_value=True,
description=
"""Set this value if you are running the launcher within a k8s cluster. If
``True``, we assume the launcher is running within the target cluster and load config
using ``kubernetes.config.load_incluster_config``. Otherwise, we will use the k8s config
specified in ``kubeconfig_file`` (using ``kubernetes.config.load_kube_config``) or fall
back to the default kubeconfig. Default: ``True``.""",
),
"kubeconfig_file":
Field(
Noneable(str),
is_required=False,
description=
"Path to a kubeconfig file to use, if not using default kubeconfig.",
),
"job_namespace":
Field(
StringSource,
is_required=False,
default_value="default",
description=
"The namespace into which to launch new jobs. Note that any "
"other Kubernetes resources the Job requires (such as the service account) must be "
'present in this namespace. Default: ``"default"``',
),
"repo_location_name":
Field(
StringSource,
is_required=False,
default_value=IN_PROCESS_NAME,
description="The repository location name to use for execution.",
),
"job_wait_timeout":
Field(
Float,
is_required=False,
default_value=DEFAULT_WAIT_TIMEOUT,
description=
f"Wait this many seconds for a job to complete before marking the run as failed. Defaults to {DEFAULT_WAIT_TIMEOUT} seconds.",
),
}
cfg = merge_dicts(CELERY_CONFIG, job_config)
cfg = merge_dicts(cfg, additional_config)
return cfg