from dagster.core.executor.step_delegating import StepDelegatingExecutor
from dagster.core.executor.step_delegating.step_handler.base import StepHandler, StepHandlerContext
from dagster.serdes.utils import hash_str
from dagster.utils import merge_dicts
from dagster.utils.backcompat import experimental
from .container_context import DockerContainerContext
from .utils import parse_env_var
@executor(
name="docker",
config_schema=merge_dicts(
DOCKER_CONFIG_SCHEMA,
{
"retries": get_retries_config(),
},
),
requirements=multiple_process_executor_requirements(),
)
@experimental
def docker_executor(init_context: InitExecutorContext) -> Executor:
"""
Executor which launches steps as Docker containers.
To use the `docker_executor`, set it as the `executor_def` when defining a job:
.. literalinclude:: ../../../../../../python_modules/libraries/dagster-docker/dagster_docker_tests/test_example_executor.py
:start-after: start_marker
:end-before: end_marker
:language: python
executor_def = ExecutorDefinition(
name=self.name,
config=self.config,
executor_creation_fn=fn,
required_resource_keys=self.required_resource_keys,
)
update_wrapper(executor_def, wrapped=fn)
return executor_def
@executor(
name='in_process',
config={
'retries': get_retries_config(),
'marker_to_close': Field(str, is_required=False),
},
)
def in_process_executor(init_context):
'''The default in-process executor.
In most Dagster environments, this will be the default executor. It is available by default on
any :py:class:`ModeDefinition` that does not provide custom executors. To select it explicitly,
include the following top-level fragment in config:
.. code-block:: yaml
execution:
in_process:
is_required=False,
default_value="rpc://",
description="The URL of the Celery results backend. Default: 'rpc://'.",
),
"include":
Field([str],
is_required=False,
description="List of modules every worker should import"),
"config_source":
Field(
Noneable(Permissive()),
is_required=False,
description="Additional settings for the Celery app.",
),
"retries":
get_retries_config(),
}
@executor(name="celery", config_schema=CELERY_CONFIG)
def celery_executor(init_context):
"""Celery-based executor.
The Celery executor exposes config settings for the underlying Celery app under
the ``config_source`` key. This config corresponds to the "new lowercase settings" introduced
in Celery version 4.0 and the object constructed from config will be passed to the
:py:class:`celery.Celery` constructor as its ``config_source`` argument.
(See https://docs.celeryproject.org/en/latest/userguide/configuration.html for details.)
The executor also exposes the ``broker``, `backend`, and ``include`` arguments to the
:py:class:`celery.Celery` constructor.
executor_def = ExecutorDefinition(
name=self.name,
config=self.config,
executor_creation_fn=fn,
required_resource_keys=self.required_resource_keys,
)
update_wrapper(executor_def, wrapped=fn)
return executor_def
@executor(
name='in_process',
config={'retries': get_retries_config(), 'marker_to_close': Field(str, is_required=False),},
)
def in_process_executor(init_context):
'''The default in-process executor.
In most Dagster environments, this will be the default executor. It is available by default on
any :py:class:`ModeDefinition` that does not provide custom executors. To select it explicitly,
include the following top-level fragment in config:
.. code-block:: yaml
execution:
in_process:
Execution priority can be configured using the ``dagster/priority`` tag via solid metadata,
where the higher the number the higher the priority. 0 is the default and both positive
