from dagster.core.executor.step_delegating import StepDelegatingExecutor from dagster.core.executor.step_delegating.step_handler.base import StepHandler, StepHandlerContext from dagster.serdes.utils import hash_str from dagster.utils import merge_dicts from dagster.utils.backcompat import experimental from .container_context import DockerContainerContext from .utils import parse_env_var @executor( name="docker", config_schema=merge_dicts( DOCKER_CONFIG_SCHEMA, { "retries": get_retries_config(), }, ), requirements=multiple_process_executor_requirements(), ) @experimental def docker_executor(init_context: InitExecutorContext) -> Executor: """ Executor which launches steps as Docker containers. To use the `docker_executor`, set it as the `executor_def` when defining a job: .. literalinclude:: ../../../../../../python_modules/libraries/dagster-docker/dagster_docker_tests/test_example_executor.py :start-after: start_marker :end-before: end_marker :language: python
executor_def = ExecutorDefinition( name=self.name, config=self.config, executor_creation_fn=fn, required_resource_keys=self.required_resource_keys, ) update_wrapper(executor_def, wrapped=fn) return executor_def @executor( name='in_process', config={ 'retries': get_retries_config(), 'marker_to_close': Field(str, is_required=False), }, ) def in_process_executor(init_context): '''The default in-process executor. In most Dagster environments, this will be the default executor. It is available by default on any :py:class:`ModeDefinition` that does not provide custom executors. To select it explicitly, include the following top-level fragment in config: .. code-block:: yaml execution: in_process:
is_required=False, default_value="rpc://", description="The URL of the Celery results backend. Default: 'rpc://'.", ), "include": Field([str], is_required=False, description="List of modules every worker should import"), "config_source": Field( Noneable(Permissive()), is_required=False, description="Additional settings for the Celery app.", ), "retries": get_retries_config(), } @executor(name="celery", config_schema=CELERY_CONFIG) def celery_executor(init_context): """Celery-based executor. The Celery executor exposes config settings for the underlying Celery app under the ``config_source`` key. This config corresponds to the "new lowercase settings" introduced in Celery version 4.0 and the object constructed from config will be passed to the :py:class:`celery.Celery` constructor as its ``config_source`` argument. (See https://docs.celeryproject.org/en/latest/userguide/configuration.html for details.) The executor also exposes the ``broker``, `backend`, and ``include`` arguments to the :py:class:`celery.Celery` constructor.
executor_def = ExecutorDefinition( name=self.name, config=self.config, executor_creation_fn=fn, required_resource_keys=self.required_resource_keys, ) update_wrapper(executor_def, wrapped=fn) return executor_def @executor( name='in_process', config={'retries': get_retries_config(), 'marker_to_close': Field(str, is_required=False),}, ) def in_process_executor(init_context): '''The default in-process executor. In most Dagster environments, this will be the default executor. It is available by default on any :py:class:`ModeDefinition` that does not provide custom executors. To select it explicitly, include the following top-level fragment in config: .. code-block:: yaml execution: in_process: Execution priority can be configured using the ``dagster/priority`` tag via solid metadata, where the higher the number the higher the priority. 0 is the default and both positive