def __init__(
self,
inst_data=None,
image=None,
registry=None,
env_vars=None,
network=None,
networks=None,
container_kwargs=None,
):
self._inst_data = inst_data
self.image = image
self.registry = registry
self.env_vars = env_vars
validate_docker_config(network, networks, container_kwargs)
if network:
self.networks = [network]
elif networks:
self.networks = networks
else:
self.networks = []
self.container_kwargs = check.opt_dict_param(container_kwargs,
"container_kwargs",
key_type=str)
super().__init__()
def docker_executor(init_context: InitExecutorContext) -> Executor:
from . import DockerRunLauncher
image = init_context.executor_config.get("image")
registry = init_context.executor_config.get("registry")
env_vars = init_context.executor_config.get("env_vars")
network = init_context.executor_config.get("network")
networks = init_context.executor_config.get("networks")
container_kwargs = init_context.executor_config.get("container_kwargs")
run_launcher = init_context.instance.run_launcher
if isinstance(run_launcher, DockerRunLauncher):
image = image or run_launcher.image
registry = registry or run_launcher.registry
env_vars = run_launcher.env_vars + (env_vars or [])
networks = run_launcher.networks + (networks or [])
container_kwargs = merge_dicts(run_launcher.container_kwargs, container_kwargs or {})
validate_docker_config(network, networks, container_kwargs)
return StepDelegatingExecutor(
DockerStepHandler(
image,
registry,
env_vars,
network,
networks,
container_kwargs,
),
retries=RetryMode.from_config(init_context.executor_config["retries"]),
)
def docker_executor(init_context: InitExecutorContext) -> Executor:
"""
Executor which launches steps as Docker containers.
To use the `docker_executor`, set it as the `executor_def` when defining a job:
.. literalinclude:: ../../../../../../python_modules/libraries/dagster-docker/dagster_docker_tests/test_example_executor.py
:start-after: start_marker
:end-before: end_marker
:language: python
Then you can configure the executor with run config as follows:
.. code-block:: YAML
execution:
config:
registry: ...
network: ...
networks: ...
container_kwargs: ...
If you're using the DockerRunLauncher, configuration set on the containers created by the run
launcher will also be set on the containers that are created for each step.
"""
from . import DockerRunLauncher
image = init_context.executor_config.get("image")
registry = init_context.executor_config.get("registry")
env_vars = init_context.executor_config.get("env_vars")
network = init_context.executor_config.get("network")
networks = init_context.executor_config.get("networks")
container_kwargs = init_context.executor_config.get("container_kwargs")
run_launcher = init_context.instance.run_launcher
if isinstance(run_launcher, DockerRunLauncher):
image = image or run_launcher.image
registry = registry or run_launcher.registry
env_vars = run_launcher.env_vars + (env_vars or [])
networks = run_launcher.networks + (networks or [])
container_kwargs = merge_dicts(run_launcher.container_kwargs,
container_kwargs or {})
validate_docker_config(network, networks, container_kwargs)
return StepDelegatingExecutor(
DockerStepHandler(
image,
registry,
env_vars,
network,
networks,
container_kwargs,
),
retries=RetryMode.from_config(init_context.executor_config["retries"]),
)
def _get_docker_container_context(
self, step_handler_context: StepHandlerContext):
# This doesn't vary per step: would be good to have a hook where it can be set once
# for the whole StepHandler but we need access to the PipelineRun for that
from .docker_run_launcher import DockerRunLauncher
run_launcher = step_handler_context.instance.run_launcher
run_target = DockerContainerContext.create_for_run(
step_handler_context.pipeline_run,
run_launcher
if isinstance(run_launcher, DockerRunLauncher) else None,
)
merged_container_context = run_target.merge(self._container_context)
validate_docker_config(
network=None,
networks=merged_container_context.networks,
container_kwargs=merged_container_context.container_kwargs,
)
return merged_container_context