def multiprocess_executor(init_context):
'''The default multiprocess executor.
This simple multiprocess executor is available by default on any :py:class:`ModeDefinition`
that does not provide custom executors. To select the multiprocess executor, include a fragment
such as the following in your config:
.. code-block:: yaml
execution:
multiprocess:
max_concurrent: 4
The ``max_concurrent`` arg is optional and tells the execution engine how many processes may run
concurrently. By default, or if you set ``max_concurrent`` to be 0, this is the return value of
:py:func:`python:multiprocessing.cpu_count`.
'''
from dagster.core.definitions.handle import ExecutionTargetHandle
from dagster.core.engine.init import InitExecutorContext
check.inst_param(init_context, 'init_context', InitExecutorContext)
handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def)
return MultiprocessExecutorConfig(
handle=handle,
max_concurrent=init_context.executor_config['max_concurrent'])
def multiprocess_executor(init_context):
'''The default multiprocess executor.
This simple multiprocess executor is available by default on any :py:class:`ModeDefinition`
that does not provide custom executors. To select the multiprocess executor, include a fragment
such as the following in your config:
.. code-block:: yaml
execution:
multiprocess:
max_concurrent: 4
The ``max_concurrent`` arg is optional and tells the execution engine how many processes may run
concurrently. By default, or if you set ``max_concurrent`` to be 0, this is the return value of
:py:func:`python:multiprocessing.cpu_count`.
Execution priority can be configured using the ``dagster/priority`` tag via solid metadata,
where the higher the number the higher the priority. 0 is the default and both positive
and negative numbers can be used.
'''
from dagster.core.definitions.handle import ExecutionTargetHandle
from dagster.core.engine.init import InitExecutorContext
check.inst_param(init_context, 'init_context', InitExecutorContext)
check_cross_process_constraints(init_context)
handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def)
return MultiprocessExecutorConfig(
handle=handle,
max_concurrent=init_context.executor_config['max_concurrent'],
retries=Retries.from_config(init_context.executor_config['retries']),
)
def multiprocess_executor(init_context):
from dagster.core.definitions.handle import ExecutionTargetHandle
from dagster.core.engine.init import InitExecutorContext
check.inst_param(init_context, 'init_context', InitExecutorContext)
handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def)
return MultiprocessExecutorConfig(
handle=handle,
max_concurrent=init_context.executor_config['max_concurrent'])
def multiprocess_executor(init_context):
'''The default multiprocess executor.
This simple multiprocess executor is available by default on any :py:class:`ModeDefinition`
that does not provide custom executors. To select the multiprocess executor, include a fragment
such as the following in your config:
.. code-block:: yaml
execution:
multiprocess:
max_concurrent: 4
The ``max_concurrent`` arg is optional and tells the execution engine how many processes may run
concurrently. By default, or if you set ``max_concurrent`` to be 0, this is the return value of
:py:func:`python:multiprocessing.cpu_count`.
Execution priority can be configured using the ``dagster/priority`` tag via solid metadata,
where the higher the number the higher the priority. 0 is the default and both positive
and negative numbers can be used.
'''
from dagster.core.definitions.handle import ExecutionTargetHandle
from dagster.core.engine.init import InitExecutorContext
check.inst_param(init_context, 'init_context', InitExecutorContext)
check_cross_process_constraints(init_context)
# ExecutionTargetHandle.get_handle returns an ExecutionTargetHandleCacheEntry, which is a tuple
# (handle, solid_subset). Right now we are throwing away the solid_subset that we store in the
# cache -- this is fragile and we should fix this with
# https://github.com/dagster-io/dagster/issues/2115 and friends so there are not multiple
# sources of truth for the solid subset
handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def)
return MultiprocessExecutorConfig(
handle=handle,
max_concurrent=init_context.executor_config['max_concurrent'],
retries=Retries.from_config(init_context.executor_config['retries']),
)