def multiprocess_executor(init_context): '''The default multiprocess executor. This simple multiprocess executor is available by default on any :py:class:`ModeDefinition` that does not provide custom executors. To select the multiprocess executor, include a fragment such as the following in your config: .. code-block:: yaml execution: multiprocess: max_concurrent: 4 The ``max_concurrent`` arg is optional and tells the execution engine how many processes may run concurrently. By default, or if you set ``max_concurrent`` to be 0, this is the return value of :py:func:`python:multiprocessing.cpu_count`. ''' from dagster.core.definitions.handle import ExecutionTargetHandle from dagster.core.engine.init import InitExecutorContext check.inst_param(init_context, 'init_context', InitExecutorContext) handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def) return MultiprocessExecutorConfig( handle=handle, max_concurrent=init_context.executor_config['max_concurrent'])
def multiprocess_executor(init_context): '''The default multiprocess executor. This simple multiprocess executor is available by default on any :py:class:`ModeDefinition` that does not provide custom executors. To select the multiprocess executor, include a fragment such as the following in your config: .. code-block:: yaml execution: multiprocess: max_concurrent: 4 The ``max_concurrent`` arg is optional and tells the execution engine how many processes may run concurrently. By default, or if you set ``max_concurrent`` to be 0, this is the return value of :py:func:`python:multiprocessing.cpu_count`. Execution priority can be configured using the ``dagster/priority`` tag via solid metadata, where the higher the number the higher the priority. 0 is the default and both positive and negative numbers can be used. ''' from dagster.core.definitions.handle import ExecutionTargetHandle from dagster.core.engine.init import InitExecutorContext check.inst_param(init_context, 'init_context', InitExecutorContext) check_cross_process_constraints(init_context) handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def) return MultiprocessExecutorConfig( handle=handle, max_concurrent=init_context.executor_config['max_concurrent'], retries=Retries.from_config(init_context.executor_config['retries']), )
def multiprocess_executor(init_context): from dagster.core.definitions.handle import ExecutionTargetHandle from dagster.core.engine.init import InitExecutorContext check.inst_param(init_context, 'init_context', InitExecutorContext) handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def) return MultiprocessExecutorConfig( handle=handle, max_concurrent=init_context.executor_config['max_concurrent'])
def multiprocess_executor(init_context): '''The default multiprocess executor. This simple multiprocess executor is available by default on any :py:class:`ModeDefinition` that does not provide custom executors. To select the multiprocess executor, include a fragment such as the following in your config: .. code-block:: yaml execution: multiprocess: max_concurrent: 4 The ``max_concurrent`` arg is optional and tells the execution engine how many processes may run concurrently. By default, or if you set ``max_concurrent`` to be 0, this is the return value of :py:func:`python:multiprocessing.cpu_count`. Execution priority can be configured using the ``dagster/priority`` tag via solid metadata, where the higher the number the higher the priority. 0 is the default and both positive and negative numbers can be used. ''' from dagster.core.definitions.handle import ExecutionTargetHandle from dagster.core.engine.init import InitExecutorContext check.inst_param(init_context, 'init_context', InitExecutorContext) check_cross_process_constraints(init_context) # ExecutionTargetHandle.get_handle returns an ExecutionTargetHandleCacheEntry, which is a tuple # (handle, solid_subset). Right now we are throwing away the solid_subset that we store in the # cache -- this is fragile and we should fix this with # https://github.com/dagster-io/dagster/issues/2115 and friends so there are not multiple # sources of truth for the solid subset handle, _ = ExecutionTargetHandle.get_handle(init_context.pipeline_def) return MultiprocessExecutorConfig( handle=handle, max_concurrent=init_context.executor_config['max_concurrent'], retries=Retries.from_config(init_context.executor_config['retries']), )