def _get_host_mode_executor(recon_pipeline, run_config, executor_defs,
instance):
execution_config = run_config.get("execution", {})
execution_config_type = Field(selector_for_named_defs(executor_defs),
default_value={
executor_defs[0].name: {}
}).config_type
config_evr = process_config(execution_config_type, execution_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error processing execution config {}".format(execution_config),
config_evr.errors,
execution_config,
)
execution_config_value = config_evr.value
executor_name, executor_config = ensure_single_item(execution_config_value)
executor_defs_by_name = {
executor_def.name: executor_def
for executor_def in executor_defs
}
executor_def = executor_defs_by_name[executor_name]
init_context = InitExecutorContext(
job=recon_pipeline,
executor_def=executor_def,
executor_config=executor_config["config"],
instance=instance,
)
check_cross_process_constraints(init_context)
return executor_def.executor_creation_fn(init_context)
def _get_host_mode_executor(recon_pipeline, run_config, get_executor_def_fn,
instance):
execution_config = run_config.get("execution")
if execution_config:
executor_name, executor_config = ensure_single_item(execution_config)
else:
executor_name = None
executor_config = {}
executor_def = get_executor_def_fn(executor_name)
executor_config_type = def_config_field(executor_def).config_type
config_evr = process_config(executor_config_type, executor_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in executor config for executor {}".format(
executor_def.name),
config_evr.errors,
executor_config,
)
executor_config_value = config_evr.value
init_context = InitExecutorContext(
pipeline=recon_pipeline,
executor_def=executor_def,
executor_config=executor_config_value["config"],
instance=instance,
)
check_cross_process_constraints(init_context)
return executor_def.executor_creation_fn(init_context)
def create_executor(context_creation_data: ContextCreationData) -> "Executor":
check.inst_param(context_creation_data, "context_creation_data", ContextCreationData)
init_context = InitExecutorContext(
job=context_creation_data.pipeline,
executor_def=context_creation_data.executor_def,
executor_config=context_creation_data.resolved_run_config.execution.execution_engine_config,
instance=context_creation_data.instance,
)
check_cross_process_constraints(init_context)
return context_creation_data.executor_def.executor_creation_fn(init_context)
def create_executor(context_creation_data):
check.inst_param(context_creation_data, "context_creation_data",
ContextCreationData)
init_context = InitExecutorContext(
pipeline=context_creation_data.pipeline,
executor_def=context_creation_data.executor_def,
executor_config=context_creation_data.environment_config.execution.
execution_engine_config,
instance=context_creation_data.instance,
)
check_cross_process_constraints(init_context)
return context_creation_data.executor_def.executor_creation_fn(
init_context)
def create_executor(context_creation_data):
check.inst_param(context_creation_data, "context_creation_data", ContextCreationData)
return context_creation_data.executor_def.executor_creation_fn(
InitExecutorContext(
pipeline=context_creation_data.pipeline,
mode_def=context_creation_data.mode_def,
executor_def=context_creation_data.executor_def,
pipeline_run=context_creation_data.pipeline_run,
environment_config=context_creation_data.environment_config,
executor_config=context_creation_data.environment_config.execution.execution_engine_config,
intermediate_storage_def=context_creation_data.intermediate_storage_def,
instance=context_creation_data.instance,
)
)