def _get_host_mode_executor(recon_pipeline, run_config, executor_defs, instance): execution_config = run_config.get("execution", {}) execution_config_type = Field(selector_for_named_defs(executor_defs), default_value={ executor_defs[0].name: {} }).config_type config_evr = process_config(execution_config_type, execution_config) if not config_evr.success: raise DagsterInvalidConfigError( "Error processing execution config {}".format(execution_config), config_evr.errors, execution_config, ) execution_config_value = config_evr.value executor_name, executor_config = ensure_single_item(execution_config_value) executor_defs_by_name = { executor_def.name: executor_def for executor_def in executor_defs } executor_def = executor_defs_by_name[executor_name] init_context = InitExecutorContext( job=recon_pipeline, executor_def=executor_def, executor_config=executor_config["config"], instance=instance, ) check_cross_process_constraints(init_context) return executor_def.executor_creation_fn(init_context)
def _get_host_mode_executor(recon_pipeline, run_config, get_executor_def_fn, instance): execution_config = run_config.get("execution") if execution_config: executor_name, executor_config = ensure_single_item(execution_config) else: executor_name = None executor_config = {} executor_def = get_executor_def_fn(executor_name) executor_config_type = def_config_field(executor_def).config_type config_evr = process_config(executor_config_type, executor_config) if not config_evr.success: raise DagsterInvalidConfigError( "Error in executor config for executor {}".format( executor_def.name), config_evr.errors, executor_config, ) executor_config_value = config_evr.value init_context = InitExecutorContext( pipeline=recon_pipeline, executor_def=executor_def, executor_config=executor_config_value["config"], instance=instance, ) check_cross_process_constraints(init_context) return executor_def.executor_creation_fn(init_context)
def create_executor(context_creation_data: ContextCreationData) -> "Executor": check.inst_param(context_creation_data, "context_creation_data", ContextCreationData) init_context = InitExecutorContext( job=context_creation_data.pipeline, executor_def=context_creation_data.executor_def, executor_config=context_creation_data.resolved_run_config.execution.execution_engine_config, instance=context_creation_data.instance, ) check_cross_process_constraints(init_context) return context_creation_data.executor_def.executor_creation_fn(init_context)
def create_executor(context_creation_data): check.inst_param(context_creation_data, "context_creation_data", ContextCreationData) init_context = InitExecutorContext( pipeline=context_creation_data.pipeline, executor_def=context_creation_data.executor_def, executor_config=context_creation_data.environment_config.execution. execution_engine_config, instance=context_creation_data.instance, ) check_cross_process_constraints(init_context) return context_creation_data.executor_def.executor_creation_fn( init_context)
def create_executor(context_creation_data): check.inst_param(context_creation_data, "context_creation_data", ContextCreationData) return context_creation_data.executor_def.executor_creation_fn( InitExecutorContext( pipeline=context_creation_data.pipeline, mode_def=context_creation_data.mode_def, executor_def=context_creation_data.executor_def, pipeline_run=context_creation_data.pipeline_run, environment_config=context_creation_data.environment_config, executor_config=context_creation_data.environment_config.execution.execution_engine_config, intermediate_storage_def=context_creation_data.intermediate_storage_def, instance=context_creation_data.instance, ) )