def _core_resource_initialization_event_generator(
execution_plan,
environment_config,
pipeline_run,
resource_keys_to_init,
resource_log_manager,
resource_managers,
):
pipeline_def = execution_plan.pipeline_def
resource_instances = {}
mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode)
resource_init_times = {}
try:
if resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan, resource_log_manager, resource_keys_to_init,
)
for resource_name, resource_def in sorted(mode_definition.resource_defs.items()):
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
pipeline_def=pipeline_def,
resource_def=resource_def,
resource_config=environment_config.resources.get(resource_name, {}).get("config"),
run_id=pipeline_run.run_id,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_def.resource_fn.__name__),
),
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def
)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(), InitializedResource)
resource_instances[resource_name] = initialized_resource.resource
resource_init_times[resource_name] = initialized_resource.duration
resource_managers.append(manager)
if resource_keys_to_init:
yield DagsterEvent.resource_init_success(
execution_plan, resource_log_manager, resource_instances, resource_init_times
)
yield ScopedResourcesBuilder(resource_instances)
except DagsterUserCodeExecutionError as dagster_user_error:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(dagster_user_error.original_exc_info),
)
raise dagster_user_error
def _core_resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
resource_log_manager: DagsterLogManager,
resource_managers: Deque[EventGenerationManager],
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[Set[str]],
instance: Optional[DagsterInstance],
resource_instances_to_override: Optional[Dict[str, "InitializedResource"]],
emit_persistent_events: Optional[bool],
):
if emit_persistent_events:
check.invariant(
execution_plan,
"If emit_persistent_events is enabled, then execution_plan must be provided",
)
resource_instances_to_override = check.opt_dict_param(
resource_instances_to_override, "resource_instances_to_override")
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init")
resource_instances: Dict[str, "InitializedResource"] = {}
resource_init_times = {}
try:
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan,
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
if resource_name in resource_instances_to_override:
# use the given resource instances instead of re-initiating it from resource def
resource_def = ResourceDefinition.hardcoded_resource(
resource_instances_to_override[resource_name])
else:
resource_def = resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=resource_configs[resource_name].config,
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(
resource_def.resource_fn.__name__),
),
resource_instance_dict=resource_instances,
required_resource_keys=resource_def.required_resource_keys,
instance=instance,
pipeline_def_for_backwards_compat=execution_plan.
pipeline_def if execution_plan else None,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
resource_managers.append(manager)
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_success(execution_plan,
resource_log_manager,
resource_instances,
resource_init_times)
yield ScopedResourcesBuilder(resource_instances)
except DagsterUserCodeExecutionError as dagster_user_error:
# Can only end up in this state if we attempt to initialize a resource, so
# resource_keys_to_init cannot be empty
if emit_persistent_events:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
def resource_initialization_event_generator(execution_plan, environment_config,
pipeline_run, log_manager,
resource_keys_to_init):
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(environment_config, 'environment_config',
EnvironmentConfig)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.inst_param(log_manager, 'log_manager', DagsterLogManager)
check.set_param(resource_keys_to_init,
'resource_keys_to_init',
of_type=str)
if execution_plan.step_key_for_single_step_plans():
step = execution_plan.get_step_by_key(
execution_plan.step_key_for_single_step_plans())
resource_log_manager = log_manager.with_tags(**step.logging_tags)
else:
resource_log_manager = log_manager
resource_instances = {}
pipeline_def = execution_plan.pipeline_def
mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode)
resource_managers = deque()
generator_closed = False
resource_init_times = {}
try:
if resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan,
resource_log_manager,
resource_keys_to_init,
)
for resource_name, resource_def in sorted(
mode_definition.resource_defs.items()):
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
pipeline_def=pipeline_def,
resource_def=resource_def,
resource_config=environment_config.resources.get(
resource_name, {}).get('config'),
run_id=pipeline_run.run_id,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_def.resource_fn.__name__),
),
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[resource_name] = initialized_resource.resource
resource_init_times[resource_name] = initialized_resource.duration
resource_managers.append(manager)
if resource_keys_to_init:
yield DagsterEvent.resource_init_success(execution_plan,
resource_log_manager,
resource_instances,
resource_init_times)
yield ScopedResourcesBuilder(resource_instances)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
except DagsterUserCodeExecutionError as dagster_user_error:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
for event in manager.generate_teardown_events():
yield event
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
error.original_exc_info),
)
def _core_resource_initialization_event_generator(
execution_plan,
environment_config,
pipeline_run,
resource_keys_to_init,
resource_log_manager,
resource_managers,
instance,
resource_instances_to_override,
):
pipeline_def = execution_plan.pipeline_def
resource_instances = {}
mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode)
resource_init_times = {}
try:
if resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan,
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(
mode_definition.resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
if resource_name in resource_instances_to_override:
# use the given resource instances instead of re-initiating it from resource def
resource_def = ResourceDefinition.hardcoded_resource(
resource_instances_to_override[resource_name])
else:
resource_def = mode_definition.resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=environment_config.resources.get(
resource_name, {}).get("config"),
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(
resource_def.resource_fn.__name__),
),
resource_instance_dict=resource_instances,
required_resource_keys=resource_def.required_resource_keys,
instance_for_backwards_compat=instance,
pipeline_def_for_backwards_compat=pipeline_def,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
resource_managers.append(manager)
if resource_keys_to_init:
yield DagsterEvent.resource_init_success(execution_plan,
resource_log_manager,
resource_instances,
resource_init_times)
yield ScopedResourcesBuilder(resource_instances)
except DagsterUserCodeExecutionError as dagster_user_error:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
def _core_resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
resource_log_manager: DagsterLogManager,
resource_managers: Deque[EventGenerationManager],
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[AbstractSet[str]],
instance: Optional[DagsterInstance],
emit_persistent_events: Optional[bool],
pipeline_def_for_backwards_compat: Optional[PipelineDefinition],
):
pipeline_name = None
contains_generator = False
if emit_persistent_events:
check.invariant(
pipeline_run and execution_plan,
"If emit_persistent_events is enabled, then pipeline_run and execution_plan must be provided",
)
pipeline_name = cast(PipelineRun, pipeline_run).pipeline_name
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init")
resource_instances: Dict[str, "InitializedResource"] = {}
resource_init_times = {}
try:
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_start(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
resource_def = resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_fn = cast(Callable[[InitResourceContext], Any],
resource_def.resource_fn)
resources = ScopedResourcesBuilder(resource_instances).build(
resource_def.required_resource_keys)
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=resource_configs[resource_name].config,
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_fn.__name__),
),
resources=resources,
instance=instance,
pipeline_def_for_backwards_compat=
pipeline_def_for_backwards_compat,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
contains_generator = contains_generator or initialized_resource.is_generator
resource_managers.append(manager)
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_success(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_instances,
resource_init_times,
)
yield ScopedResourcesBuilder(resource_instances, contains_generator)
except DagsterUserCodeExecutionError as dagster_user_error:
# Can only end up in this state if we attempt to initialize a resource, so
# resource_keys_to_init cannot be empty
if emit_persistent_events:
yield DagsterEvent.resource_init_failure(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error