def resource_initialization_event_generator(
execution_plan,
environment_config,
pipeline_run,
log_manager,
resource_keys_to_init,
instance,
resource_instances_to_override=None,
):
check.inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.inst_param(environment_config, "environment_config", EnvironmentConfig)
check.inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.inst_param(log_manager, "log_manager", DagsterLogManager)
check.set_param(resource_keys_to_init, "resource_keys_to_init", of_type=str)
check.inst_param(instance, "instance", DagsterInstance)
check.opt_dict_param(resource_instances_to_override, "resource_instances_to_override")
if execution_plan.step_handle_for_single_step_plans():
step = execution_plan.get_step(execution_plan.step_handle_for_single_step_plans())
resource_log_manager = log_manager.with_tags(**step.logging_tags)
else:
resource_log_manager = log_manager
generator_closed = False
resource_managers = deque()
try:
yield from _core_resource_initialization_event_generator(
execution_plan=execution_plan,
environment_config=environment_config,
pipeline_run=pipeline_run,
resource_keys_to_init=resource_keys_to_init,
resource_log_manager=resource_log_manager,
resource_managers=resource_managers,
instance=instance,
resource_instances_to_override=resource_instances_to_override,
)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
yield from manager.generate_teardown_events()
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(error.original_exc_info),
)
def resource_initialization_event_generator(execution_plan, environment_config,
pipeline_run, log_manager,
resource_keys_to_init):
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(environment_config, 'environment_config',
EnvironmentConfig)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.inst_param(log_manager, 'log_manager', DagsterLogManager)
check.set_param(resource_keys_to_init,
'resource_keys_to_init',
of_type=str)
if execution_plan.step_key_for_single_step_plans():
step = execution_plan.get_step_by_key(
execution_plan.step_key_for_single_step_plans())
resource_log_manager = log_manager.with_tags(**step.logging_tags)
else:
resource_log_manager = log_manager
generator_closed = False
resource_managers = deque()
try:
for event in _core_resource_initialization_event_generator(
execution_plan=execution_plan,
environment_config=environment_config,
pipeline_run=pipeline_run,
resource_keys_to_init=resource_keys_to_init,
resource_log_manager=resource_log_manager,
resource_managers=resource_managers,
):
yield event
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
for event in manager.generate_teardown_events():
yield event
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
error.original_exc_info),
)
def resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
log_manager: DagsterLogManager,
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[Set[str]],
instance: Optional[DagsterInstance],
resource_instances_to_override: Optional[Dict[str, "InitializedResource"]],
emit_persistent_events: Optional[bool],
):
check.inst_param(log_manager, "log_manager", DagsterLogManager)
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init",
of_type=str)
check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan)
check.opt_inst_param(pipeline_run, "pipeline_run", PipelineRun)
check.opt_inst_param(instance, "instance", DagsterInstance)
check.opt_dict_param(resource_instances_to_override,
"resource_instances_to_override")
if execution_plan and execution_plan.step_handle_for_single_step_plans():
step = execution_plan.get_step(
cast(
StepHandleUnion,
cast(ExecutionPlan,
execution_plan).step_handle_for_single_step_plans(),
))
resource_log_manager = log_manager.with_tags(
**cast(ExecutionStep, step).logging_tags)
else:
resource_log_manager = log_manager
generator_closed = False
resource_managers: Deque[EventGenerationManager] = deque()
try:
yield from _core_resource_initialization_event_generator(
resource_defs=resource_defs,
resource_configs=resource_configs,
resource_log_manager=resource_log_manager,
resource_managers=resource_managers,
execution_plan=execution_plan,
pipeline_run=pipeline_run,
resource_keys_to_init=resource_keys_to_init,
instance=instance,
resource_instances_to_override=resource_instances_to_override,
emit_persistent_events=emit_persistent_events,
)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
yield from manager.generate_teardown_events()
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
error.original_exc_info),
)
def resource_initialization_event_generator(execution_plan, environment_config,
pipeline_run, log_manager,
resource_keys_to_init):
check.inst_param(execution_plan, 'execution_plan', ExecutionPlan)
check.inst_param(environment_config, 'environment_config',
EnvironmentConfig)
check.inst_param(pipeline_run, 'pipeline_run', PipelineRun)
check.inst_param(log_manager, 'log_manager', DagsterLogManager)
check.set_param(resource_keys_to_init,
'resource_keys_to_init',
of_type=str)
if execution_plan.step_key_for_single_step_plans():
step = execution_plan.get_step_by_key(
execution_plan.step_key_for_single_step_plans())
resource_log_manager = log_manager.with_tags(**step.logging_tags)
else:
resource_log_manager = log_manager
resource_instances = {}
pipeline_def = execution_plan.pipeline_def
mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode)
resource_managers = deque()
generator_closed = False
resource_init_times = {}
try:
if resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan,
resource_log_manager,
resource_keys_to_init,
)
for resource_name, resource_def in sorted(
mode_definition.resource_defs.items()):
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
pipeline_def=pipeline_def,
resource_def=resource_def,
resource_config=environment_config.resources.get(
resource_name, {}).get('config'),
run_id=pipeline_run.run_id,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_def.resource_fn.__name__),
),
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[resource_name] = initialized_resource.resource
resource_init_times[resource_name] = initialized_resource.duration
resource_managers.append(manager)
if resource_keys_to_init:
yield DagsterEvent.resource_init_success(execution_plan,
resource_log_manager,
resource_instances,
resource_init_times)
yield ScopedResourcesBuilder(resource_instances)
except GeneratorExit:
# Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed
# (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/).
generator_closed = True
raise
except DagsterUserCodeExecutionError as dagster_user_error:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
finally:
if not generator_closed:
error = None
while len(resource_managers) > 0:
manager = resource_managers.pop()
try:
for event in manager.generate_teardown_events():
yield event
except DagsterUserCodeExecutionError as dagster_user_error:
error = dagster_user_error
if error:
yield DagsterEvent.resource_teardown_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
error.original_exc_info),
)
