def resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init, instance, resource_instances_to_override=None, ): check.inst_param(execution_plan, "execution_plan", ExecutionPlan) check.inst_param(environment_config, "environment_config", EnvironmentConfig) check.inst_param(pipeline_run, "pipeline_run", PipelineRun) check.inst_param(log_manager, "log_manager", DagsterLogManager) check.set_param(resource_keys_to_init, "resource_keys_to_init", of_type=str) check.inst_param(instance, "instance", DagsterInstance) check.opt_dict_param(resource_instances_to_override, "resource_instances_to_override") if execution_plan.step_handle_for_single_step_plans(): step = execution_plan.get_step(execution_plan.step_handle_for_single_step_plans()) resource_log_manager = log_manager.with_tags(**step.logging_tags) else: resource_log_manager = log_manager generator_closed = False resource_managers = deque() try: yield from _core_resource_initialization_event_generator( execution_plan=execution_plan, environment_config=environment_config, pipeline_run=pipeline_run, resource_keys_to_init=resource_keys_to_init, resource_log_manager=resource_log_manager, resource_managers=resource_managers, instance=instance, resource_instances_to_override=resource_instances_to_override, ) except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: yield from manager.generate_teardown_events() except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info(error.original_exc_info), )
def resource_initialization_event_generator(execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init): check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) check.inst_param(environment_config, 'environment_config', EnvironmentConfig) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) check.inst_param(log_manager, 'log_manager', DagsterLogManager) check.set_param(resource_keys_to_init, 'resource_keys_to_init', of_type=str) if execution_plan.step_key_for_single_step_plans(): step = execution_plan.get_step_by_key( execution_plan.step_key_for_single_step_plans()) resource_log_manager = log_manager.with_tags(**step.logging_tags) else: resource_log_manager = log_manager generator_closed = False resource_managers = deque() try: for event in _core_resource_initialization_event_generator( execution_plan=execution_plan, environment_config=environment_config, pipeline_run=pipeline_run, resource_keys_to_init=resource_keys_to_init, resource_log_manager=resource_log_manager, resource_managers=resource_managers, ): yield event except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: for event in manager.generate_teardown_events(): yield event except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( error.original_exc_info), )
def resource_initialization_event_generator( resource_defs: Dict[str, ResourceDefinition], resource_configs: Dict[str, ResourceConfig], log_manager: DagsterLogManager, execution_plan: Optional[ExecutionPlan], pipeline_run: Optional[PipelineRun], resource_keys_to_init: Optional[Set[str]], instance: Optional[DagsterInstance], resource_instances_to_override: Optional[Dict[str, "InitializedResource"]], emit_persistent_events: Optional[bool], ): check.inst_param(log_manager, "log_manager", DagsterLogManager) resource_keys_to_init = check.opt_set_param(resource_keys_to_init, "resource_keys_to_init", of_type=str) check.opt_inst_param(execution_plan, "execution_plan", ExecutionPlan) check.opt_inst_param(pipeline_run, "pipeline_run", PipelineRun) check.opt_inst_param(instance, "instance", DagsterInstance) check.opt_dict_param(resource_instances_to_override, "resource_instances_to_override") if execution_plan and execution_plan.step_handle_for_single_step_plans(): step = execution_plan.get_step( cast( StepHandleUnion, cast(ExecutionPlan, execution_plan).step_handle_for_single_step_plans(), )) resource_log_manager = log_manager.with_tags( **cast(ExecutionStep, step).logging_tags) else: resource_log_manager = log_manager generator_closed = False resource_managers: Deque[EventGenerationManager] = deque() try: yield from _core_resource_initialization_event_generator( resource_defs=resource_defs, resource_configs=resource_configs, resource_log_manager=resource_log_manager, resource_managers=resource_managers, execution_plan=execution_plan, pipeline_run=pipeline_run, resource_keys_to_init=resource_keys_to_init, instance=instance, resource_instances_to_override=resource_instances_to_override, emit_persistent_events=emit_persistent_events, ) except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: yield from manager.generate_teardown_events() except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( error.original_exc_info), )
def resource_initialization_event_generator(execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init): check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) check.inst_param(environment_config, 'environment_config', EnvironmentConfig) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) check.inst_param(log_manager, 'log_manager', DagsterLogManager) check.set_param(resource_keys_to_init, 'resource_keys_to_init', of_type=str) if execution_plan.step_key_for_single_step_plans(): step = execution_plan.get_step_by_key( execution_plan.step_key_for_single_step_plans()) resource_log_manager = log_manager.with_tags(**step.logging_tags) else: resource_log_manager = log_manager resource_instances = {} pipeline_def = execution_plan.pipeline_def mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode) resource_managers = deque() generator_closed = False resource_init_times = {} try: if resource_keys_to_init: yield DagsterEvent.resource_init_start( execution_plan, resource_log_manager, resource_keys_to_init, ) for resource_name, resource_def in sorted( mode_definition.resource_defs.items()): if not resource_name in resource_keys_to_init: continue resource_context = InitResourceContext( pipeline_def=pipeline_def, resource_def=resource_def, resource_config=environment_config.resources.get( resource_name, {}).get('config'), run_id=pipeline_run.run_id, # Add tags with information about the resource log_manager=resource_log_manager.with_tags( resource_name=resource_name, resource_fn_name=str(resource_def.resource_fn.__name__), ), ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[resource_name] = initialized_resource.resource resource_init_times[resource_name] = initialized_resource.duration resource_managers.append(manager) if resource_keys_to_init: yield DagsterEvent.resource_init_success(execution_plan, resource_log_manager, resource_instances, resource_init_times) yield ScopedResourcesBuilder(resource_instances) except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise except DagsterUserCodeExecutionError as dagster_user_error: yield DagsterEvent.resource_init_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( dagster_user_error.original_exc_info), ) raise dagster_user_error finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: for event in manager.generate_teardown_events(): yield event except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( error.original_exc_info), )