def _core_resource_initialization_event_generator( resource_defs: Dict[str, ResourceDefinition], resource_configs: Dict[str, ResourceConfig], resource_log_manager: DagsterLogManager, resource_managers: Deque[EventGenerationManager], execution_plan: Optional[ExecutionPlan], pipeline_run: Optional[PipelineRun], resource_keys_to_init: Optional[AbstractSet[str]], instance: Optional[DagsterInstance], emit_persistent_events: Optional[bool], pipeline_def_for_backwards_compat: Optional[PipelineDefinition], ): pipeline_name = None contains_generator = False if emit_persistent_events: check.invariant( pipeline_run and execution_plan, "If emit_persistent_events is enabled, then pipeline_run and execution_plan must be provided", ) pipeline_name = cast(PipelineRun, pipeline_run).pipeline_name resource_keys_to_init = check.opt_set_param(resource_keys_to_init, "resource_keys_to_init") resource_instances: Dict[str, "InitializedResource"] = {} resource_init_times = {} try: if emit_persistent_events and resource_keys_to_init: yield DagsterEvent.resource_init_start( cast(str, pipeline_name), cast(ExecutionPlan, execution_plan), resource_log_manager, resource_keys_to_init, ) resource_dependencies = _resolve_resource_dependencies(resource_defs) for level in toposort(resource_dependencies): for resource_name in level: resource_def = resource_defs[resource_name] if not resource_name in resource_keys_to_init: continue resource_fn = cast(Callable[[InitResourceContext], Any], resource_def.resource_fn) resource_context = InitResourceContext( resource_def=resource_def, resource_config=resource_configs[resource_name].config, pipeline_run=pipeline_run, # Add tags with information about the resource log_manager=resource_log_manager.with_tags( resource_name=resource_name, resource_fn_name=str(resource_fn.__name__), ), resource_instance_dict=resource_instances, required_resource_keys=resource_def.required_resource_keys, instance=instance, pipeline_def_for_backwards_compat= pipeline_def_for_backwards_compat, ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[ resource_name] = initialized_resource.resource resource_init_times[ resource_name] = initialized_resource.duration contains_generator = contains_generator or initialized_resource.is_generator resource_managers.append(manager) if emit_persistent_events and resource_keys_to_init: yield DagsterEvent.resource_init_success( cast(str, pipeline_name), cast(ExecutionPlan, execution_plan), resource_log_manager, resource_instances, resource_init_times, ) yield ScopedResourcesBuilder(resource_instances, contains_generator) except DagsterUserCodeExecutionError as dagster_user_error: # Can only end up in this state if we attempt to initialize a resource, so # resource_keys_to_init cannot be empty if emit_persistent_events: yield DagsterEvent.resource_init_failure( cast(str, pipeline_name), cast(ExecutionPlan, execution_plan), resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( dagster_user_error.original_exc_info), ) raise dagster_user_error
def _core_resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, resource_keys_to_init, resource_log_manager, resource_managers, instance, resource_instances_to_override, ): pipeline_def = execution_plan.pipeline_def resource_instances = {} mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode) resource_init_times = {} try: if resource_keys_to_init: yield DagsterEvent.resource_init_start( execution_plan, resource_log_manager, resource_keys_to_init, ) resource_dependencies = _resolve_resource_dependencies( mode_definition.resource_defs) for level in toposort(resource_dependencies): for resource_name in level: if resource_name in resource_instances_to_override: # use the given resource instances instead of re-initiating it from resource def resource_def = ResourceDefinition.hardcoded_resource( resource_instances_to_override[resource_name]) else: resource_def = mode_definition.resource_defs[resource_name] if not resource_name in resource_keys_to_init: continue resource_context = InitResourceContext( resource_def=resource_def, resource_config=environment_config.resources.get( resource_name, {}).get("config"), pipeline_run=pipeline_run, # Add tags with information about the resource log_manager=resource_log_manager.with_tags( resource_name=resource_name, resource_fn_name=str( resource_def.resource_fn.__name__), ), resource_instance_dict=resource_instances, required_resource_keys=resource_def.required_resource_keys, instance_for_backwards_compat=instance, pipeline_def_for_backwards_compat=pipeline_def, ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[ resource_name] = initialized_resource.resource resource_init_times[ resource_name] = initialized_resource.duration resource_managers.append(manager) if resource_keys_to_init: yield DagsterEvent.resource_init_success(execution_plan, resource_log_manager, resource_instances, resource_init_times) yield ScopedResourcesBuilder(resource_instances) except DagsterUserCodeExecutionError as dagster_user_error: yield DagsterEvent.resource_init_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( dagster_user_error.original_exc_info), ) raise dagster_user_error
def resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init ): check.inst_param(execution_plan, 'execution_plan', ExecutionPlan) check.inst_param(environment_config, 'environment_config', EnvironmentConfig) check.inst_param(pipeline_run, 'pipeline_run', PipelineRun) check.inst_param(log_manager, 'log_manager', DagsterLogManager) check.set_param(resource_keys_to_init, 'resource_keys_to_init', of_type=str) resource_instances = {} pipeline_def = execution_plan.pipeline_def mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode) resource_managers = deque() generator_closed = False resource_init_times = {} try: if resource_keys_to_init: yield DagsterEvent.resource_init_start( execution_plan, log_manager, resource_keys_to_init, ) for resource_name, resource_def in sorted(mode_definition.resource_defs.items()): if not resource_name in resource_keys_to_init: continue resource_context = InitResourceContext( pipeline_def=pipeline_def, resource_def=resource_def, resource_config=environment_config.resources.get(resource_name, {}).get('config'), run_id=pipeline_run.run_id, log_manager=log_manager, ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def ) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[resource_name] = initialized_resource.resource resource_init_times[resource_name] = initialized_resource.duration resource_managers.append(manager) if resource_keys_to_init: yield DagsterEvent.resource_init_success( execution_plan, log_manager, resource_init_times ) yield ScopedResourcesBuilder(resource_instances) except GeneratorExit: # Shouldn't happen, but avoid runtime-exception in case this generator gets GC-ed # (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/). generator_closed = True raise except DagsterUserCodeExecutionError as dagster_user_error: yield DagsterEvent.resource_init_failure( execution_plan, log_manager, resource_keys_to_init, serializable_error_info_from_exc_info(dagster_user_error.original_exc_info), ) raise dagster_user_error finally: if not generator_closed: error = None while len(resource_managers) > 0: manager = resource_managers.pop() try: for event in manager.generate_teardown_events(): yield event except DagsterUserCodeExecutionError as dagster_user_error: error = dagster_user_error if error: yield DagsterEvent.resource_teardown_failure( execution_plan, log_manager, resource_keys_to_init, serializable_error_info_from_exc_info(error.original_exc_info), )
def _core_resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, resource_keys_to_init, resource_log_manager, resource_managers, ): pipeline_def = execution_plan.pipeline_def resource_instances = {} mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode) resource_init_times = {} try: if resource_keys_to_init: yield DagsterEvent.resource_init_start( execution_plan, resource_log_manager, resource_keys_to_init, ) for resource_name, resource_def in sorted( mode_definition.resource_defs.items()): if not resource_name in resource_keys_to_init: continue resource_context = InitResourceContext( pipeline_def=pipeline_def, resource_def=resource_def, resource_config=environment_config.resources.get( resource_name, {}).get("config"), run_id=pipeline_run.run_id, # Add tags with information about the resource log_manager=resource_log_manager.with_tags( resource_name=resource_name, resource_fn_name=str(resource_def.resource_fn.__name__), ), ) manager = single_resource_generation_manager( resource_context, resource_name, resource_def) for event in manager.generate_setup_events(): if event: yield event initialized_resource = check.inst(manager.get_object(), InitializedResource) resource_instances[resource_name] = initialized_resource.resource resource_init_times[resource_name] = initialized_resource.duration resource_managers.append(manager) if resource_keys_to_init: yield DagsterEvent.resource_init_success(execution_plan, resource_log_manager, resource_instances, resource_init_times) yield ScopedResourcesBuilder(resource_instances) except DagsterUserCodeExecutionError as dagster_user_error: yield DagsterEvent.resource_init_failure( execution_plan, resource_log_manager, resource_keys_to_init, serializable_error_info_from_exc_info( dagster_user_error.original_exc_info), ) raise dagster_user_error