def topological_step_levels(self):
if self._topological_step_levels is None:
self._topological_step_levels = [[
self._step_index[step_key]
for step_key in sorted(step_key_level)
] for step_key_level in toposort(self.execution_deps())]
return self._topological_step_levels
def _get_steps_to_execute_by_level(step_dict, step_handles_to_execute,
executable_map):
return [[
cast(ExecutionStep, _get_step_by_key(step_dict, step_key))
for step_key in sorted(step_key_level)
] for step_key_level in toposort(
_get_executable_step_deps(step_dict, step_handles_to_execute,
executable_map))]
def get_steps_to_execute_by_level(self) -> List[List[ExecutionStep]]:
return [
[
cast(ExecutionStep, self.get_step_by_key(step_key))
for step_key in sorted(step_key_level)
]
for step_key_level in toposort(self.get_executable_step_deps())
]
def topological_step_levels(self):
return [[
self.step_dict[step_key] for step_key in sorted(step_key_level)
] for step_key_level in toposort(self.deps)]
def _core_resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
resource_log_manager: DagsterLogManager,
resource_managers: Deque[EventGenerationManager],
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[Set[str]],
instance: Optional[DagsterInstance],
resource_instances_to_override: Optional[Dict[str, "InitializedResource"]],
emit_persistent_events: Optional[bool],
):
if emit_persistent_events:
check.invariant(
execution_plan,
"If emit_persistent_events is enabled, then execution_plan must be provided",
)
resource_instances_to_override = check.opt_dict_param(
resource_instances_to_override, "resource_instances_to_override")
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init")
resource_instances: Dict[str, "InitializedResource"] = {}
resource_init_times = {}
try:
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan,
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
if resource_name in resource_instances_to_override:
# use the given resource instances instead of re-initiating it from resource def
resource_def = ResourceDefinition.hardcoded_resource(
resource_instances_to_override[resource_name])
else:
resource_def = resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=resource_configs[resource_name].config,
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(
resource_def.resource_fn.__name__),
),
resource_instance_dict=resource_instances,
required_resource_keys=resource_def.required_resource_keys,
instance=instance,
pipeline_def_for_backwards_compat=execution_plan.
pipeline_def if execution_plan else None,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
resource_managers.append(manager)
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_success(execution_plan,
resource_log_manager,
resource_instances,
resource_init_times)
yield ScopedResourcesBuilder(resource_instances)
except DagsterUserCodeExecutionError as dagster_user_error:
# Can only end up in this state if we attempt to initialize a resource, so
# resource_keys_to_init cannot be empty
if emit_persistent_events:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error
def _core_resource_initialization_event_generator(
execution_plan,
environment_config,
pipeline_run,
resource_keys_to_init,
resource_log_manager,
resource_managers,
instance,
):
pipeline_def = execution_plan.pipeline_def
resource_instances = {}
mode_definition = pipeline_def.get_mode_definition(pipeline_run.mode)
resource_init_times = {}
try:
if resource_keys_to_init:
yield DagsterEvent.resource_init_start(
execution_plan, resource_log_manager, resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(mode_definition.resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
resource_def = mode_definition.resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=environment_config.resources.get(resource_name, {}).get(
"config"
),
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_def.resource_fn.__name__),
),
resource_instance_dict=resource_instances,
required_resource_keys=resource_def.required_resource_keys,
instance_for_backwards_compat=instance,
pipeline_def_for_backwards_compat=pipeline_def,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def
)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(), InitializedResource)
resource_instances[resource_name] = initialized_resource.resource
resource_init_times[resource_name] = initialized_resource.duration
resource_managers.append(manager)
if resource_keys_to_init:
yield DagsterEvent.resource_init_success(
execution_plan, resource_log_manager, resource_instances, resource_init_times
)
yield ScopedResourcesBuilder(resource_instances)
except DagsterUserCodeExecutionError as dagster_user_error:
yield DagsterEvent.resource_init_failure(
execution_plan,
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(dagster_user_error.original_exc_info),
)
raise dagster_user_error
def get_all_steps_by_level(self) -> List[List[IExecutionStep]]:
return [[
self.get_step_by_key(step_key)
for step_key in sorted(step_key_level)
] for step_key_level in toposort(self.get_all_step_deps())]
def execution_step_levels(self):
return [[
self.step_dict[step_key] for step_key in sorted(step_key_level)
] for step_key_level in toposort(self.execution_deps())]
def get_steps_to_execute_by_level(self):
return [[
self.get_step_by_key(step_key)
for step_key in sorted(step_key_level)
] for step_key_level in toposort(self.get_executable_step_deps())]
def get_all_steps_by_level(self):
return [[
self.get_step_by_key(step_key)
for step_key in sorted(step_key_level)
] for step_key_level in toposort(self.get_all_step_deps())]
def _core_resource_initialization_event_generator(
resource_defs: Dict[str, ResourceDefinition],
resource_configs: Dict[str, ResourceConfig],
resource_log_manager: DagsterLogManager,
resource_managers: Deque[EventGenerationManager],
execution_plan: Optional[ExecutionPlan],
pipeline_run: Optional[PipelineRun],
resource_keys_to_init: Optional[AbstractSet[str]],
instance: Optional[DagsterInstance],
emit_persistent_events: Optional[bool],
pipeline_def_for_backwards_compat: Optional[PipelineDefinition],
):
pipeline_name = None
contains_generator = False
if emit_persistent_events:
check.invariant(
pipeline_run and execution_plan,
"If emit_persistent_events is enabled, then pipeline_run and execution_plan must be provided",
)
pipeline_name = cast(PipelineRun, pipeline_run).pipeline_name
resource_keys_to_init = check.opt_set_param(resource_keys_to_init,
"resource_keys_to_init")
resource_instances: Dict[str, "InitializedResource"] = {}
resource_init_times = {}
try:
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_start(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_keys_to_init,
)
resource_dependencies = _resolve_resource_dependencies(resource_defs)
for level in toposort(resource_dependencies):
for resource_name in level:
resource_def = resource_defs[resource_name]
if not resource_name in resource_keys_to_init:
continue
resource_fn = cast(Callable[[InitResourceContext], Any],
resource_def.resource_fn)
resources = ScopedResourcesBuilder(resource_instances).build(
resource_def.required_resource_keys)
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=resource_configs[resource_name].config,
pipeline_run=pipeline_run,
# Add tags with information about the resource
log_manager=resource_log_manager.with_tags(
resource_name=resource_name,
resource_fn_name=str(resource_fn.__name__),
),
resources=resources,
instance=instance,
pipeline_def_for_backwards_compat=
pipeline_def_for_backwards_compat,
)
manager = single_resource_generation_manager(
resource_context, resource_name, resource_def)
for event in manager.generate_setup_events():
if event:
yield event
initialized_resource = check.inst(manager.get_object(),
InitializedResource)
resource_instances[
resource_name] = initialized_resource.resource
resource_init_times[
resource_name] = initialized_resource.duration
contains_generator = contains_generator or initialized_resource.is_generator
resource_managers.append(manager)
if emit_persistent_events and resource_keys_to_init:
yield DagsterEvent.resource_init_success(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_instances,
resource_init_times,
)
yield ScopedResourcesBuilder(resource_instances, contains_generator)
except DagsterUserCodeExecutionError as dagster_user_error:
# Can only end up in this state if we attempt to initialize a resource, so
# resource_keys_to_init cannot be empty
if emit_persistent_events:
yield DagsterEvent.resource_init_failure(
cast(str, pipeline_name),
cast(ExecutionPlan, execution_plan),
resource_log_manager,
resource_keys_to_init,
serializable_error_info_from_exc_info(
dagster_user_error.original_exc_info),
)
raise dagster_user_error