def _setup_resources(
self,
resource_defs,
resource_configs,
log_manager,
execution_plan,
pipeline_run,
resource_keys_to_init,
instance,
emit_persistent_events,
pipeline_def_for_backwards_compat,
):
"""
Drop-in replacement for
`dagster.core.execution.resources_init.resource_initialization_manager`. It uses a
`DagstermillResourceEventGenerationManager` and explicitly calls `teardown` on it
"""
generator = resource_initialization_event_generator(
resource_defs=resource_defs,
resource_configs=resource_configs,
log_manager=log_manager,
execution_plan=execution_plan,
pipeline_run=pipeline_run,
resource_keys_to_init=resource_keys_to_init,
instance=instance,
emit_persistent_events=emit_persistent_events,
pipeline_def_for_backwards_compat=pipeline_def_for_backwards_compat,
)
self.resource_manager = DagstermillResourceEventGenerationManager(
generator, ScopedResourcesBuilder)
return self.resource_manager
def _setup_resources(
self,
execution_plan,
environment_config,
pipeline_run,
log_manager,
resource_keys_to_init,
instance,
):
"""
Drop-in replacement for
`dagster.core.execution.resources_init.resource_initialization_manager`. It uses a
`DagstermillResourceEventGenerationManager` and explicitly calls `teardown` on it
"""
generator = resource_initialization_event_generator(
execution_plan,
environment_config,
pipeline_run,
log_manager,
resource_keys_to_init,
instance,
)
self.resource_manager = DagstermillResourceEventGenerationManager(
generator, ScopedResourcesBuilder)
return self.resource_manager
def test_clean_event_generator_exit():
"""Testing for generator cleanup
(see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/)
"""
from dagster.core.execution.context.init import InitResourceContext
from dagster.core.definitions.resource import ScopedResourcesBuilder
pipeline_def = gen_basic_resource_pipeline()
instance = DagsterInstance.ephemeral()
execution_plan = create_execution_plan(pipeline_def)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def, execution_plan=execution_plan)
log_manager = DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[])
resolved_run_config = ResolvedRunConfig.build(pipeline_def)
execution_plan = create_execution_plan(pipeline_def)
resource_name, resource_def = next(
iter(pipeline_def.get_default_mode().resource_defs.items()))
resource_context = InitResourceContext(
resource_def=resource_def,
resources=ScopedResourcesBuilder().build(None),
resource_config=None,
pipeline_run=pipeline_run,
instance=instance,
)
generator = single_resource_event_generator(resource_context,
resource_name, resource_def)
next(generator)
generator.close()
resource_defs = pipeline_def.get_mode_definition(resolved_run_config.mode)
generator = resource_initialization_event_generator(
resource_defs=resource_defs,
resource_configs=resolved_run_config.resources,
log_manager=log_manager,
execution_plan=execution_plan,
pipeline_run=pipeline_run,
resource_keys_to_init={"a"},
instance=instance,
emit_persistent_events=True,
pipeline_def_for_backwards_compat=pipeline_def,
)
next(generator)
generator.close()
generator = PlanExecutionContextManager( # pylint: disable=protected-access
pipeline=InMemoryPipeline(pipeline_def),
execution_plan=execution_plan,
run_config={},
pipeline_run=pipeline_run,
instance=instance,
retry_mode=RetryMode.DISABLED,
scoped_resources_builder_cm=resource_initialization_manager,
).get_generator()
next(generator)
generator.close()
def test_clean_event_generator_exit():
"""Testing for generator cleanup
(see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/)
"""
from dagster.core.execution.context.init import InitResourceContext
pipeline_def = gen_basic_resource_pipeline()
instance = DagsterInstance.ephemeral()
execution_plan = create_execution_plan(pipeline_def)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def, execution_plan=execution_plan)
log_manager = DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[])
environment_config = EnvironmentConfig.build(pipeline_def)
execution_plan = create_execution_plan(pipeline_def)
resource_name, resource_def = next(
iter(pipeline_def.get_default_mode().resource_defs.items()))
resource_context = InitResourceContext(
resource_def=resource_def,
resource_config=None,
pipeline_run=pipeline_run,
instance=instance,
)
generator = single_resource_event_generator(resource_context,
resource_name, resource_def)
next(generator)
generator.close()
resource_defs = execution_plan.pipeline_def.get_mode_definition(
environment_config.mode)
generator = resource_initialization_event_generator(
resource_defs=resource_defs,
resource_configs=environment_config.resources,
log_manager=log_manager,
execution_plan=execution_plan,
pipeline_run=pipeline_run,
resource_keys_to_init={"a"},
instance=instance,
resource_instances_to_override=None,
emit_persistent_events=True,
)
next(generator)
generator.close()
generator = PipelineExecutionContextManager( # pylint: disable=protected-access
execution_plan,
{},
pipeline_run,
instance,
resource_initialization_manager,
).get_generator()
next(generator)
generator.close()
def test_clean_event_generator_exit():
''' Testing for generator cleanup
(see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/)
'''
from dagster.core.execution.context.init import InitResourceContext
pipeline = gen_basic_resource_pipeline()
instance = DagsterInstance.ephemeral()
execution_plan = create_execution_plan(pipeline)
pipeline_run = instance.create_run_for_pipeline(
pipeline=pipeline, execution_plan=execution_plan)
log_manager = DagsterLogManager(run_id=pipeline_run.run_id,
logging_tags={},
loggers=[])
environment_config = EnvironmentConfig.build(pipeline)
execution_plan = create_execution_plan(pipeline)
resource_name, resource_def = next(
iter(pipeline.get_default_mode().resource_defs.items()))
resource_context = InitResourceContext(
pipeline_def=pipeline,
resource_def=resource_def,
resource_config=None,
run_id=make_new_run_id(),
)
generator = single_resource_event_generator(resource_context,
resource_name, resource_def)
next(generator)
generator.close()
generator = resource_initialization_event_generator(
execution_plan, environment_config, pipeline_run, log_manager, {'a'})
next(generator)
generator.close()
generator = pipeline_initialization_event_generator(
pipeline,
{},
pipeline_run,
instance,
execution_plan,
resource_initialization_manager,
)
next(generator)
generator.close()
