def _setup_resources( self, resource_defs, resource_configs, log_manager, execution_plan, pipeline_run, resource_keys_to_init, instance, emit_persistent_events, pipeline_def_for_backwards_compat, ): """ Drop-in replacement for `dagster.core.execution.resources_init.resource_initialization_manager`. It uses a `DagstermillResourceEventGenerationManager` and explicitly calls `teardown` on it """ generator = resource_initialization_event_generator( resource_defs=resource_defs, resource_configs=resource_configs, log_manager=log_manager, execution_plan=execution_plan, pipeline_run=pipeline_run, resource_keys_to_init=resource_keys_to_init, instance=instance, emit_persistent_events=emit_persistent_events, pipeline_def_for_backwards_compat=pipeline_def_for_backwards_compat, ) self.resource_manager = DagstermillResourceEventGenerationManager( generator, ScopedResourcesBuilder) return self.resource_manager
def _setup_resources( self, execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init, instance, ): """ Drop-in replacement for `dagster.core.execution.resources_init.resource_initialization_manager`. It uses a `DagstermillResourceEventGenerationManager` and explicitly calls `teardown` on it """ generator = resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, log_manager, resource_keys_to_init, instance, ) self.resource_manager = DagstermillResourceEventGenerationManager( generator, ScopedResourcesBuilder) return self.resource_manager
def test_clean_event_generator_exit(): """Testing for generator cleanup (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/) """ from dagster.core.execution.context.init import InitResourceContext from dagster.core.definitions.resource import ScopedResourcesBuilder pipeline_def = gen_basic_resource_pipeline() instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline_def) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, execution_plan=execution_plan) log_manager = DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]) resolved_run_config = ResolvedRunConfig.build(pipeline_def) execution_plan = create_execution_plan(pipeline_def) resource_name, resource_def = next( iter(pipeline_def.get_default_mode().resource_defs.items())) resource_context = InitResourceContext( resource_def=resource_def, resources=ScopedResourcesBuilder().build(None), resource_config=None, pipeline_run=pipeline_run, instance=instance, ) generator = single_resource_event_generator(resource_context, resource_name, resource_def) next(generator) generator.close() resource_defs = pipeline_def.get_mode_definition(resolved_run_config.mode) generator = resource_initialization_event_generator( resource_defs=resource_defs, resource_configs=resolved_run_config.resources, log_manager=log_manager, execution_plan=execution_plan, pipeline_run=pipeline_run, resource_keys_to_init={"a"}, instance=instance, emit_persistent_events=True, pipeline_def_for_backwards_compat=pipeline_def, ) next(generator) generator.close() generator = PlanExecutionContextManager( # pylint: disable=protected-access pipeline=InMemoryPipeline(pipeline_def), execution_plan=execution_plan, run_config={}, pipeline_run=pipeline_run, instance=instance, retry_mode=RetryMode.DISABLED, scoped_resources_builder_cm=resource_initialization_manager, ).get_generator() next(generator) generator.close()
def test_clean_event_generator_exit(): """Testing for generator cleanup (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/) """ from dagster.core.execution.context.init import InitResourceContext pipeline_def = gen_basic_resource_pipeline() instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline_def) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, execution_plan=execution_plan) log_manager = DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]) environment_config = EnvironmentConfig.build(pipeline_def) execution_plan = create_execution_plan(pipeline_def) resource_name, resource_def = next( iter(pipeline_def.get_default_mode().resource_defs.items())) resource_context = InitResourceContext( resource_def=resource_def, resource_config=None, pipeline_run=pipeline_run, instance=instance, ) generator = single_resource_event_generator(resource_context, resource_name, resource_def) next(generator) generator.close() resource_defs = execution_plan.pipeline_def.get_mode_definition( environment_config.mode) generator = resource_initialization_event_generator( resource_defs=resource_defs, resource_configs=environment_config.resources, log_manager=log_manager, execution_plan=execution_plan, pipeline_run=pipeline_run, resource_keys_to_init={"a"}, instance=instance, resource_instances_to_override=None, emit_persistent_events=True, ) next(generator) generator.close() generator = PipelineExecutionContextManager( # pylint: disable=protected-access execution_plan, {}, pipeline_run, instance, resource_initialization_manager, ).get_generator() next(generator) generator.close()
def test_clean_event_generator_exit(): ''' Testing for generator cleanup (see https://amir.rachum.com/blog/2017/03/03/generator-cleanup/) ''' from dagster.core.execution.context.init import InitResourceContext pipeline = gen_basic_resource_pipeline() instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline) pipeline_run = instance.create_run_for_pipeline( pipeline=pipeline, execution_plan=execution_plan) log_manager = DagsterLogManager(run_id=pipeline_run.run_id, logging_tags={}, loggers=[]) environment_config = EnvironmentConfig.build(pipeline) execution_plan = create_execution_plan(pipeline) resource_name, resource_def = next( iter(pipeline.get_default_mode().resource_defs.items())) resource_context = InitResourceContext( pipeline_def=pipeline, resource_def=resource_def, resource_config=None, run_id=make_new_run_id(), ) generator = single_resource_event_generator(resource_context, resource_name, resource_def) next(generator) generator.close() generator = resource_initialization_event_generator( execution_plan, environment_config, pipeline_run, log_manager, {'a'}) next(generator) generator.close() generator = pipeline_initialization_event_generator( pipeline, {}, pipeline_run, instance, execution_plan, resource_initialization_manager, ) next(generator) generator.close()