def create_test_pipeline_execution_context(logger_defs=None):
loggers = check.opt_dict_param(
logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition
)
mode_def = ModeDefinition(logger_defs=loggers)
pipeline_def = PipelineDefinition(
name='test_legacy_context', solid_defs=[], mode_defs=[mode_def]
)
environment_dict = {'loggers': {key: {} for key in loggers}}
pipeline_run = PipelineRun(
pipeline_name='test_legacy_context', environment_dict=environment_dict
)
instance = DagsterInstance.ephemeral()
execution_plan = create_execution_plan(pipeline=pipeline_def, environment_dict=environment_dict)
creation_data = create_context_creation_data(
pipeline_def, environment_dict, pipeline_run, instance, execution_plan
)
log_manager = create_log_manager(creation_data)
scoped_resources_builder = ScopedResourcesBuilder()
executor_config = create_executor_config(creation_data)
return construct_pipeline_execution_context(
context_creation_data=creation_data,
scoped_resources_builder=scoped_resources_builder,
system_storage_data=SystemStorageData(
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_instance(instance, pipeline_run.run_id),
),
log_manager=log_manager,
executor_config=executor_config,
raise_on_error=True,
)
def create_test_pipeline_execution_context(
loggers=None, resources=None, tags=None, run_config_loggers=None
):
run_id = str(uuid.uuid4())
loggers = check.opt_dict_param(loggers, 'loggers', key_type=str, value_type=LoggerDefinition)
mode_def = ModeDefinition(loggers=loggers)
pipeline_def = PipelineDefinition(
name='test_legacy_context', solids=[], mode_definitions=[mode_def]
)
run_config_loggers = check.opt_list_param(
run_config_loggers, 'run_config_loggers', of_type=logging.Logger
)
run_config = RunConfig(run_id, tags=tags, loggers=run_config_loggers)
environment_config = create_environment_config(
pipeline_def, {'loggers': {key: {} for key in loggers}}
)
loggers = _create_loggers(environment_config, run_config, pipeline_def, mode_def)
log_manager = DagsterLogManager(run_config.run_id, {}, loggers)
return construct_pipeline_execution_context(
run_config=run_config,
pipeline_def=pipeline_def,
resources=resources,
environment_config=environment_config,
run_storage=InMemoryRunStorage(),
intermediates_manager=InMemoryIntermediatesManager(),
log_manager=log_manager,
)
def create_mem_system_storage_data(init_context):
return SystemStorageData(
run_storage=InMemoryRunStorage(),
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_run_id(
init_context.run_config.run_id),
)
def create_test_pipeline_execution_context(logger_defs=None):
from dagster.core.storage.intermediates_manager import build_in_mem_intermediates_storage
loggers = check.opt_dict_param(
logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition
)
mode_def = ModeDefinition(logger_defs=loggers)
pipeline_def = PipelineDefinition(
name='test_legacy_context', solid_defs=[], mode_defs=[mode_def]
)
run_config = {'loggers': {key: {} for key in loggers}}
pipeline_run = PipelineRun(pipeline_name='test_legacy_context', run_config=run_config)
instance = DagsterInstance.ephemeral()
execution_plan = create_execution_plan(pipeline=pipeline_def, run_config=run_config)
creation_data = create_context_creation_data(execution_plan, run_config, pipeline_run, instance)
log_manager = create_log_manager(creation_data)
scoped_resources_builder = ScopedResourcesBuilder()
executor = create_executor(creation_data)
return SystemPipelineExecutionContext(
construct_execution_context_data(
context_creation_data=creation_data,
scoped_resources_builder=scoped_resources_builder,
intermediate_storage=build_in_mem_intermediates_storage(pipeline_run.run_id),
system_storage_data=SystemStorageData(
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_instance(instance, pipeline_run.run_id),
),
log_manager=log_manager,
retries=executor.retries,
raise_on_error=True,
),
executor=executor,
log_manager=log_manager,
)
def create_test_pipeline_execution_context(
logger_defs=None, scoped_resources_builder=None, tags=None, run_config_loggers=None
):
run_id = str(uuid.uuid4())
loggers = check.opt_dict_param(
logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition
)
mode_def = ModeDefinition(logger_defs=loggers)
pipeline_def = PipelineDefinition(
name='test_legacy_context', solid_defs=[], mode_defs=[mode_def]
)
run_config_loggers = check.opt_list_param(
run_config_loggers, 'run_config_loggers', of_type=logging.Logger
)
run_config = RunConfig(run_id, tags=tags, loggers=run_config_loggers)
environment_dict = {'loggers': {key: {} for key in loggers}}
creation_data = create_context_creation_data(pipeline_def, environment_dict, run_config)
log_manager = create_log_manager(creation_data)
scoped_resources_builder = check.opt_inst_param(
scoped_resources_builder,
'scoped_resources_builder',
ScopedResourcesBuilder,
default=ScopedResourcesBuilder(),
)
return construct_pipeline_execution_context(
context_creation_data=creation_data,
scoped_resources_builder=scoped_resources_builder,
system_storage_data=SystemStorageData(
run_storage=InMemoryRunStorage(),
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_run_id(run_id),
),
log_manager=log_manager,
)
def create_test_pipeline_execution_context(logger_defs=None,
scoped_resources_builder=None,
tags=None):
run_id = str(uuid.uuid4())
loggers = check.opt_dict_param(logger_defs,
'logger_defs',
key_type=str,
value_type=LoggerDefinition)
mode_def = ModeDefinition(logger_defs=loggers)
pipeline_def = PipelineDefinition(name='test_legacy_context',
solid_defs=[],
mode_defs=[mode_def])
run_config = RunConfig(run_id, tags=tags)
environment_dict = {'loggers': {key: {} for key in loggers}}
instance = DagsterInstance.ephemeral()
creation_data = create_context_creation_data(pipeline_def,
environment_dict, run_config,
instance)
log_manager = create_log_manager(creation_data)
scoped_resources_builder = check.opt_inst_param(
scoped_resources_builder,
'scoped_resources_builder',
ScopedResourcesBuilder,
default=ScopedResourcesBuilder(),
)
executor_config = create_executor_config(creation_data)
return construct_pipeline_execution_context(
context_creation_data=creation_data,
scoped_resources_builder=scoped_resources_builder,
system_storage_data=SystemStorageData(
intermediates_manager=InMemoryIntermediatesManager(),
file_manager=LocalFileManager.for_instance(instance, run_id),
),
log_manager=log_manager,
executor_config=executor_config,
raise_on_error=True,
)