def create_test_pipeline_execution_context(logger_defs=None): loggers = check.opt_dict_param( logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition ) mode_def = ModeDefinition(logger_defs=loggers) pipeline_def = PipelineDefinition( name='test_legacy_context', solid_defs=[], mode_defs=[mode_def] ) environment_dict = {'loggers': {key: {} for key in loggers}} pipeline_run = PipelineRun( pipeline_name='test_legacy_context', environment_dict=environment_dict ) instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline=pipeline_def, environment_dict=environment_dict) creation_data = create_context_creation_data( pipeline_def, environment_dict, pipeline_run, instance, execution_plan ) log_manager = create_log_manager(creation_data) scoped_resources_builder = ScopedResourcesBuilder() executor_config = create_executor_config(creation_data) return construct_pipeline_execution_context( context_creation_data=creation_data, scoped_resources_builder=scoped_resources_builder, system_storage_data=SystemStorageData( intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_instance(instance, pipeline_run.run_id), ), log_manager=log_manager, executor_config=executor_config, raise_on_error=True, )
def create_test_pipeline_execution_context( loggers=None, resources=None, tags=None, run_config_loggers=None ): run_id = str(uuid.uuid4()) loggers = check.opt_dict_param(loggers, 'loggers', key_type=str, value_type=LoggerDefinition) mode_def = ModeDefinition(loggers=loggers) pipeline_def = PipelineDefinition( name='test_legacy_context', solids=[], mode_definitions=[mode_def] ) run_config_loggers = check.opt_list_param( run_config_loggers, 'run_config_loggers', of_type=logging.Logger ) run_config = RunConfig(run_id, tags=tags, loggers=run_config_loggers) environment_config = create_environment_config( pipeline_def, {'loggers': {key: {} for key in loggers}} ) loggers = _create_loggers(environment_config, run_config, pipeline_def, mode_def) log_manager = DagsterLogManager(run_config.run_id, {}, loggers) return construct_pipeline_execution_context( run_config=run_config, pipeline_def=pipeline_def, resources=resources, environment_config=environment_config, run_storage=InMemoryRunStorage(), intermediates_manager=InMemoryIntermediatesManager(), log_manager=log_manager, )
def create_mem_system_storage_data(init_context): return SystemStorageData( run_storage=InMemoryRunStorage(), intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_run_id( init_context.run_config.run_id), )
def create_test_pipeline_execution_context(logger_defs=None): from dagster.core.storage.intermediates_manager import build_in_mem_intermediates_storage loggers = check.opt_dict_param( logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition ) mode_def = ModeDefinition(logger_defs=loggers) pipeline_def = PipelineDefinition( name='test_legacy_context', solid_defs=[], mode_defs=[mode_def] ) run_config = {'loggers': {key: {} for key in loggers}} pipeline_run = PipelineRun(pipeline_name='test_legacy_context', run_config=run_config) instance = DagsterInstance.ephemeral() execution_plan = create_execution_plan(pipeline=pipeline_def, run_config=run_config) creation_data = create_context_creation_data(execution_plan, run_config, pipeline_run, instance) log_manager = create_log_manager(creation_data) scoped_resources_builder = ScopedResourcesBuilder() executor = create_executor(creation_data) return SystemPipelineExecutionContext( construct_execution_context_data( context_creation_data=creation_data, scoped_resources_builder=scoped_resources_builder, intermediate_storage=build_in_mem_intermediates_storage(pipeline_run.run_id), system_storage_data=SystemStorageData( intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_instance(instance, pipeline_run.run_id), ), log_manager=log_manager, retries=executor.retries, raise_on_error=True, ), executor=executor, log_manager=log_manager, )
def create_test_pipeline_execution_context( logger_defs=None, scoped_resources_builder=None, tags=None, run_config_loggers=None ): run_id = str(uuid.uuid4()) loggers = check.opt_dict_param( logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition ) mode_def = ModeDefinition(logger_defs=loggers) pipeline_def = PipelineDefinition( name='test_legacy_context', solid_defs=[], mode_defs=[mode_def] ) run_config_loggers = check.opt_list_param( run_config_loggers, 'run_config_loggers', of_type=logging.Logger ) run_config = RunConfig(run_id, tags=tags, loggers=run_config_loggers) environment_dict = {'loggers': {key: {} for key in loggers}} creation_data = create_context_creation_data(pipeline_def, environment_dict, run_config) log_manager = create_log_manager(creation_data) scoped_resources_builder = check.opt_inst_param( scoped_resources_builder, 'scoped_resources_builder', ScopedResourcesBuilder, default=ScopedResourcesBuilder(), ) return construct_pipeline_execution_context( context_creation_data=creation_data, scoped_resources_builder=scoped_resources_builder, system_storage_data=SystemStorageData( run_storage=InMemoryRunStorage(), intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_run_id(run_id), ), log_manager=log_manager, )
def create_test_pipeline_execution_context(logger_defs=None, scoped_resources_builder=None, tags=None): run_id = str(uuid.uuid4()) loggers = check.opt_dict_param(logger_defs, 'logger_defs', key_type=str, value_type=LoggerDefinition) mode_def = ModeDefinition(logger_defs=loggers) pipeline_def = PipelineDefinition(name='test_legacy_context', solid_defs=[], mode_defs=[mode_def]) run_config = RunConfig(run_id, tags=tags) environment_dict = {'loggers': {key: {} for key in loggers}} instance = DagsterInstance.ephemeral() creation_data = create_context_creation_data(pipeline_def, environment_dict, run_config, instance) log_manager = create_log_manager(creation_data) scoped_resources_builder = check.opt_inst_param( scoped_resources_builder, 'scoped_resources_builder', ScopedResourcesBuilder, default=ScopedResourcesBuilder(), ) executor_config = create_executor_config(creation_data) return construct_pipeline_execution_context( context_creation_data=creation_data, scoped_resources_builder=scoped_resources_builder, system_storage_data=SystemStorageData( intermediates_manager=InMemoryIntermediatesManager(), file_manager=LocalFileManager.for_instance(instance, run_id), ), log_manager=log_manager, executor_config=executor_config, raise_on_error=True, )