def mode_def(event_callback): return ModeDefinition( logger_defs={ "callback": construct_event_logger(event_callback), "console": colored_console_logger, } )
def test_execute_pipeline_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) pipeline = PipelineDefinition( name='basic_resource_pipeline', solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={'a': resource_a, 'b': resource_b}, logger_defs={'callback': construct_event_logger(event_callback)}, ) ], ) iterator = execute_pipeline_iterator( pipeline, environment_dict={'loggers': {'callback': {}}}, instance=DagsterInstance.local_temp(), ) event_type = None while event_type != 'STEP_START': event = next(iterator) event_type = event.event_type_value iterator.close() events = [record.dagster_event for record in records if record.is_dagster_event] messages = [record.user_message for record in records if not record.is_dagster_event] assert len([event for event in events if event.is_pipeline_failure]) > 0 assert len([message for message in messages if message == 'CLEANING A']) > 0 assert len([message for message in messages if message == 'CLEANING B']) > 0
def test_execute_pipeline_iterator(): with instance_for_test() as instance: records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) pipeline = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={"a": resource_a, "b": resource_b}, logger_defs={"callback": construct_event_logger(event_callback)}, ) ], ) iterator = execute_pipeline_iterator( pipeline, run_config={"loggers": {"callback": {}}}, instance=instance ) event_type = None while event_type != "STEP_START": event = next(iterator) event_type = event.event_type_value iterator.close() events = [record.dagster_event for record in records if record.is_dagster_event] messages = [record.user_message for record in records if not record.is_dagster_event] pipeline_failure_events = [event for event in events if event.is_pipeline_failure] assert len(pipeline_failure_events) == 1 assert "GeneratorExit" in pipeline_failure_events[0].pipeline_failure_data.error.message assert len([message for message in messages if message == "CLEANING A"]) > 0 assert len([message for message in messages if message == "CLEANING B"]) > 0
def _create_context_free_log_manager(run_config, pipeline_def): '''In the event of pipeline initialization failure, we want to be able to log the failure without a dependency on the ExecutionContext to initialize DagsterLogManager. Args: run_config (dagster.core.execution_context.RunConfig) pipeline_def (dagster.definitions.PipelineDefinition) ''' check.inst_param(run_config, 'run_config', RunConfig) check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) loggers = [] # Use the default logger for (logger_def, logger_config) in default_system_loggers(): loggers += [ logger_def.logger_fn( InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id) ) ] if run_config.event_callback: event_logger_def = construct_event_logger(run_config.event_callback) loggers += [ event_logger_def.logger_fn( InitLoggerContext({}, pipeline_def, event_logger_def, run_config.run_id) ) ] elif run_config.loggers: loggers += run_config.loggers return DagsterLogManager(run_config.run_id, get_logging_tags(run_config, pipeline_def), loggers)
def test_execute_canceled_state(): def event_callback(_record): pass with instance_for_test() as instance: pipeline_def = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={ "a": resource_a, "b": resource_b }, logger_defs={ "callback": construct_event_logger(event_callback) }, ) ], ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={ "loggers": { "callback": {} } }, mode="default", ).with_status(PipelineRunStatus.CANCELED) execute_run(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance) logs = instance.all_logs(pipeline_run.run_id) assert len(logs) == 1 assert ( "Not starting execution since the run was canceled before execution could start" in logs[0].message) iter_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={ "loggers": { "callback": {} } }, mode="default", ).with_status(PipelineRunStatus.CANCELED) iter_events = list( execute_run_iterator(InMemoryPipeline(pipeline_def), iter_run, instance=instance)) assert len(iter_events) == 1 assert ( "Not starting execution since the run was canceled before execution could start" in iter_events[0].message)
def test_execute_run_bad_state(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) with instance_for_test() as instance: pipeline_def = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={"a": resource_a, "b": resource_b}, logger_defs={"callback": construct_event_logger(event_callback)}, ) ], ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={"loggers": {"callback": {}}}, mode="default", ).with_status(PipelineRunStatus.SUCCESS) with pytest.raises( check.CheckError, match=r"Pipeline run basic_resource_pipeline \({}\) in state" r" PipelineRunStatus.SUCCESS, expected PipelineRunStatus.NOT_STARTED".format( pipeline_run.run_id ), ): execute_run(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance)
def mode_def(event_callback): return ModeDefinition( loggers={ 'callback': construct_event_logger(event_callback), 'console': colored_console_logger, } )
def test_single_step_resource_event_logs(): # Test to attribute logs for single-step plans which are often the representation of # sub-plans in a multiprocessing execution environment. Most likely will need to be rewritten # with the refactor detailed in https://github.com/dagster-io/dagster/issues/2239 USER_SOLID_MESSAGE = "I AM A SOLID" USER_RESOURCE_MESSAGE = "I AM A RESOURCE" events = [] def event_callback(record): assert isinstance(record, EventRecord) events.append(record) @solid(required_resource_keys={"a"}) def resource_solid(context): context.log.info(USER_SOLID_MESSAGE) @resource def resource_a(context): context.log.info(USER_RESOURCE_MESSAGE) return "A" the_pipeline = PipelineDefinition( name="resource_logging_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={"a": resource_a}, logger_defs={ "callback": construct_event_logger(event_callback) }, ) ], ) with instance_for_test() as instance: pipeline_run = instance.create_run_for_pipeline( the_pipeline, run_config={"loggers": { "callback": {} }}, step_keys_to_execute=["resource_solid"], ) result = execute_run(InMemoryPipeline(the_pipeline), pipeline_run, instance) assert result.success log_messages = [ event for event in events if isinstance(event, EventRecord) and event.level == coerce_valid_log_level("INFO") ] assert len(log_messages) == 2 resource_log_message = next( iter([ message for message in log_messages if message.user_message == USER_RESOURCE_MESSAGE ])) assert resource_log_message.step_key == "resource_solid"
def test_single_step_resource_event_logs(): # Test to attribute logs for single-step plans which are often the representation of # sub-plans in a multiprocessing execution environment. Most likely will need to be rewritten # with the refactor detailed in https://github.com/dagster-io/dagster/issues/2239 USER_SOLID_MESSAGE = 'I AM A SOLID' USER_RESOURCE_MESSAGE = 'I AM A RESOURCE' events = [] def event_callback(record): assert isinstance(record, EventRecord) events.append(record) @solid(required_resource_keys={'a'}) def resource_solid(context): context.log.info(USER_SOLID_MESSAGE) @resource def resource_a(context): context.log.info(USER_RESOURCE_MESSAGE) return 'A' pipeline = PipelineDefinition( name='resource_logging_pipeline', solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={'a': resource_a}, logger_defs={ 'callback': construct_event_logger(event_callback) }, ) ], ) instance = DagsterInstance.local_temp() pipeline_run = instance.create_run_for_pipeline( pipeline, run_config={'loggers': { 'callback': {} }}, step_keys_to_execute=['resource_solid.compute'], ) result = execute_run(InMemoryExecutablePipeline(pipeline), pipeline_run, instance) assert result.success log_messages = [ event for event in events if isinstance(event, LogMessageRecord) ] assert len(log_messages) == 2 resource_log_message = next( iter([ message for message in log_messages if message.user_message == USER_RESOURCE_MESSAGE ])) assert resource_log_message.step_key == 'resource_solid.compute'
def test_execute_plan_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) with instance_for_test() as instance: pipeline = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={ "a": resource_a, "b": resource_b }, logger_defs={ "callback": construct_event_logger(event_callback) }, ) ], ) run_config = {"loggers": {"callback": {}}} execution_plan = create_execution_plan(pipeline, run_config=run_config) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline, run_config={"loggers": { "callback": {} }}, execution_plan=execution_plan, ) iterator = execute_plan_iterator( execution_plan, InMemoryPipeline(pipeline), pipeline_run, instance, run_config=run_config, ) event_type = None while event_type != "STEP_START": event = next(iterator) event_type = event.event_type_value iterator.close() messages = [ record.user_message for record in records if not record.is_dagster_event ] assert len( [message for message in messages if message == "CLEANING A"]) > 0 assert len( [message for message in messages if message == "CLEANING B"]) > 0
def test_execute_plan_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) instance = DagsterInstance.local_temp() pipeline = PipelineDefinition( name='basic_resource_pipeline', solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={ 'a': resource_a, 'b': resource_b }, logger_defs={ 'callback': construct_event_logger(event_callback) }, ) ], ) environment_dict = {'loggers': {'callback': {}}} pipeline_run = instance.create_run( PipelineRun( pipeline_name=pipeline.name, run_id=make_new_run_id(), environment_dict={'loggers': { 'callback': {} }}, mode='default', status=PipelineRunStatus.NOT_STARTED, )) execution_plan = create_execution_plan(pipeline, environment_dict) iterator = execute_plan_iterator(execution_plan, pipeline_run, instance, environment_dict=environment_dict) event_type = None while event_type != 'STEP_START': event = next(iterator) event_type = event.event_type_value iterator.close() messages = [ record.user_message for record in records if not record.is_dagster_event ] assert len([message for message in messages if message == 'CLEANING A']) > 0 assert len([message for message in messages if message == 'CLEANING B']) > 0
def test_execute_plan_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) instance = DagsterInstance.local_temp() pipeline = PipelineDefinition( name='basic_resource_pipeline', solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={ 'a': resource_a, 'b': resource_b }, logger_defs={ 'callback': construct_event_logger(event_callback) }, ) ], ) run_config = {'loggers': {'callback': {}}} execution_plan = create_execution_plan(pipeline, run_config=run_config) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline, run_config={'loggers': { 'callback': {} }}, execution_plan=execution_plan, ) iterator = execute_plan_iterator(execution_plan, pipeline_run, instance, run_config=run_config) event_type = None while event_type != 'STEP_START': event = next(iterator) event_type = event.event_type_value iterator.close() messages = [ record.user_message for record in records if not record.is_dagster_event ] assert len([message for message in messages if message == 'CLEANING A']) > 0 assert len([message for message in messages if message == 'CLEANING B']) > 0
def test_execute_run_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) instance = DagsterInstance.local_temp() pipeline_def = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={ "a": resource_a, "b": resource_b }, logger_defs={ "callback": construct_event_logger(event_callback) }, ) ], ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={"loggers": { "callback": {} }}, mode="default", ) iterator = execute_run_iterator(InMemoryExecutablePipeline(pipeline_def), pipeline_run, instance=instance) event_type = None while event_type != "STEP_START": event = next(iterator) event_type = event.event_type_value iterator.close() events = [ record.dagster_event for record in records if record.is_dagster_event ] messages = [ record.user_message for record in records if not record.is_dagster_event ] assert len([event for event in events if event.is_pipeline_failure]) > 0 assert len([message for message in messages if message == "CLEANING A"]) > 0 assert len([message for message in messages if message == "CLEANING B"]) > 0
def test_execute_run_iterator(): records = [] def event_callback(record): assert isinstance(record, EventRecord) records.append(record) with instance_for_test() as instance: pipeline_def = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={"a": resource_a, "b": resource_b}, logger_defs={"callback": construct_event_logger(event_callback)}, ) ], ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={"loggers": {"callback": {}}}, mode="default", ) iterator = execute_run_iterator( InMemoryPipeline(pipeline_def), pipeline_run, instance=instance ) event_type = None while event_type != "STEP_START": event = next(iterator) event_type = event.event_type_value iterator.close() events = [record.dagster_event for record in records if record.is_dagster_event] messages = [record.user_message for record in records if not record.is_dagster_event] pipeline_failure_events = [event for event in events if event.is_pipeline_failure] assert len(pipeline_failure_events) == 1 assert "GeneratorExit" in pipeline_failure_events[0].pipeline_failure_data.error.message assert len([message for message in messages if message == "CLEANING A"]) > 0 assert len([message for message in messages if message == "CLEANING B"]) > 0 pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={"loggers": {"callback": {}}}, mode="default", ).with_status(PipelineRunStatus.SUCCESS) with pytest.raises( check.CheckError, match=r"Pipeline run basic_resource_pipeline \({}\) in state" r" PipelineRunStatus.SUCCESS, expected PipelineRunStatus.NOT_STARTED".format( pipeline_run.run_id ), ): execute_run_iterator(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance)
def create_log_manager(context_creation_data): check.inst_param(context_creation_data, 'context_creation_data', ContextCreationData) pipeline_def, mode_def, environment_config, run_config = ( context_creation_data.pipeline_def, context_creation_data.mode_def, context_creation_data.environment_config, context_creation_data.run_config, ) loggers = [] for logger_key, logger_def in mode_def.loggers.items() or default_loggers( ).items(): if logger_key in environment_config.loggers: loggers.append( logger_def.logger_fn( InitLoggerContext( environment_config.loggers.get(logger_key, {}).get('config'), pipeline_def, logger_def, run_config.run_id, ))) if run_config.loggers: for logger in run_config.loggers: loggers.append(logger) if not loggers: for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id))) if run_config.event_callback: init_logger_context = InitLoggerContext({}, pipeline_def, logger_def, run_config.run_id) loggers.append( construct_event_logger( run_config.event_callback).logger_fn(init_logger_context)) return DagsterLogManager( run_id=run_config.run_id, logging_tags=get_logging_tags(context_creation_data.run_config, context_creation_data.pipeline_def), loggers=loggers, )
def _create_loggers(environment_config, run_config, pipeline_def, mode_def): check.inst_param(environment_config, 'environment_config', EnvironmentConfig) check.inst_param(run_config, 'run_config', RunConfig) check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) check.inst_param(mode_def, 'mode_def', ModeDefinition) loggers = [] for logger_key, logger_def in mode_def.loggers.items() or default_loggers().items(): if logger_key in environment_config.loggers: loggers.append( logger_def.logger_fn( InitLoggerContext( environment_config.loggers.get(logger_key, {}).get('config'), pipeline_def, logger_def, run_config.run_id, ) ) ) if run_config.loggers: for logger in run_config.loggers: loggers.append(logger) if not loggers: for (logger_def, logger_config) in default_system_loggers(): loggers.append( logger_def.logger_fn( InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id) ) ) if run_config.event_callback: init_logger_context = InitLoggerContext({}, pipeline_def, logger_def, run_config.run_id) loggers.append( construct_event_logger(run_config.event_callback).logger_fn(init_logger_context) ) return loggers
def test_execute_run_iterator(): records = [] def event_callback(record): assert isinstance(record, EventLogEntry) records.append(record) with instance_for_test() as instance: pipeline_def = PipelineDefinition( name="basic_resource_pipeline", solid_defs=[resource_solid], mode_defs=[ ModeDefinition( resource_defs={ "a": resource_a, "b": resource_b }, logger_defs={ "callback": construct_event_logger(event_callback) }, ) ], ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={"loggers": { "callback": {} }}, mode="default", ) iterator = execute_run_iterator(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance) event_type = None while event_type != "STEP_START": event = next(iterator) event_type = event.event_type_value iterator.close() events = [ record.dagster_event for record in records if record.is_dagster_event ] messages = [ record.user_message for record in records if not record.is_dagster_event ] pipeline_failure_events = [ event for event in events if event.is_pipeline_failure ] assert len(pipeline_failure_events) == 1 assert "GeneratorExit" in pipeline_failure_events[ 0].pipeline_failure_data.error.message assert len( [message for message in messages if message == "CLEANING A"]) > 0 assert len( [message for message in messages if message == "CLEANING B"]) > 0 pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={ "loggers": { "callback": {} } }, mode="default", ).with_status(PipelineRunStatus.SUCCESS) with pytest.raises( Exception, match=r"basic_resource_pipeline \({}\) started a new " r"run while the run was already in state DagsterRunStatus.SUCCESS." .format(pipeline_run.run_id), ): execute_run_iterator(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance) with instance_for_test( overrides={ "run_launcher": { "module": "dagster_tests.daemon_tests.test_monitoring_daemon", "class": "TestRunLauncher", }, "run_monitoring": { "enabled": True }, }) as run_monitoring_instance: event = next( execute_run_iterator(InMemoryPipeline(pipeline_def), pipeline_run, instance=run_monitoring_instance)) assert ( "Ignoring a duplicate run that was started from somewhere other than the run monitor daemon" in event.message) with pytest.raises( check.CheckError, match= r"in state DagsterRunStatus.SUCCESS, expected STARTED or STARTING " r"because it's resuming from a run worker failure", ): execute_run_iterator( InMemoryPipeline(pipeline_def), pipeline_run, instance=run_monitoring_instance, resume_from_failure=True, ) pipeline_run = instance.create_run_for_pipeline( pipeline_def=pipeline_def, run_config={ "loggers": { "callback": {} } }, mode="default", ).with_status(PipelineRunStatus.CANCELED) events = list( execute_run_iterator(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance)) assert len(events) == 1 assert ( events[0].message == "Not starting execution since the run was canceled before execution could start" )