def mode_def(event_callback):
return ModeDefinition(
logger_defs={
"callback": construct_event_logger(event_callback),
"console": colored_console_logger,
}
)
def test_execute_pipeline_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
pipeline = PipelineDefinition(
name='basic_resource_pipeline',
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={'a': resource_a, 'b': resource_b},
logger_defs={'callback': construct_event_logger(event_callback)},
)
],
)
iterator = execute_pipeline_iterator(
pipeline,
environment_dict={'loggers': {'callback': {}}},
instance=DagsterInstance.local_temp(),
)
event_type = None
while event_type != 'STEP_START':
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [record.dagster_event for record in records if record.is_dagster_event]
messages = [record.user_message for record in records if not record.is_dagster_event]
assert len([event for event in events if event.is_pipeline_failure]) > 0
assert len([message for message in messages if message == 'CLEANING A']) > 0
assert len([message for message in messages if message == 'CLEANING B']) > 0
def test_execute_pipeline_iterator():
with instance_for_test() as instance:
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
pipeline = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={"a": resource_a, "b": resource_b},
logger_defs={"callback": construct_event_logger(event_callback)},
)
],
)
iterator = execute_pipeline_iterator(
pipeline, run_config={"loggers": {"callback": {}}}, instance=instance
)
event_type = None
while event_type != "STEP_START":
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [record.dagster_event for record in records if record.is_dagster_event]
messages = [record.user_message for record in records if not record.is_dagster_event]
pipeline_failure_events = [event for event in events if event.is_pipeline_failure]
assert len(pipeline_failure_events) == 1
assert "GeneratorExit" in pipeline_failure_events[0].pipeline_failure_data.error.message
assert len([message for message in messages if message == "CLEANING A"]) > 0
assert len([message for message in messages if message == "CLEANING B"]) > 0
def _create_context_free_log_manager(run_config, pipeline_def):
'''In the event of pipeline initialization failure, we want to be able to log the failure
without a dependency on the ExecutionContext to initialize DagsterLogManager.
Args:
run_config (dagster.core.execution_context.RunConfig)
pipeline_def (dagster.definitions.PipelineDefinition)
'''
check.inst_param(run_config, 'run_config', RunConfig)
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
loggers = []
# Use the default logger
for (logger_def, logger_config) in default_system_loggers():
loggers += [
logger_def.logger_fn(
InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id)
)
]
if run_config.event_callback:
event_logger_def = construct_event_logger(run_config.event_callback)
loggers += [
event_logger_def.logger_fn(
InitLoggerContext({}, pipeline_def, event_logger_def, run_config.run_id)
)
]
elif run_config.loggers:
loggers += run_config.loggers
return DagsterLogManager(run_config.run_id, get_logging_tags(run_config, pipeline_def), loggers)
def test_execute_canceled_state():
def event_callback(_record):
pass
with instance_for_test() as instance:
pipeline_def = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={
"a": resource_a,
"b": resource_b
},
logger_defs={
"callback": construct_event_logger(event_callback)
},
)
],
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
run_config={
"loggers": {
"callback": {}
}
},
mode="default",
).with_status(PipelineRunStatus.CANCELED)
execute_run(InMemoryPipeline(pipeline_def),
pipeline_run,
instance=instance)
logs = instance.all_logs(pipeline_run.run_id)
assert len(logs) == 1
assert (
"Not starting execution since the run was canceled before execution could start"
in logs[0].message)
iter_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
run_config={
"loggers": {
"callback": {}
}
},
mode="default",
).with_status(PipelineRunStatus.CANCELED)
iter_events = list(
execute_run_iterator(InMemoryPipeline(pipeline_def),
iter_run,
instance=instance))
assert len(iter_events) == 1
assert (
"Not starting execution since the run was canceled before execution could start"
in iter_events[0].message)
def test_execute_run_bad_state():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
with instance_for_test() as instance:
pipeline_def = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={"a": resource_a, "b": resource_b},
logger_defs={"callback": construct_event_logger(event_callback)},
)
],
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def, run_config={"loggers": {"callback": {}}}, mode="default",
).with_status(PipelineRunStatus.SUCCESS)
with pytest.raises(
check.CheckError,
match=r"Pipeline run basic_resource_pipeline \({}\) in state"
r" PipelineRunStatus.SUCCESS, expected PipelineRunStatus.NOT_STARTED".format(
pipeline_run.run_id
),
):
execute_run(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance)
def mode_def(event_callback):
return ModeDefinition(
loggers={
'callback': construct_event_logger(event_callback),
'console': colored_console_logger,
}
)
def test_single_step_resource_event_logs():
# Test to attribute logs for single-step plans which are often the representation of
# sub-plans in a multiprocessing execution environment. Most likely will need to be rewritten
# with the refactor detailed in https://github.com/dagster-io/dagster/issues/2239
USER_SOLID_MESSAGE = "I AM A SOLID"
USER_RESOURCE_MESSAGE = "I AM A RESOURCE"
events = []
def event_callback(record):
assert isinstance(record, EventRecord)
events.append(record)
@solid(required_resource_keys={"a"})
def resource_solid(context):
context.log.info(USER_SOLID_MESSAGE)
@resource
def resource_a(context):
context.log.info(USER_RESOURCE_MESSAGE)
return "A"
the_pipeline = PipelineDefinition(
name="resource_logging_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={"a": resource_a},
logger_defs={
"callback": construct_event_logger(event_callback)
},
)
],
)
with instance_for_test() as instance:
pipeline_run = instance.create_run_for_pipeline(
the_pipeline,
run_config={"loggers": {
"callback": {}
}},
step_keys_to_execute=["resource_solid"],
)
result = execute_run(InMemoryPipeline(the_pipeline), pipeline_run,
instance)
assert result.success
log_messages = [
event for event in events if isinstance(event, EventRecord)
and event.level == coerce_valid_log_level("INFO")
]
assert len(log_messages) == 2
resource_log_message = next(
iter([
message for message in log_messages
if message.user_message == USER_RESOURCE_MESSAGE
]))
assert resource_log_message.step_key == "resource_solid"
def test_single_step_resource_event_logs():
# Test to attribute logs for single-step plans which are often the representation of
# sub-plans in a multiprocessing execution environment. Most likely will need to be rewritten
# with the refactor detailed in https://github.com/dagster-io/dagster/issues/2239
USER_SOLID_MESSAGE = 'I AM A SOLID'
USER_RESOURCE_MESSAGE = 'I AM A RESOURCE'
events = []
def event_callback(record):
assert isinstance(record, EventRecord)
events.append(record)
@solid(required_resource_keys={'a'})
def resource_solid(context):
context.log.info(USER_SOLID_MESSAGE)
@resource
def resource_a(context):
context.log.info(USER_RESOURCE_MESSAGE)
return 'A'
pipeline = PipelineDefinition(
name='resource_logging_pipeline',
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={'a': resource_a},
logger_defs={
'callback': construct_event_logger(event_callback)
},
)
],
)
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline,
run_config={'loggers': {
'callback': {}
}},
step_keys_to_execute=['resource_solid.compute'],
)
result = execute_run(InMemoryExecutablePipeline(pipeline), pipeline_run,
instance)
assert result.success
log_messages = [
event for event in events if isinstance(event, LogMessageRecord)
]
assert len(log_messages) == 2
resource_log_message = next(
iter([
message for message in log_messages
if message.user_message == USER_RESOURCE_MESSAGE
]))
assert resource_log_message.step_key == 'resource_solid.compute'
def test_execute_plan_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
with instance_for_test() as instance:
pipeline = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={
"a": resource_a,
"b": resource_b
},
logger_defs={
"callback": construct_event_logger(event_callback)
},
)
],
)
run_config = {"loggers": {"callback": {}}}
execution_plan = create_execution_plan(pipeline, run_config=run_config)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline,
run_config={"loggers": {
"callback": {}
}},
execution_plan=execution_plan,
)
iterator = execute_plan_iterator(
execution_plan,
InMemoryPipeline(pipeline),
pipeline_run,
instance,
run_config=run_config,
)
event_type = None
while event_type != "STEP_START":
event = next(iterator)
event_type = event.event_type_value
iterator.close()
messages = [
record.user_message for record in records
if not record.is_dagster_event
]
assert len(
[message for message in messages if message == "CLEANING A"]) > 0
assert len(
[message for message in messages if message == "CLEANING B"]) > 0
def test_execute_plan_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
instance = DagsterInstance.local_temp()
pipeline = PipelineDefinition(
name='basic_resource_pipeline',
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={
'a': resource_a,
'b': resource_b
},
logger_defs={
'callback': construct_event_logger(event_callback)
},
)
],
)
environment_dict = {'loggers': {'callback': {}}}
pipeline_run = instance.create_run(
PipelineRun(
pipeline_name=pipeline.name,
run_id=make_new_run_id(),
environment_dict={'loggers': {
'callback': {}
}},
mode='default',
status=PipelineRunStatus.NOT_STARTED,
))
execution_plan = create_execution_plan(pipeline, environment_dict)
iterator = execute_plan_iterator(execution_plan,
pipeline_run,
instance,
environment_dict=environment_dict)
event_type = None
while event_type != 'STEP_START':
event = next(iterator)
event_type = event.event_type_value
iterator.close()
messages = [
record.user_message for record in records
if not record.is_dagster_event
]
assert len([message
for message in messages if message == 'CLEANING A']) > 0
assert len([message
for message in messages if message == 'CLEANING B']) > 0
def test_execute_plan_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
instance = DagsterInstance.local_temp()
pipeline = PipelineDefinition(
name='basic_resource_pipeline',
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={
'a': resource_a,
'b': resource_b
},
logger_defs={
'callback': construct_event_logger(event_callback)
},
)
],
)
run_config = {'loggers': {'callback': {}}}
execution_plan = create_execution_plan(pipeline, run_config=run_config)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline,
run_config={'loggers': {
'callback': {}
}},
execution_plan=execution_plan,
)
iterator = execute_plan_iterator(execution_plan,
pipeline_run,
instance,
run_config=run_config)
event_type = None
while event_type != 'STEP_START':
event = next(iterator)
event_type = event.event_type_value
iterator.close()
messages = [
record.user_message for record in records
if not record.is_dagster_event
]
assert len([message
for message in messages if message == 'CLEANING A']) > 0
assert len([message
for message in messages if message == 'CLEANING B']) > 0
def test_execute_run_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
instance = DagsterInstance.local_temp()
pipeline_def = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={
"a": resource_a,
"b": resource_b
},
logger_defs={
"callback": construct_event_logger(event_callback)
},
)
],
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
run_config={"loggers": {
"callback": {}
}},
mode="default",
)
iterator = execute_run_iterator(InMemoryExecutablePipeline(pipeline_def),
pipeline_run,
instance=instance)
event_type = None
while event_type != "STEP_START":
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [
record.dagster_event for record in records if record.is_dagster_event
]
messages = [
record.user_message for record in records
if not record.is_dagster_event
]
assert len([event for event in events if event.is_pipeline_failure]) > 0
assert len([message
for message in messages if message == "CLEANING A"]) > 0
assert len([message
for message in messages if message == "CLEANING B"]) > 0
def test_execute_run_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventRecord)
records.append(record)
with instance_for_test() as instance:
pipeline_def = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={"a": resource_a, "b": resource_b},
logger_defs={"callback": construct_event_logger(event_callback)},
)
],
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def, run_config={"loggers": {"callback": {}}}, mode="default",
)
iterator = execute_run_iterator(
InMemoryPipeline(pipeline_def), pipeline_run, instance=instance
)
event_type = None
while event_type != "STEP_START":
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [record.dagster_event for record in records if record.is_dagster_event]
messages = [record.user_message for record in records if not record.is_dagster_event]
pipeline_failure_events = [event for event in events if event.is_pipeline_failure]
assert len(pipeline_failure_events) == 1
assert "GeneratorExit" in pipeline_failure_events[0].pipeline_failure_data.error.message
assert len([message for message in messages if message == "CLEANING A"]) > 0
assert len([message for message in messages if message == "CLEANING B"]) > 0
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def, run_config={"loggers": {"callback": {}}}, mode="default",
).with_status(PipelineRunStatus.SUCCESS)
with pytest.raises(
check.CheckError,
match=r"Pipeline run basic_resource_pipeline \({}\) in state"
r" PipelineRunStatus.SUCCESS, expected PipelineRunStatus.NOT_STARTED".format(
pipeline_run.run_id
),
):
execute_run_iterator(InMemoryPipeline(pipeline_def), pipeline_run, instance=instance)
def create_log_manager(context_creation_data):
check.inst_param(context_creation_data, 'context_creation_data',
ContextCreationData)
pipeline_def, mode_def, environment_config, run_config = (
context_creation_data.pipeline_def,
context_creation_data.mode_def,
context_creation_data.environment_config,
context_creation_data.run_config,
)
loggers = []
for logger_key, logger_def in mode_def.loggers.items() or default_loggers(
).items():
if logger_key in environment_config.loggers:
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
environment_config.loggers.get(logger_key,
{}).get('config'),
pipeline_def,
logger_def,
run_config.run_id,
)))
if run_config.loggers:
for logger in run_config.loggers:
loggers.append(logger)
if not loggers:
for (logger_def, logger_config) in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(logger_config, pipeline_def, logger_def,
run_config.run_id)))
if run_config.event_callback:
init_logger_context = InitLoggerContext({}, pipeline_def, logger_def,
run_config.run_id)
loggers.append(
construct_event_logger(
run_config.event_callback).logger_fn(init_logger_context))
return DagsterLogManager(
run_id=run_config.run_id,
logging_tags=get_logging_tags(context_creation_data.run_config,
context_creation_data.pipeline_def),
loggers=loggers,
)
def _create_loggers(environment_config, run_config, pipeline_def, mode_def):
check.inst_param(environment_config, 'environment_config', EnvironmentConfig)
check.inst_param(run_config, 'run_config', RunConfig)
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
check.inst_param(mode_def, 'mode_def', ModeDefinition)
loggers = []
for logger_key, logger_def in mode_def.loggers.items() or default_loggers().items():
if logger_key in environment_config.loggers:
loggers.append(
logger_def.logger_fn(
InitLoggerContext(
environment_config.loggers.get(logger_key, {}).get('config'),
pipeline_def,
logger_def,
run_config.run_id,
)
)
)
if run_config.loggers:
for logger in run_config.loggers:
loggers.append(logger)
if not loggers:
for (logger_def, logger_config) in default_system_loggers():
loggers.append(
logger_def.logger_fn(
InitLoggerContext(logger_config, pipeline_def, logger_def, run_config.run_id)
)
)
if run_config.event_callback:
init_logger_context = InitLoggerContext({}, pipeline_def, logger_def, run_config.run_id)
loggers.append(
construct_event_logger(run_config.event_callback).logger_fn(init_logger_context)
)
return loggers
def test_execute_run_iterator():
records = []
def event_callback(record):
assert isinstance(record, EventLogEntry)
records.append(record)
with instance_for_test() as instance:
pipeline_def = PipelineDefinition(
name="basic_resource_pipeline",
solid_defs=[resource_solid],
mode_defs=[
ModeDefinition(
resource_defs={
"a": resource_a,
"b": resource_b
},
logger_defs={
"callback": construct_event_logger(event_callback)
},
)
],
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
run_config={"loggers": {
"callback": {}
}},
mode="default",
)
iterator = execute_run_iterator(InMemoryPipeline(pipeline_def),
pipeline_run,
instance=instance)
event_type = None
while event_type != "STEP_START":
event = next(iterator)
event_type = event.event_type_value
iterator.close()
events = [
record.dagster_event for record in records
if record.is_dagster_event
]
messages = [
record.user_message for record in records
if not record.is_dagster_event
]
pipeline_failure_events = [
event for event in events if event.is_pipeline_failure
]
assert len(pipeline_failure_events) == 1
assert "GeneratorExit" in pipeline_failure_events[
0].pipeline_failure_data.error.message
assert len(
[message for message in messages if message == "CLEANING A"]) > 0
assert len(
[message for message in messages if message == "CLEANING B"]) > 0
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
run_config={
"loggers": {
"callback": {}
}
},
mode="default",
).with_status(PipelineRunStatus.SUCCESS)
with pytest.raises(
Exception,
match=r"basic_resource_pipeline \({}\) started a new "
r"run while the run was already in state DagsterRunStatus.SUCCESS."
.format(pipeline_run.run_id),
):
execute_run_iterator(InMemoryPipeline(pipeline_def),
pipeline_run,
instance=instance)
with instance_for_test(
overrides={
"run_launcher": {
"module":
"dagster_tests.daemon_tests.test_monitoring_daemon",
"class": "TestRunLauncher",
},
"run_monitoring": {
"enabled": True
},
}) as run_monitoring_instance:
event = next(
execute_run_iterator(InMemoryPipeline(pipeline_def),
pipeline_run,
instance=run_monitoring_instance))
assert (
"Ignoring a duplicate run that was started from somewhere other than the run monitor daemon"
in event.message)
with pytest.raises(
check.CheckError,
match=
r"in state DagsterRunStatus.SUCCESS, expected STARTED or STARTING "
r"because it's resuming from a run worker failure",
):
execute_run_iterator(
InMemoryPipeline(pipeline_def),
pipeline_run,
instance=run_monitoring_instance,
resume_from_failure=True,
)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
run_config={
"loggers": {
"callback": {}
}
},
mode="default",
).with_status(PipelineRunStatus.CANCELED)
events = list(
execute_run_iterator(InMemoryPipeline(pipeline_def),
pipeline_run,
instance=instance))
assert len(events) == 1
assert (
events[0].message ==
"Not starting execution since the run was canceled before execution could start"
)
