def test_execute_execute_plan_mutation_raw():
pipeline_name = 'sleepy_pipeline'
workspace = workspace_from_load_target(
ModuleTarget('dagster_examples.toys.sleepy', pipeline_name))
pipeline = ReconstructablePipeline.for_module(
'dagster_examples.toys.sleepy', pipeline_name)
instance = DagsterInstance.local_temp()
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline.get_definition())
variables = {
'executionParams': {
'runConfigData': {},
'mode': 'default',
'selector': {
'repositoryLocationName': pipeline_name,
'repositoryName': '<<unnamed>>',
'pipelineName': pipeline_name,
},
'executionMetadata': {
'runId': pipeline_run.run_id
},
}
}
result = execute_execute_plan_mutation_raw(workspace,
variables,
instance_ref=instance.get_ref())
seen_events = set()
for event in result:
seen_events.add((event.dagster_event.event_type_value, event.step_key))
assert seen_events == EXPECTED_EVENTS
def load_sleepy_workspace(instance):
return workspace_from_load_target(
PythonFileTarget(
file_relative_path(__file__, 'sleepy.py'), 'sleepy_pipeline', working_directory=None
),
instance,
)
def test_all_step_events(): # pylint: disable=too-many-locals
workspace = workspace_from_load_target(
PythonFileTarget(__file__, define_test_events_pipeline.__name__))
pipeline_def = define_test_events_pipeline()
mode = pipeline_def.get_default_mode_name()
instance = DagsterInstance.ephemeral()
execution_plan = create_execution_plan(pipeline_def, mode=mode)
pipeline_run = instance.create_run_for_pipeline(
pipeline_def=pipeline_def, execution_plan=execution_plan, mode=mode)
step_levels = execution_plan.topological_step_levels()
unhandled_events = STEP_EVENTS.copy()
# Exclude types that are not step events
ignored_events = {
'LogMessageEvent',
'PipelineStartEvent',
'PipelineSuccessEvent',
'PipelineInitFailureEvent',
'PipelineFailureEvent',
}
event_counts = defaultdict(int)
for step_level in step_levels:
for step in step_level:
variables = {
'executionParams': {
'selector': {
'repositoryLocationName': 'test_events',
'repositoryName': '<<unnamed>>',
'pipelineName': pipeline_def.name,
},
'runConfigData': {
'storage': {
'filesystem': {}
}
},
'mode': mode,
'executionMetadata': {
'runId': pipeline_run.run_id
},
'stepKeys': [step.key],
}
}
res = execute_query(
workspace,
EXECUTE_PLAN_MUTATION,
variables,
instance=instance,
)
# go through the same dict, decrement all the event records we've seen from the GraphQL
# response
if not res.get('errors'):
assert 'data' in res, res
assert 'executePlan' in res['data'], res
assert 'stepEvents' in res['data']['executePlan'], res
step_events = res['data']['executePlan']['stepEvents']
events = [
dagster_event_from_dict(e, pipeline_def.name)
for e in step_events
if e['__typename'] not in ignored_events
]
for event in events:
if event.step_key:
key = event.step_key + '.' + event.event_type_value
else:
key = event.event_type_value
event_counts[key] -= 1
unhandled_events -= {
DagsterEventType(e.event_type_value)
for e in events
}
else:
raise Exception(res['errors'])
# build up a dict, incrementing all the event records we've produced in the run storage
logs = instance.all_logs(pipeline_run.run_id)
for log in logs:
if not log.dagster_event or (DagsterEventType(
log.dagster_event.event_type_value) not in STEP_EVENTS.union(
set([DagsterEventType.ENGINE_EVENT]))):
continue
if log.dagster_event.step_key:
key = log.dagster_event.step_key + '.' + log.dagster_event.event_type_value
else:
key = log.dagster_event.event_type_value
event_counts[key] += 1
# Ensure we've processed all the events that were generated in the run storage
assert sum(event_counts.values()) == 0
# Ensure we've handled the universe of event types
# Why are these retry events not handled? Because right now there is no way to configure retries
# on executePlan -- this needs to change, and we should separate the ExecutionParams that get
# sent to executePlan fromm those that get sent to startPipelineExecution and friends
assert unhandled_events == {
DagsterEventType.STEP_UP_FOR_RETRY, DagsterEventType.STEP_RESTARTED
}
def load_sleepy_workspace():
return workspace_from_load_target(
PythonFileTarget(file_relative_path(__file__, 'sleepy.py'),
'sleepy_pipeline'))
