def create_execution_plan_index(self,
external_pipeline,
environment_dict,
mode,
step_keys_to_execute=None):
check.inst_param(external_pipeline, 'external_pipeline',
ExternalPipeline)
check.dict_param(environment_dict, 'environment_dict')
check.str_param(mode, 'mode')
check.opt_list_param(step_keys_to_execute,
'step_keys_to_execute',
of_type=str)
return ExecutionPlanIndex(
execution_plan_snapshot=snapshot_from_execution_plan(
create_execution_plan(
pipeline=self.get_pipeline(external_pipeline.name),
environment_dict=environment_dict,
mode=mode,
step_keys_to_execute=step_keys_to_execute,
),
external_pipeline.pipeline_snapshot_id,
),
pipeline_index=external_pipeline.pipeline_index,
)
def test_run_created_in_0_7_9_snapshot_id_change():
test_dir = file_relative_path(
__file__, 'snapshot_0_7_9_shapshot_id_creation_change/sqlite')
with restore_directory(test_dir):
instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir))
# run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6'
old_pipeline_snapshot_id = '88528edde2ed64da3c39cca0da8ba2f7586c1a5d'
old_execution_plan_snapshot_id = '2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d'
pipeline_snapshot = instance.get_pipeline_snapshot(
old_pipeline_snapshot_id)
ep_snapshot = instance.get_execution_plan_snapshot(
old_execution_plan_snapshot_id)
# It is the pipeline snapshot that changed
# Verify that snapshot ids are not equal. This changed in 0.7.10
assert create_pipeline_snapshot_id(
pipeline_snapshot) != old_pipeline_snapshot_id
# We also changed execution plan schema in 0.7.11.post1
assert create_execution_plan_snapshot_id(
ep_snapshot) != old_execution_plan_snapshot_id
# This previously failed with a check error
assert ExecutionPlanIndex(ep_snapshot,
PipelineIndex(pipeline_snapshot))
def resolve_executionPlan(self, graphene_info):
if not (self._pipeline_run.execution_plan_snapshot_id
and self._pipeline_run.pipeline_snapshot_id):
return None
from .execution import DauphinExecutionPlan
instance = graphene_info.context.instance
pipeline_snapshot = instance.get_pipeline_snapshot(
self._pipeline_run.pipeline_snapshot_id)
execution_plan_snapshot = instance.get_execution_plan_snapshot(
self._pipeline_run.execution_plan_snapshot_id)
return (DauphinExecutionPlan(
ExecutionPlanIndex(
execution_plan_snapshot=execution_plan_snapshot,
pipeline_index=PipelineIndex(pipeline_snapshot),
)) if execution_plan_snapshot and pipeline_snapshot else None)
def get_pipeline_run_observable(graphene_info, run_id, after=None):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.str_param(run_id, 'run_id')
check.opt_int_param(after, 'after')
instance = graphene_info.context.instance
run = instance.get_run_by_id(run_id)
if not run:
def _get_error_observable(observer):
observer.on_next(
graphene_info.schema.
type_named('PipelineRunLogsSubscriptionFailure')(
missingRunId=run_id,
message='Could not load run with id {}'.format(run_id)))
return Observable.create(_get_error_observable) # pylint: disable=E1101
execution_plan_index = (ExecutionPlanIndex(
execution_plan_snapshot=instance.get_execution_plan_snapshot(
run.execution_plan_snapshot_id),
pipeline_index=PipelineIndex(
instance.get_pipeline_snapshot(run.pipeline_snapshot_id)),
) if run.pipeline_snapshot_id and run.execution_plan_snapshot_id else None)
# pylint: disable=E1101
return Observable.create(
PipelineRunObservableSubscribe(instance, run_id, after_cursor=after)
).map(lambda events: graphene_info.schema.type_named(
'PipelineRunLogsSubscriptionSuccess')(
run=graphene_info.schema.type_named('PipelineRun')(run),
messages=[
from_event_record(event, run.pipeline_name,
execution_plan_index) for event in events
],
))
def _do_execute_plan(graphene_info, execution_params, pipeline_def):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(execution_params, 'execution_params', ExecutionParams)
run_id = execution_params.execution_metadata.run_id
mode = execution_params.mode or pipeline_def.get_default_mode_name()
pipeline_run = graphene_info.context.instance.get_run_by_id(run_id)
execution_plan = create_execution_plan(
pipeline=pipeline_def,
environment_dict=execution_params.environment_dict,
mode=mode,
)
if not pipeline_run:
# TODO switch to raising a UserFacingError if the run_id cannot be found
# https://github.com/dagster-io/dagster/issues/1876
pipeline_run = graphene_info.context.instance.create_run_for_pipeline(
pipeline_def=pipeline_def,
execution_plan=execution_plan,
run_id=run_id,
environment_dict=execution_params.environment_dict,
mode=mode,
tags=execution_params.execution_metadata.tags or {},
)
execution_plan = create_execution_plan(
pipeline=pipeline_def,
environment_dict=execution_params.environment_dict,
mode=pipeline_run.mode,
)
if execution_params.step_keys:
for step_key in execution_params.step_keys:
if not execution_plan.has_step(step_key):
raise UserFacingGraphQLError(
graphene_info.schema.type_named('InvalidStepError')(
invalid_step_key=step_key))
execution_plan = execution_plan.build_subset_plan(
execution_params.step_keys)
event_logs = []
def _on_event_record(record):
if record.is_dagster_event:
event_logs.append(record)
graphene_info.context.instance.add_event_listener(run_id, _on_event_record)
execute_plan(
execution_plan=execution_plan,
environment_dict=execution_params.environment_dict,
pipeline_run=pipeline_run,
instance=graphene_info.context.instance,
)
dauphin_pipeline = DauphinPipeline.from_pipeline_def(pipeline_def)
execution_plan_index = ExecutionPlanIndex.from_plan_and_index(
execution_plan, pipeline_def.get_pipeline_index())
def to_graphql_event(event_record):
return from_dagster_event_record(event_record,
pipeline_run.pipeline_name,
execution_plan_index)
return graphene_info.schema.type_named('ExecutePlanSuccess')(
pipeline=dauphin_pipeline,
has_failures=any(
er for er in event_logs if er.is_dagster_event
and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE),
step_events=list(map(to_graphql_event, event_logs)),
raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)),
)