def get_external_execution_plan( self, external_pipeline, run_config, mode, step_keys_to_execute ): check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline) check.dict_param(run_config, "run_config") check.str_param(mode, "mode") check.opt_list_param(step_keys_to_execute, "step_keys_to_execute", of_type=str) execution_plan_snapshot_or_error = sync_get_external_execution_plan_grpc( api_client=self._handle.client, pipeline_origin=external_pipeline.get_external_origin(), run_config=run_config, mode=mode, pipeline_snapshot_id=external_pipeline.identifying_pipeline_snapshot_id, solid_selection=external_pipeline.solid_selection, step_keys_to_execute=step_keys_to_execute, ) if isinstance(execution_plan_snapshot_or_error, ExecutionPlanSnapshotErrorData): return execution_plan_snapshot_or_error return ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot_or_error, represented_pipeline=external_pipeline, )
def test_run_created_in_0_7_9_snapshot_id_change(): test_dir = file_relative_path( __file__, 'snapshot_0_7_9_shapshot_id_creation_change/sqlite') with restore_directory(test_dir): instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = '88528edde2ed64da3c39cca0da8ba2f7586c1a5d' old_execution_plan_snapshot_id = '2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d' with pytest.warns( UserWarning, match=re.escape( '"input_hydration_schema_key" is deprecated and will be removed in 0.10.0, use ' '"loader_schema_key" instead.'), ): historical_pipeline = instance.get_historical_pipeline( old_pipeline_snapshot_id) pipeline_snapshot = historical_pipeline.pipeline_snapshot ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 assert create_pipeline_snapshot_id( pipeline_snapshot) != old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExternalExecutionPlan(ep_snapshot, historical_pipeline)
def resolve_executionPlan(self, graphene_info): if not ( self._pipeline_run.execution_plan_snapshot_id and self._pipeline_run.pipeline_snapshot_id ): return None from .execution import DauphinExecutionPlan instance = graphene_info.context.instance historical_pipeline = instance.get_historical_pipeline( self._pipeline_run.pipeline_snapshot_id ) execution_plan_snapshot = instance.get_execution_plan_snapshot( self._pipeline_run.execution_plan_snapshot_id ) return ( DauphinExecutionPlan( ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot, represented_pipeline=historical_pipeline, ) ) if execution_plan_snapshot and historical_pipeline else None )
def get_external_execution_plan(self, external_pipeline, run_config, mode, step_keys_to_execute): check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) check.dict_param(run_config, 'run_config') check.str_param(mode, 'mode') check.opt_list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str) execution_plan_snapshot = sync_get_external_execution_plan_grpc( api_client=self._handle.client, pipeline_origin=external_pipeline.get_origin(), run_config=run_config, mode=mode, pipeline_snapshot_id=external_pipeline. identifying_pipeline_snapshot_id, solid_selection=external_pipeline.solid_selection, step_keys_to_execute=step_keys_to_execute, ) return ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot, represented_pipeline=external_pipeline, )
def get_external_execution_plan(self, external_pipeline, run_config, mode, step_keys_to_execute): from dagster.api.snapshot_execution_plan import sync_get_external_execution_plan check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) check.dict_param(run_config, 'run_config') check.str_param(mode, 'mode') check.opt_list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str) execution_plan_snapshot_or_error = sync_get_external_execution_plan( pipeline_origin=external_pipeline.get_origin(), solid_selection=external_pipeline.solid_selection, run_config=run_config, mode=mode, step_keys_to_execute=step_keys_to_execute, pipeline_snapshot_id=external_pipeline. identifying_pipeline_snapshot_id, ) if isinstance(execution_plan_snapshot_or_error, ExecutionPlanSnapshotErrorData): return execution_plan_snapshot_or_error return ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot_or_error, represented_pipeline=external_pipeline, )
def get_external_execution_plan(self, external_pipeline, run_config, mode, step_keys_to_execute): check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) check.dict_param(run_config, 'run_config') check.str_param(mode, 'mode') check.opt_list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str) return ExternalExecutionPlan( execution_plan_snapshot=snapshot_from_execution_plan( create_execution_plan( pipeline=self.get_reconstructable_pipeline( external_pipeline.name). subset_for_execution_from_existing_pipeline( external_pipeline.solids_to_execute), run_config=run_config, mode=mode, step_keys_to_execute=step_keys_to_execute, ), external_pipeline.identifying_pipeline_snapshot_id, ), represented_pipeline=external_pipeline, )
def get_external_execution_plan(self, external_pipeline, environment_dict, mode, step_keys_to_execute): from dagster.api.snapshot_execution_plan import sync_get_external_execution_plan check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) check.dict_param(environment_dict, 'environment_dict') check.str_param(mode, 'mode') check.opt_list_param(step_keys_to_execute, 'step_keys_to_execute', of_type=str) execution_plan_snapshot = sync_get_external_execution_plan( pipeline_origin=external_pipeline.get_origin(), solid_selection=external_pipeline.solid_selection, environment_dict=environment_dict, mode=mode, step_keys_to_execute=step_keys_to_execute, snapshot_id=external_pipeline.identifying_pipeline_snapshot_id, ) return ExternalExecutionPlan( execution_plan_snapshot=execution_plan_snapshot, represented_pipeline=external_pipeline, )
def test_run_created_in_0_7_9_snapshot_id_change(): src_dir = file_relative_path( __file__, "snapshot_0_7_9_shapshot_id_creation_change/sqlite") with copy_directory(src_dir) as test_dir: instance = DagsterInstance.from_ref(InstanceRef.from_dir(test_dir)) # run_id = 'e297fa70-49e8-43f8-abfe-1634f02644f6' old_pipeline_snapshot_id = "88528edde2ed64da3c39cca0da8ba2f7586c1a5d" old_execution_plan_snapshot_id = "2246f8e5a10d21e15fbfa3773d7b2d0bc1fa9d3d" historical_pipeline = instance.get_historical_pipeline( old_pipeline_snapshot_id) pipeline_snapshot = historical_pipeline.pipeline_snapshot ep_snapshot = instance.get_execution_plan_snapshot( old_execution_plan_snapshot_id) # It is the pipeline snapshot that changed # Verify that snapshot ids are not equal. This changed in 0.7.10 created_snapshot_id = create_pipeline_snapshot_id(pipeline_snapshot) assert created_snapshot_id != old_pipeline_snapshot_id # verify that both are accessible off of the historical pipeline assert historical_pipeline.computed_pipeline_snapshot_id == created_snapshot_id assert historical_pipeline.identifying_pipeline_snapshot_id == old_pipeline_snapshot_id # We also changed execution plan schema in 0.7.11.post1 assert create_execution_plan_snapshot_id( ep_snapshot) != old_execution_plan_snapshot_id # This previously failed with a check error assert ExternalExecutionPlan(ep_snapshot)
def get_pipeline_run_observable(graphene_info, run_id, after=None): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.str_param(run_id, 'run_id') check.opt_int_param(after, 'after') instance = graphene_info.context.instance run = instance.get_run_by_id(run_id) if not run: def _get_error_observable(observer): observer.on_next( graphene_info.schema.type_named('PipelineRunLogsSubscriptionFailure')( missingRunId=run_id, message='Could not load run with id {}'.format(run_id) ) ) return Observable.create(_get_error_observable) # pylint: disable=E1101 external_execution_plan = ( ExternalExecutionPlan( execution_plan_snapshot=instance.get_execution_plan_snapshot( run.execution_plan_snapshot_id ), represented_pipeline=instance.get_historical_pipeline(run.pipeline_snapshot_id), ) if run.pipeline_snapshot_id and run.execution_plan_snapshot_id else None ) # pylint: disable=E1101 return Observable.create( PipelineRunObservableSubscribe(instance, run_id, after_cursor=after) ).map( lambda events: graphene_info.schema.type_named('PipelineRunLogsSubscriptionSuccess')( run=graphene_info.schema.type_named('PipelineRun')(run), messages=[ from_event_record(event, run.pipeline_name, external_execution_plan) for event in events ], ) )
def get_retry_steps_from_parent_run( instance, parent_run_id ) -> Tuple[List[str], Optional[KnownExecutionState]]: check.inst_param(instance, "instance", DagsterInstance) check.str_param(parent_run_id, "parent_run_id") parent_run = instance.get_run_by_id(parent_run_id) parent_run_logs = instance.all_logs(parent_run_id) execution_plan_snapshot = instance.get_execution_plan_snapshot( parent_run.execution_plan_snapshot_id ) if not execution_plan_snapshot: raise DagsterExecutionPlanSnapshotNotFoundError( f"Could not load execution plan snapshot for run {parent_run_id}" ) execution_plan = ExternalExecutionPlan(execution_plan_snapshot=execution_plan_snapshot) # keep track of steps with dicts that point: # * step_key -> set(step_handle) in the normal case # * unresolved_step_key -> set(resolved_step_handle, ...) for dynamic outputs all_steps_in_parent_run_logs: Dict[str, set] = defaultdict(set) failed_steps_in_parent_run_logs: Dict[str, set] = defaultdict(set) successful_steps_in_parent_run_logs: Dict[str, set] = defaultdict(set) interrupted_steps_in_parent_run_logs: Dict[str, set] = defaultdict(set) skipped_steps_in_parent_run_logs: Dict[str, set] = defaultdict(set) for record in parent_run_logs: if record.dagster_event and record.dagster_event.step_handle: step_handle = record.dagster_event.step_handle _update_tracking_dict(all_steps_in_parent_run_logs, step_handle) if record.dagster_event_type == DagsterEventType.STEP_FAILURE: _update_tracking_dict(failed_steps_in_parent_run_logs, step_handle) if record.dagster_event_type == DagsterEventType.STEP_SUCCESS: _update_tracking_dict(successful_steps_in_parent_run_logs, step_handle) if record.dagster_event_type == DagsterEventType.STEP_SKIPPED: _update_tracking_dict(skipped_steps_in_parent_run_logs, step_handle) for step_set in all_steps_in_parent_run_logs.values(): for step_handle in step_set: if ( not _in_tracking_dict(step_handle, failed_steps_in_parent_run_logs) and not _in_tracking_dict(step_handle, successful_steps_in_parent_run_logs) and not _in_tracking_dict(step_handle, skipped_steps_in_parent_run_logs) ): _update_tracking_dict(interrupted_steps_in_parent_run_logs, step_handle) to_retry = defaultdict(set) execution_deps = execution_plan.execution_deps() for step_snap in execution_plan.topological_steps(): step_key = step_snap.key step_handle = StepHandle.parse_from_key(step_snap.key) if parent_run.step_keys_to_execute and step_snap.key not in parent_run.step_keys_to_execute: continue if step_snap.key in failed_steps_in_parent_run_logs: to_retry[step_key].update(failed_steps_in_parent_run_logs[step_key]) # Interrupted steps can occur when graceful cleanup from a step failure fails to run, # and a step failure event is not generated if step_key in interrupted_steps_in_parent_run_logs: to_retry[step_key].update(interrupted_steps_in_parent_run_logs[step_key]) # Missing steps did not execute, e.g. when a run was terminated if step_key not in all_steps_in_parent_run_logs: to_retry[step_key].add(step_handle) step_dep_keys = execution_deps[step_key] retrying_dep_keys = step_dep_keys.intersection(to_retry.keys()) # this step is downstream of a step we are about to retry if retrying_dep_keys: for retrying_key in retrying_dep_keys: # If this step and its ancestor are both downstream of a dynamic output, # add resolved instances of this step for the retrying mapping keys if isinstance(step_handle, UnresolvedStepHandle) and all( map( lambda handle: isinstance(handle, ResolvedFromDynamicStepHandle), to_retry[retrying_key], ) ): for resolved_handle in to_retry[retrying_key]: to_retry[step_key].add(step_handle.resolve(resolved_handle.mapping_key)) else: to_retry[step_key].add(step_handle) steps_to_retry = [ step_handle.to_key() for step_set in to_retry.values() for step_handle in step_set ] return steps_to_retry, KnownExecutionState.for_reexecution(parent_run_logs, steps_to_retry)