def get_dauphin_pipeline_from_pipeline_index(graphene_info, pipeline_index): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(pipeline_index, 'pipeline_index', PipelineIndex) if isinstance(graphene_info.context, DagsterGraphQLContext): pipeline = get_pipeline_definition(graphene_info, pipeline_index.pipeline_snapshot.name) return DauphinPipeline(pipeline_index, presets=pipeline.get_presets()) return DauphinPipeline(pipeline_index)
def get_subset_external_pipeline(self, selector): check.inst_param(selector, "selector", PipelineSelector) # We have to grab the pipeline from the location instead of the repository directly # since we may have to request a subset we don't have in memory yet repository_location = self._repository_locations[ selector.location_name] external_repository = repository_location.get_repository( selector.repository_name) subset_result = repository_location.get_subset_external_pipeline_result( selector) if not subset_result.success: error_info = subset_result.error raise UserFacingGraphQLError( DauphinInvalidSubsetError( message="{message}{cause_message}".format( message=error_info.message, cause_message="\n{}".format(error_info.cause.message) if error_info.cause else "", ), pipeline=DauphinPipeline( self.get_full_external_pipeline(selector)), )) return ExternalPipeline( subset_result.external_pipeline_data, repository_handle=external_repository.handle, )
def get_dauphin_pipeline_from_selector(graphene_info, selector): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(selector, 'selector', ExecutionSelector) return DauphinPipeline( get_external_pipeline_or_raise(graphene_info, selector.name, selector.solid_subset))
def get_used_solid_map(repo): check.inst_param(repo, "repo", ExternalRepository) inv_by_def_name = defaultdict(list) definitions = [] for external_pipeline in repo.get_all_external_pipelines(): for handle in build_dauphin_solid_handles( external_pipeline, external_pipeline.dep_structure_index ): definition = handle.solid.get_dauphin_solid_definition() if definition.name not in inv_by_def_name: definitions.append(definition) inv_by_def_name[definition.name].append( DauphinSolidInvocationSite( pipeline=DauphinPipeline(external_pipeline), solidHandle=handle, ) ) return OrderedDict( ( definition.name, DauphinUsedSolid( definition=definition, invocations=sorted( inv_by_def_name[definition.name], key=lambda i: i.solidHandle.handleID.to_string(), ), ), ) for definition in sorted(definitions, key=lambda d: d.name) )
def validate_pipeline_config(graphene_info, selector, environment_dict, mode): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(selector, 'selector', ExecutionSelector) check.opt_str_param(mode, 'mode') pipeline_def = get_pipeline_def_from_selector(graphene_info, selector) get_validated_config(graphene_info, pipeline_def, environment_dict, mode) return graphene_info.schema.type_named('PipelineConfigValidationValid')( DauphinPipeline(pipeline_def))
def get_execution_plan(graphene_info, selector, environment_dict, mode): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(selector, 'selector', ExecutionSelector) check.opt_str_param(mode, 'mode') pipeline_def = get_pipeline_def_from_selector(graphene_info, selector) get_validated_config(graphene_info, pipeline_def, environment_dict, mode) return graphene_info.schema.type_named('ExecutionPlan')( DauphinPipeline(pipeline_def), create_execution_plan(pipeline_def, environment_dict, RunConfig(mode=mode)), )
def get_dauphin_pipeline_from_selector(graphene_info, selector): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(selector, 'selector', ExecutionSelector) if not isinstance(graphene_info.context, DagsterGraphQLInProcessRepositoryContext): # TODO: Support solid sub selection. check.invariant( not selector.solid_subset, desc= "DagsterGraphQLOutOfProcessRepositoryContext doesn't support pipeline sub-selection.", ) return DauphinPipeline( get_external_pipeline_subset_or_raise(graphene_info, selector.name, selector.solid_subset))
def get_dauphin_pipeline_from_selector(graphene_info, selector): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(selector, 'selector', ExecutionSelector) if isinstance(graphene_info.context, DagsterGraphQLContext): pipeline_definition = get_pipeline_def_from_selector(graphene_info, selector) return DauphinPipeline.from_pipeline_def(pipeline_definition) # TODO: Support solid sub selection. check.invariant( not selector.solid_subset, desc="DagsterSnapshotGraphQLContext doesn't support pipeline sub-selection.", ) repository_index = graphene_info.context.get_repository_index() if not repository_index.has_pipeline_index(selector.name): raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineNotFoundError')(pipeline_name=selector.name) ) return DauphinPipeline(repository_index.get_pipeline_index(selector.name))
def get_validated_config(graphene_info, pipeline_def, environment_dict, mode): check.str_param(mode, 'mode') check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) environment_schema = create_environment_schema(pipeline_def, mode) validated_config = validate_config(environment_schema.environment_type, environment_dict) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=DauphinPipeline(pipeline_def), errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( pipeline_def.get_config_schema_snapshot(), err) for err in validated_config.errors ], )) return validated_config
def get_used_solid_map(graphene_info): inv_by_def_name = defaultdict(list) definitions = [] for external_pipeline in graphene_info.context.get_all_external_pipelines( ): for handle in build_dauphin_solid_handles( external_pipeline, external_pipeline.dep_structure_index): definition = handle.solid.get_dauphin_solid_definition() if definition.name not in inv_by_def_name: definitions.append(definition) inv_by_def_name[definition.name].append( DauphinSolidInvocationSite( pipeline=DauphinPipeline(external_pipeline), solidHandle=handle, )) return OrderedDict(( definition.name, DauphinUsedSolid( definition=definition, invocations=sorted(inv_by_def_name[definition.name], key=lambda i: i.solidHandle.handleID), ), ) for definition in sorted(definitions, key=lambda d: d.name))
def get_dauphin_pipeline_from_selector(graphene_info, selector): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(selector, 'selector', PipelineSelector) return DauphinPipeline( get_external_pipeline_or_raise(graphene_info, selector))
def _do_execute_plan(graphene_info, execution_params, external_pipeline, retries): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(execution_params, 'execution_params', ExecutionParams) check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline) check.opt_inst_param(retries, 'retries', Retries) run_id = execution_params.execution_metadata.run_id mode = execution_params.mode or external_pipeline.get_default_mode_name() pipeline_run = graphene_info.context.instance.get_run_by_id(run_id) external_execution_plan = graphene_info.context.get_external_execution_plan( external_pipeline=external_pipeline, run_config=execution_params.run_config, mode=mode, step_keys_to_execute=None, ) if not pipeline_run: # TODO switch to raising a UserFacingError if the run_id cannot be found # https://github.com/dagster-io/dagster/issues/1876 pipeline_run = graphene_info.context.instance.create_run( pipeline_name=external_pipeline.name, run_id=run_id, run_config=execution_params.run_config, mode=mode, solids_to_execute=None, step_keys_to_execute=None, status=None, tags=execution_params.execution_metadata.tags or {}, root_run_id=None, parent_run_id=None, pipeline_snapshot=external_pipeline.pipeline_snapshot, execution_plan_snapshot=external_execution_plan. execution_plan_snapshot, parent_pipeline_snapshot=external_pipeline. parent_pipeline_snapshot, ) ensure_valid_step_keys(external_execution_plan, execution_params.step_keys) if execution_params.step_keys: external_execution_plan = graphene_info.context.get_external_execution_plan( external_pipeline=external_pipeline, run_config=execution_params.run_config, mode=mode, step_keys_to_execute=execution_params.step_keys, ) event_logs = [] def _on_event_record(record): if record.is_dagster_event: event_logs.append(record) graphene_info.context.instance.add_event_listener(run_id, _on_event_record) graphene_info.context.execute_plan( external_pipeline=external_pipeline, run_config=execution_params.run_config, pipeline_run=pipeline_run, step_keys_to_execute=execution_params.step_keys, retries=retries, ) def to_graphql_event(event_record): return from_dagster_event_record(event_record, external_pipeline.name) for event in event_logs: if event.dagster_event.is_pipeline_init_failure: return graphene_info.schema.type_named('PythonError')( event.dagster_event.pipeline_init_failure_data.error) return graphene_info.schema.type_named('ExecutePlanSuccess')( pipeline=DauphinPipeline(external_pipeline), has_failures=any( er for er in event_logs if er.is_dagster_event and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE), step_events=list(map(to_graphql_event, event_logs)), raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)), )
def _do_execute_plan(graphene_info, execution_params, pipeline_def): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(execution_params, 'execution_params', ExecutionParams) run_id = execution_params.execution_metadata.run_id pipeline_run = graphene_info.context.instance.get_run_by_id(run_id) if not pipeline_run: # TODO switch to raising a UserFacingError if the run_id cannot be found # https://github.com/dagster-io/dagster/issues/1876 pipeline_run = PipelineRun( pipeline_name=pipeline_def.name, run_id=run_id, environment_dict=execution_params.environment_dict, mode=execution_params.mode or pipeline_def.get_default_mode_name(), tags=execution_params.execution_metadata.tags or {}, ) execution_plan = create_execution_plan( pipeline=pipeline_def, environment_dict=execution_params.environment_dict, run_config=pipeline_run, ) if execution_params.step_keys: for step_key in execution_params.step_keys: if not execution_plan.has_step(step_key): raise UserFacingGraphQLError( graphene_info.schema.type_named('InvalidStepError')( invalid_step_key=step_key)) execution_plan = execution_plan.build_subset_plan( execution_params.step_keys) event_logs = [] def _on_event_record(record): if record.is_dagster_event: event_logs.append(record) graphene_info.context.instance.add_event_listener(run_id, _on_event_record) execute_plan( execution_plan=execution_plan, environment_dict=execution_params.environment_dict, pipeline_run=pipeline_run, instance=graphene_info.context.instance, ) dauphin_pipeline = DauphinPipeline(pipeline_def) def to_graphql_event(event_record): return from_dagster_event_record(graphene_info, event_record, dauphin_pipeline, execution_plan) return graphene_info.schema.type_named('ExecutePlanSuccess')( pipeline=dauphin_pipeline, has_failures=any( er for er in event_logs if er.is_dagster_event and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE), step_events=list(map(to_graphql_event, event_logs)), raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)), )
def get_pipeline_or_raise(graphene_info, selector): '''Returns a DauphinPipeline or raises a UserFacingGraphQLError if one cannot be retrieved from the selector, e.g., the pipeline is not present in the loaded repository.''' return DauphinPipeline( get_pipeline_def_from_selector(graphene_info, selector))
def get_pipeline_or_error(graphene_info, selector): '''Returns a DauphinPipelineOrError.''' return DauphinPipeline( get_pipeline_def_from_selector(graphene_info, selector))