def get_dauphin_pipeline_from_pipeline_index(graphene_info, pipeline_index):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(pipeline_index, 'pipeline_index', PipelineIndex)
if isinstance(graphene_info.context, DagsterGraphQLContext):
pipeline = get_pipeline_definition(graphene_info, pipeline_index.pipeline_snapshot.name)
return DauphinPipeline(pipeline_index, presets=pipeline.get_presets())
return DauphinPipeline(pipeline_index)
def get_subset_external_pipeline(self, selector):
check.inst_param(selector, "selector", PipelineSelector)
# We have to grab the pipeline from the location instead of the repository directly
# since we may have to request a subset we don't have in memory yet
repository_location = self._repository_locations[
selector.location_name]
external_repository = repository_location.get_repository(
selector.repository_name)
subset_result = repository_location.get_subset_external_pipeline_result(
selector)
if not subset_result.success:
error_info = subset_result.error
raise UserFacingGraphQLError(
DauphinInvalidSubsetError(
message="{message}{cause_message}".format(
message=error_info.message,
cause_message="\n{}".format(error_info.cause.message)
if error_info.cause else "",
),
pipeline=DauphinPipeline(
self.get_full_external_pipeline(selector)),
))
return ExternalPipeline(
subset_result.external_pipeline_data,
repository_handle=external_repository.handle,
)
def get_dauphin_pipeline_from_selector(graphene_info, selector):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(selector, 'selector', ExecutionSelector)
return DauphinPipeline(
get_external_pipeline_or_raise(graphene_info, selector.name,
selector.solid_subset))
def get_used_solid_map(repo):
check.inst_param(repo, "repo", ExternalRepository)
inv_by_def_name = defaultdict(list)
definitions = []
for external_pipeline in repo.get_all_external_pipelines():
for handle in build_dauphin_solid_handles(
external_pipeline, external_pipeline.dep_structure_index
):
definition = handle.solid.get_dauphin_solid_definition()
if definition.name not in inv_by_def_name:
definitions.append(definition)
inv_by_def_name[definition.name].append(
DauphinSolidInvocationSite(
pipeline=DauphinPipeline(external_pipeline), solidHandle=handle,
)
)
return OrderedDict(
(
definition.name,
DauphinUsedSolid(
definition=definition,
invocations=sorted(
inv_by_def_name[definition.name],
key=lambda i: i.solidHandle.handleID.to_string(),
),
),
)
for definition in sorted(definitions, key=lambda d: d.name)
)
def validate_pipeline_config(graphene_info, selector, environment_dict, mode):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(selector, 'selector', ExecutionSelector)
check.opt_str_param(mode, 'mode')
pipeline_def = get_pipeline_def_from_selector(graphene_info, selector)
get_validated_config(pipeline_def, environment_dict, mode)
return graphene_info.schema.type_named('PipelineConfigValidationValid')(
DauphinPipeline.from_pipeline_def(pipeline_def))
def for_validation_errors(pipeline, errors):
return DauphinPipelineConfigValidationInvalid(
pipeline=DauphinPipeline.from_pipeline_def(pipeline),
errors=[
DauphinPipelineConfigValidationError.from_dagster_error(
pipeline.get_config_schema_snapshot(), err)
for err in errors
],
)
def get_dauphin_pipeline_from_selector(graphene_info, selector):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(selector, 'selector', ExecutionSelector)
if isinstance(graphene_info.context, DagsterGraphQLContext):
pipeline_definition = get_pipeline_def_from_selector(graphene_info, selector)
return DauphinPipeline.from_pipeline_def(pipeline_definition)
# TODO: Support solid sub selection.
check.invariant(
not selector.solid_subset,
desc="DagsterSnapshotGraphQLContext doesn't support pipeline sub-selection.",
)
repository_index = graphene_info.context.get_repository_index()
if not repository_index.has_pipeline_index(selector.name):
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineNotFoundError')(pipeline_name=selector.name)
)
return DauphinPipeline(repository_index.get_pipeline_index(selector.name))
def get_execution_plan(graphene_info, selector, environment_dict, mode):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(selector, 'selector', ExecutionSelector)
check.opt_str_param(mode, 'mode')
pipeline_def = get_pipeline_def_from_selector(graphene_info, selector)
get_validated_config(graphene_info, pipeline_def, environment_dict, mode)
return graphene_info.schema.type_named('ExecutionPlan')(
DauphinPipeline.from_pipeline_def(pipeline_def),
create_execution_plan(pipeline_def, environment_dict,
RunConfig(mode=mode)),
)
def get_dauphin_pipeline_from_selector(graphene_info, selector):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(selector, 'selector', ExecutionSelector)
if not isinstance(graphene_info.context,
DagsterGraphQLInProcessRepositoryContext):
# TODO: Support solid sub selection.
check.invariant(
not selector.solid_subset,
desc=
"DagsterGraphQLOutOfProcessRepositoryContext doesn't support pipeline sub-selection.",
)
return DauphinPipeline(
get_external_pipeline_subset_or_raise(graphene_info, selector.name,
selector.solid_subset))
def get_validated_config(graphene_info, pipeline_def, environment_dict, mode):
check.str_param(mode, 'mode')
check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition)
environment_schema = create_environment_schema(pipeline_def, mode)
validated_config = validate_config(environment_schema.environment_type,
environment_dict)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline=DauphinPipeline.from_pipeline_def(pipeline_def),
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
pipeline_def.get_config_schema_snapshot(), err)
for err in validated_config.errors
],
))
return validated_config
def get_used_solid_map(graphene_info):
inv_by_def_name = defaultdict(list)
definitions = []
for external_pipeline in graphene_info.context.get_all_external_pipelines(
):
for handle in build_dauphin_solid_handles(
external_pipeline, external_pipeline.dep_structure_index):
definition = handle.solid.get_dauphin_solid_definition()
if definition.name not in inv_by_def_name:
definitions.append(definition)
inv_by_def_name[definition.name].append(
DauphinSolidInvocationSite(
pipeline=DauphinPipeline(external_pipeline),
solidHandle=handle,
))
return OrderedDict((
definition.name,
DauphinUsedSolid(
definition=definition,
invocations=sorted(inv_by_def_name[definition.name],
key=lambda i: i.solidHandle.handleID),
),
) for definition in sorted(definitions, key=lambda d: d.name))
def get_dauphin_pipeline_from_selector(graphene_info, selector):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(selector, 'selector', PipelineSelector)
return DauphinPipeline(
get_external_pipeline_or_raise(graphene_info, selector))
def _do_execute_plan(graphene_info, execution_params, external_pipeline,
retries):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(execution_params, 'execution_params', ExecutionParams)
check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline)
check.opt_inst_param(retries, 'retries', Retries)
run_id = execution_params.execution_metadata.run_id
mode = execution_params.mode or external_pipeline.get_default_mode_name()
pipeline_run = graphene_info.context.instance.get_run_by_id(run_id)
external_execution_plan = graphene_info.context.get_external_execution_plan(
external_pipeline=external_pipeline,
run_config=execution_params.run_config,
mode=mode,
step_keys_to_execute=None,
)
if not pipeline_run:
# TODO switch to raising a UserFacingError if the run_id cannot be found
# https://github.com/dagster-io/dagster/issues/1876
pipeline_run = graphene_info.context.instance.create_run(
pipeline_name=external_pipeline.name,
run_id=run_id,
run_config=execution_params.run_config,
mode=mode,
solids_to_execute=None,
step_keys_to_execute=None,
status=None,
tags=execution_params.execution_metadata.tags or {},
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=external_execution_plan.
execution_plan_snapshot,
parent_pipeline_snapshot=external_pipeline.
parent_pipeline_snapshot,
)
ensure_valid_step_keys(external_execution_plan, execution_params.step_keys)
if execution_params.step_keys:
external_execution_plan = graphene_info.context.get_external_execution_plan(
external_pipeline=external_pipeline,
run_config=execution_params.run_config,
mode=mode,
step_keys_to_execute=execution_params.step_keys,
)
event_logs = []
def _on_event_record(record):
if record.is_dagster_event:
event_logs.append(record)
graphene_info.context.instance.add_event_listener(run_id, _on_event_record)
graphene_info.context.execute_plan(
external_pipeline=external_pipeline,
run_config=execution_params.run_config,
pipeline_run=pipeline_run,
step_keys_to_execute=execution_params.step_keys,
retries=retries,
)
def to_graphql_event(event_record):
return from_dagster_event_record(event_record, external_pipeline.name)
for event in event_logs:
if event.dagster_event.is_pipeline_init_failure:
return graphene_info.schema.type_named('PythonError')(
event.dagster_event.pipeline_init_failure_data.error)
return graphene_info.schema.type_named('ExecutePlanSuccess')(
pipeline=DauphinPipeline(external_pipeline),
has_failures=any(
er for er in event_logs if er.is_dagster_event
and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE),
step_events=list(map(to_graphql_event, event_logs)),
raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)),
)
def _do_execute_plan(graphene_info, execution_params, pipeline_def):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(execution_params, 'execution_params', ExecutionParams)
run_id = execution_params.execution_metadata.run_id
pipeline_run = graphene_info.context.instance.get_run_by_id(run_id)
if not pipeline_run:
# TODO switch to raising a UserFacingError if the run_id cannot be found
# https://github.com/dagster-io/dagster/issues/1876
pipeline_run = PipelineRun(
pipeline_name=pipeline_def.name,
run_id=run_id,
environment_dict=execution_params.environment_dict,
mode=execution_params.mode or pipeline_def.get_default_mode_name(),
tags=execution_params.execution_metadata.tags or {},
)
execution_plan = create_execution_plan(
pipeline=pipeline_def,
environment_dict=execution_params.environment_dict,
run_config=pipeline_run,
)
if execution_params.step_keys:
for step_key in execution_params.step_keys:
if not execution_plan.has_step(step_key):
raise UserFacingGraphQLError(
graphene_info.schema.type_named('InvalidStepError')(invalid_step_key=step_key)
)
execution_plan = execution_plan.build_subset_plan(execution_params.step_keys)
event_logs = []
def _on_event_record(record):
if record.is_dagster_event:
event_logs.append(record)
graphene_info.context.instance.add_event_listener(run_id, _on_event_record)
execute_plan(
execution_plan=execution_plan,
environment_dict=execution_params.environment_dict,
pipeline_run=pipeline_run,
instance=graphene_info.context.instance,
)
dauphin_pipeline = DauphinPipeline.from_pipeline_def(pipeline_def)
def to_graphql_event(event_record):
return from_dagster_event_record(
graphene_info, event_record, dauphin_pipeline, execution_plan
)
return graphene_info.schema.type_named('ExecutePlanSuccess')(
pipeline=dauphin_pipeline,
has_failures=any(
er
for er in event_logs
if er.is_dagster_event and er.dagster_event.event_type == DagsterEventType.STEP_FAILURE
),
step_events=list(map(to_graphql_event, event_logs)),
raw_event_records=list(map(serialize_dagster_namedtuple, event_logs)),
)
def get_pipeline_or_raise(graphene_info, selector):
'''Returns a DauphinPipeline or raises a UserFacingGraphQLError if one cannot be retrieved
from the selector, e.g., the pipeline is not present in the loaded repository.'''
return DauphinPipeline(
get_pipeline_def_from_selector(graphene_info, selector))
def get_pipeline_or_error(graphene_info, selector):
'''Returns a DauphinPipelineOrError.'''
return DauphinPipeline(
get_pipeline_def_from_selector(graphene_info, selector))