def resolve_is_run_config_valid(graphene_info, represented_pipeline, mode,
run_config):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(represented_pipeline, 'represented_pipeline',
RepresentedPipeline)
check.str_param(mode, 'mode')
check.dict_param(run_config, 'run_config', key_type=str)
mode_def_snap = represented_pipeline.get_mode_def_snap(mode)
if not mode_def_snap.root_config_key:
# historical pipeline with unknown environment type. blindly pass validation
return graphene_info.schema.type_named(
'PipelineConfigValidationValid')(represented_pipeline.name)
validated_config = validate_config_from_snap(
represented_pipeline.config_schema_snapshot,
mode_def_snap.root_config_key, run_config)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline_name=represented_pipeline.name,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError').from_dagster_error(
represented_pipeline.config_schema_snapshot,
err,
) for err in validated_config.errors
],
))
return graphene_info.schema.type_named('PipelineConfigValidationValid')(
represented_pipeline.name)
def resolve_is_environment_config_valid(graphene_info, pipeline_index, mode, environment_dict):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.inst_param(pipeline_index, 'pipeline_index', PipelineIndex)
check.str_param(mode, 'mode')
check.dict_param(environment_dict, 'environment_dict', key_type=str)
mode_def_snap = pipeline_index.get_mode_def_snap(mode)
if not mode_def_snap.root_config_key:
# historical pipeline with unknown environment type. blindly pass validation
return graphene_info.schema.type_named('PipelineConfigValidationValid')(pipeline_index.name)
validated_config = validate_config_from_snap(
pipeline_index.config_schema_snapshot, mode_def_snap.root_config_key, environment_dict
)
if not validated_config.success:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineConfigValidationInvalid')(
pipeline_name=pipeline_index.name,
errors=[
graphene_info.schema.type_named(
'PipelineConfigValidationError'
).from_dagster_error(
pipeline_index.config_schema_snapshot, err,
)
for err in validated_config.errors
],
)
)
return graphene_info.schema.type_named('PipelineConfigValidationValid')(pipeline_index.name)
def _start_pipeline_execution_for_created_run(graphene_info, run_id):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
instance = graphene_info.context.instance
execution_manager_settings = instance.dagit_settings.get(
'execution_manager')
if execution_manager_settings and execution_manager_settings.get(
'disabled'):
return graphene_info.schema.type_named(
'StartPipelineRunDisabledError')()
pipeline_run = instance.get_run_by_id(run_id)
if not pipeline_run:
return graphene_info.schema.type_named('PipelineRunNotFoundError')(
run_id)
external_pipeline = get_external_pipeline_subset_or_raise(
graphene_info, pipeline_run.selector.name,
pipeline_run.selector.solid_subset)
validated_config = validate_config_from_snap(
external_pipeline.config_schema_snapshot,
external_pipeline.root_config_key_for_mode(pipeline_run.mode),
pipeline_run.environment_dict,
)
if not validated_config.success:
# If the config is invalid, we construct a DagsterInvalidConfigError exception and
# insert it into the event log. We also return a PipelineConfigValidationInvalid user facing
# graphql error.
# We currently re-use the engine events machinery to add the error to the event log, but
# may need to create a new event type and instance method to handle these erros.
invalid_config_exception = DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(external_pipeline.name),
validated_config.errors,
pipeline_run.environment_dict,
)
instance.report_engine_event(
str(invalid_config_exception.message),
pipeline_run,
EngineEventData.engine_error(
SerializableErrorInfo(
invalid_config_exception.message,
[],
DagsterInvalidConfigError.__class__.__name__,
None,
)),
)
instance.report_run_failed(pipeline_run)
return DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline.pipeline_index, validated_config.errors)
graphene_info.context.execute_pipeline(external_pipeline, pipeline_run)
return graphene_info.schema.type_named('StartPipelineRunSuccess')(
run=graphene_info.schema.type_named('PipelineRun')(pipeline_run))
def _execute_schedule(graphene_info, external_pipeline, execution_params,
errors):
check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline)
instance = graphene_info.context.instance
mode, environment_dict = execution_params.mode, execution_params.environment_dict
validation_result = validate_config_from_snap(
external_pipeline.config_schema_snapshot,
external_pipeline.root_config_key_for_mode(mode),
environment_dict,
)
external_execution_plan = (
graphene_info.context.get_external_execution_plan(
external_pipeline, environment_dict, mode,
execution_params.step_keys) if validation_result.success else None)
pipeline_run = instance.create_run(
pipeline_name=external_pipeline.name,
run_id=None,
environment_dict=environment_dict,
mode=mode,
solid_subset=(execution_params.selector.solid_subset
if execution_params.selector is not None else None),
step_keys_to_execute=None,
status=None,
tags=execution_params.execution_metadata.tags,
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=external_execution_plan.execution_plan_snapshot
if external_execution_plan else None,
parent_pipeline_snapshot=external_pipeline.parent_pipeline_snapshot,
)
# Inject errors into event log at this point
if len(errors) > 0:
for error in errors:
instance.report_engine_event(error.message, pipeline_run,
EngineEventData.engine_error(error))
# Launch run if run launcher is defined
run_launcher = graphene_info.context.instance.run_launcher
if run_launcher:
result = do_launch_for_created_run(graphene_info, pipeline_run.run_id)
else:
result = _synchronously_execute_run_within_hosted_user_process(
graphene_info, pipeline_run.run_id)
return pipeline_run, result
def ensure_valid_config(external_pipeline, mode, run_config):
check.inst_param(external_pipeline, "external_pipeline", ExternalPipeline)
check.str_param(mode, "mode")
# do not type check run_config so that validate_config_from_snap throws
validated_config = validate_config_from_snap(
config_schema_snapshot=external_pipeline.config_schema_snapshot,
config_type_key=external_pipeline.root_config_key_for_mode(mode),
config_value=run_config,
)
if not validated_config.success:
from dagster_graphql.schema.errors import DauphinPipelineConfigValidationInvalid
raise UserFacingGraphQLError(
DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline, validated_config.errors))
return validated_config
def ensure_valid_config(external_pipeline, mode, environment_dict):
check.inst_param(external_pipeline, 'external_pipeline', ExternalPipeline)
check.str_param(mode, 'mode')
# do not type check environment_dict so that validate_config_from_snap throws
validated_config = validate_config_from_snap(
config_schema_snapshot=external_pipeline.config_schema_snapshot,
config_type_key=external_pipeline.root_config_key_for_mode(mode),
config_value=environment_dict,
)
if not validated_config.success:
from dagster_graphql.schema.errors import DauphinPipelineConfigValidationInvalid
raise UserFacingGraphQLError(
DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline.pipeline_index, validated_config.errors))
return validated_config
def create_possibly_invalid_run(
graphene_info,
external_pipeline,
execution_params,
):
mode, environment_dict, step_keys = (
execution_params.mode,
execution_params.environment_dict,
execution_params.step_keys,
)
validation_result = validate_config_from_snap(
external_pipeline.config_schema_snapshot,
external_pipeline.root_config_key_for_mode(mode),
environment_dict,
)
external_execution_plan = (
graphene_info.context.get_external_execution_plan(
external_pipeline, environment_dict, mode, step_keys)
if validation_result.success else None)
return graphene_info.context.instance.create_run(
pipeline_name=external_pipeline.name,
run_id=None,
environment_dict=environment_dict,
mode=mode,
solids_to_execute=frozenset(execution_params.selector.solid_selection)
if execution_params.selector.solid_selection else None,
step_keys_to_execute=step_keys,
status=None,
tags=merge_dicts(external_pipeline.tags,
execution_params.execution_metadata.tags),
root_run_id=None,
parent_run_id=None,
pipeline_snapshot=external_pipeline.pipeline_snapshot,
execution_plan_snapshot=external_execution_plan.execution_plan_snapshot
if external_execution_plan else None,
parent_pipeline_snapshot=external_pipeline.parent_pipeline_snapshot,
)
def resolve_is_run_config_valid(graphene_info, represented_pipeline, mode, run_config):
from ..schema.pipelines.config import (
GraphenePipelineConfigValidationError,
GrapheneRunConfigValidationInvalid,
GraphenePipelineConfigValidationValid,
)
check.inst_param(graphene_info, "graphene_info", ResolveInfo)
check.inst_param(represented_pipeline, "represented_pipeline", RepresentedPipeline)
check.str_param(mode, "mode")
check.dict_param(run_config, "run_config", key_type=str)
mode_def_snap = represented_pipeline.get_mode_def_snap(mode)
if not mode_def_snap.root_config_key:
# historical pipeline with unknown environment type. blindly pass validation
return GraphenePipelineConfigValidationValid(represented_pipeline.name)
validated_config = validate_config_from_snap(
represented_pipeline.config_schema_snapshot, mode_def_snap.root_config_key, run_config
)
if not validated_config.success:
raise UserFacingGraphQLError(
GrapheneRunConfigValidationInvalid(
pipeline_name=represented_pipeline.name,
errors=[
GraphenePipelineConfigValidationError.from_dagster_error(
represented_pipeline.config_schema_snapshot,
err,
)
for err in validated_config.errors
],
)
)
return GraphenePipelineConfigValidationValid(represented_pipeline.name)
def get_run_execution_info_for_created_run_or_error(
graphene_info, repository_location_name, repository_name, run_id
):
"""
Previously created run could either be created in a different process *or*
during the launchScheduledRun call where we want to have a record of
a run the was created but have invalid configuration
"""
check.inst_param(graphene_info, "graphene_info", ResolveInfo)
check.str_param(repository_location_name, "repository_location_name")
check.str_param(repository_name, "repository_name")
check.str_param(run_id, "run_id")
instance = graphene_info.context.instance
pipeline_run = instance.get_run_by_id(run_id)
if not pipeline_run:
return graphene_info.schema.type_named("PipelineRunNotFoundError")(run_id)
external_pipeline = get_external_pipeline_or_raise(
graphene_info,
_get_selector_with_workaround(
graphene_info.context, repository_location_name, repository_name, pipeline_run
),
)
validated_config = validate_config_from_snap(
external_pipeline.config_schema_snapshot,
external_pipeline.root_config_key_for_mode(pipeline_run.mode),
pipeline_run.run_config,
)
if not validated_config.success:
# If the config is invalid, we construct a DagsterInvalidConfigError exception and
# insert it into the event log. We also return a PipelineConfigValidationInvalid user facing
# graphql error.
# We currently re-use the engine events machinery to add the error to the event log, but
# may need to create a new event type and instance method to handle these errors.
invalid_config_exception = DagsterInvalidConfigError(
"Error in config for pipeline {}".format(external_pipeline.name),
validated_config.errors,
pipeline_run.run_config,
)
instance.report_engine_event(
str(invalid_config_exception.message),
pipeline_run,
EngineEventData.engine_error(
SerializableErrorInfo(
invalid_config_exception.message,
[],
DagsterInvalidConfigError.__class__.__name__,
None,
)
),
)
instance.report_run_failed(pipeline_run)
return DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline, validated_config.errors
)
return RunExecutionInfo(external_pipeline, pipeline_run)
def do_launch_for_created_run(graphene_info, run_id):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.str_param(run_id, 'run_id')
# First retrieve the pipeline run
instance = graphene_info.context.instance
pipeline_run = instance.get_run_by_id(run_id)
if not pipeline_run:
raise UserFacingGraphQLError(
graphene_info.schema.type_named('PipelineRunNotFoundError')(run_id)
)
external_pipeline = legacy_get_external_pipeline_or_raise(
graphene_info, pipeline_run.pipeline_name, pipeline_run.solid_subset
)
# Run config validation
# If there are any config errors, then inject them into the event log
validated_config = validate_config_from_snap(
external_pipeline.config_schema_snapshot,
external_pipeline.root_config_key_for_mode(pipeline_run.mode),
pipeline_run.environment_dict,
)
if not validated_config.success:
# If the config is invalid, we construct a DagsterInvalidConfigError exception and
# insert it into the event log. We also return a PipelineConfigValidationInvalid user facing
# graphql error.
# We currently re-use the engine events machinery to add the error to the event log, but
# may need to create a new event type and instance method to handle these errors.
invalid_config_exception = DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(external_pipeline.name),
validated_config.errors,
pipeline_run.environment_dict,
)
instance.report_engine_event(
str(invalid_config_exception.message),
pipeline_run,
EngineEventData.engine_error(
SerializableErrorInfo(
invalid_config_exception.message,
[],
DagsterInvalidConfigError.__class__.__name__,
None,
)
),
)
instance.report_run_failed(pipeline_run)
return DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline, validated_config.errors
)
try:
launched_run = instance.launch_run(pipeline_run.run_id, external_pipeline)
return graphene_info.schema.type_named('LaunchPipelineRunSuccess')(
run=graphene_info.schema.type_named('PipelineRun')(launched_run)
)
except DagsterLaunchFailedError:
error = serializable_error_info_from_exc_info(sys.exc_info())
instance.report_engine_event(
error.message, pipeline_run, EngineEventData.engine_error(error),
)
instance.report_run_failed(pipeline_run)
# https://github.com/dagster-io/dagster/issues/2508
# We should return a proper GraphQL error here
raise
