def __init__(
self,
config,
default_value=FIELD_NO_DEFAULT_PROVIDED,
is_required=None,
description=None,
):
from .validate import validate_config
from .post_process import resolve_defaults
self.config_type = check.inst(self._resolve_config_arg(config),
ConfigType)
self.description = check.opt_str_param(description, 'description')
check.opt_bool_param(is_required, 'is_required')
if default_value != FIELD_NO_DEFAULT_PROVIDED:
check.param_invariant(not (callable(default_value)),
'default_value',
'default_value cannot be a callable')
if is_required is True:
check.param_invariant(
default_value == FIELD_NO_DEFAULT_PROVIDED,
'default_value',
'required arguments should not specify default values',
)
self._default_value = default_value
# check explicit default value
if self.default_provided:
if self.config_type.kind == ConfigTypeKind.ENUM and is_enum_value(
default_value):
raise DagsterInvalidDefinitionError((
'You have passed into a python enum value as the default value '
'into of a config enum type {name}. You must pass in the underlying '
'string represention as the default value. One of {value_set}.'
).format(
value_set=[
ev.config_value for ev in self.config_type.enum_values
],
name=self.config_type.given_name,
))
evr = validate_config(self.config_type, default_value)
if not evr.success:
raise DagsterInvalidConfigError(
'Invalid default_value for Field.',
evr.errors,
default_value,
)
if is_required is None:
is_required = not all_optional_type(self.config_type)
# on implicitly optional - set the default value
# by resolving the defaults of the type
if not is_required and not self.default_provided:
evr = resolve_defaults(self.config_type, None)
if not evr.success:
raise DagsterInvalidConfigError(
'Unable to resolve implicit default_value for Field.',
evr.errors,
None,
)
self._default_value = evr.value
self._is_required = is_required
def _launch_pipeline_execution_for_created_run(graphene_info, run_id):
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.str_param(run_id, 'run_id')
# First retrieve the pipeline run
instance = graphene_info.context.instance
pipeline_run = instance.get_run_by_id(run_id)
if not pipeline_run:
return graphene_info.schema.type_named('PipelineRunNotFoundError')(
run_id)
external_pipeline = get_external_pipeline_or_raise(
graphene_info, pipeline_run.selector.name,
pipeline_run.selector.solid_subset)
# Run config validation
# If there are any config errors, then inject them into the event log
validated_config = ensure_valid_config(external_pipeline,
pipeline_run.mode,
pipeline_run.environment_dict)
if not validated_config.success:
# If the config is invalid, we construct a DagsterInvalidConfigError exception and
# insert it into the event log. We also return a PipelineConfigValidationInvalid user facing
# graphql error.
# We currently re-use the engine events machinery to add the error to the event log, but
# may need to create a new event type and instance method to handle these errors.
invalid_config_exception = DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(external_pipeline.name),
validated_config.errors,
pipeline_run.environment_dict,
)
instance.report_engine_event(
str(invalid_config_exception.message),
pipeline_run,
EngineEventData.engine_error(
SerializableErrorInfo(
invalid_config_exception.message,
[],
DagsterInvalidConfigError.__class__.__name__,
None,
)),
)
instance.report_run_failed(pipeline_run)
return DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline, validated_config.errors)
try:
pipeline_run = instance.launch_run(pipeline_run.run_id)
except DagsterLaunchFailedError:
error = serializable_error_info_from_exc_info(sys.exc_info())
instance.report_engine_event(
error.message,
pipeline_run,
EngineEventData.engine_error(error),
)
instance.report_run_failed(pipeline_run)
return graphene_info.schema.type_named('LaunchPipelineRunSuccess')(
run=graphene_info.schema.type_named('PipelineRun')(pipeline_run))
def get_run_execution_info_for_created_run_or_error(
graphene_info, repository_location_name, repository_name, run_id
):
'''
Previously created run could either be created in a different process *or*
during the launchScheduledRun call where we want to have a record of
a run the was created but have invalid configuration
'''
check.inst_param(graphene_info, 'graphene_info', ResolveInfo)
check.str_param(repository_location_name, 'repository_location_name')
check.str_param(repository_name, 'repository_name')
check.str_param(run_id, 'run_id')
instance = graphene_info.context.instance
pipeline_run = instance.get_run_by_id(run_id)
if not pipeline_run:
return graphene_info.schema.type_named('PipelineRunNotFoundError')(run_id)
external_pipeline = get_external_pipeline_or_raise(
graphene_info,
PipelineSelector(
location_name=repository_location_name,
repository_name=repository_name,
pipeline_name=pipeline_run.pipeline_name,
solid_selection=list(pipeline_run.solids_to_execute)
if pipeline_run.solids_to_execute
else None,
),
)
validated_config = validate_config_from_snap(
external_pipeline.config_schema_snapshot,
external_pipeline.root_config_key_for_mode(pipeline_run.mode),
pipeline_run.run_config,
)
if not validated_config.success:
# If the config is invalid, we construct a DagsterInvalidConfigError exception and
# insert it into the event log. We also return a PipelineConfigValidationInvalid user facing
# graphql error.
# We currently re-use the engine events machinery to add the error to the event log, but
# may need to create a new event type and instance method to handle these errors.
invalid_config_exception = DagsterInvalidConfigError(
'Error in config for pipeline {}'.format(external_pipeline.name),
validated_config.errors,
pipeline_run.run_config,
)
instance.report_engine_event(
str(invalid_config_exception.message),
pipeline_run,
EngineEventData.engine_error(
SerializableErrorInfo(
invalid_config_exception.message,
[],
DagsterInvalidConfigError.__class__.__name__,
None,
)
),
)
instance.report_run_failed(pipeline_run)
return DauphinPipelineConfigValidationInvalid.for_validation_errors(
external_pipeline, validated_config.errors
)
return RunExecutionInfo(external_pipeline, pipeline_run)
def build(pipeline_def, run_config=None, mode=None):
"""This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
"""
from dagster.config.validate import process_config
from dagster.core.definitions.executor import ExecutorDefinition
from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition
from .composite_descent import composite_descent
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config = check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(
environment_type, run_config_storage_field_backcompat(run_config)
)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for pipeline {}".format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
config_mapped_intermediate_storage_configs = config_map_objects(
config_value,
mode_def.intermediate_storage_defs,
"intermediate_storage",
IntermediateStorageDefinition,
"intermediate storage",
)
config_mapped_execution_configs = config_map_objects(
config_value, mode_def.executor_defs, "execution", ExecutorDefinition, "executor"
)
resource_defs = pipeline_def.get_mode_definition(mode).resource_defs
resource_configs = config_value.get("resources", {})
config_mapped_resource_configs = config_map_resources(resource_defs, resource_configs)
config_mapped_logger_configs = config_map_loggers(pipeline_def, config_value, mode)
solid_config_dict = composite_descent(
pipeline_def, config_value.get("solids", {}), mode_def.resource_defs
)
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(config_mapped_execution_configs),
intermediate_storage=IntermediateStorageConfig.from_dict(
config_mapped_intermediate_storage_configs
),
loggers=config_mapped_logger_configs,
original_config_dict=run_config,
resources=config_mapped_resource_configs,
mode=mode,
)
def __init__(
self,
config,
default_value=FIELD_NO_DEFAULT_PROVIDED,
is_optional=None,
is_required=None,
description=None,
):
from .validate import validate_config
from .post_process import post_process_config
self.config_type = check.inst(self._resolve_config_arg(config),
ConfigType)
self.description = check.opt_str_param(description, 'description')
check.opt_bool_param(is_optional, 'is_optional')
check.opt_bool_param(is_required, 'is_required')
canonical_is_required = canonicalize_backcompat_args(
new_val=is_required,
new_arg='is_required',
old_val=is_optional,
old_arg='is_optional',
coerce_old_to_new=lambda val: not val,
additional_warn_txt=
'"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.',
)
if canonical_is_required is True:
check.param_invariant(
default_value == FIELD_NO_DEFAULT_PROVIDED,
'default_value',
'required arguments should not specify default values',
)
self._default_value = default_value
# check explicit default value
if self.default_provided:
# invoke through property in case it is callable
value = self.default_value
evr = validate_config(self.config_type, value)
if not evr.success:
raise DagsterInvalidConfigError(
'Invalid default_value for Field.',
evr.errors,
default_value,
)
if canonical_is_required is None:
# neither is_required nor is_optional were specified
canonical_is_required = not all_optional_type(self.config_type)
# on implicitly optional - set the default value
# by resolving the defaults of the type
if not canonical_is_required and self._default_value == FIELD_NO_DEFAULT_PROVIDED:
evr = post_process_config(self.config_type, None)
if not evr.success:
raise DagsterInvalidConfigError(
'Unable to resolve implicit default_value for Field.',
evr.errors,
None,
)
self._default_value = evr.value
self._is_required = canonical_is_required
def build(pipeline_def, run_config=None, mode=None):
"""This method validates a given run config against the pipeline config schema. If
successful, we instantiate an EnvironmentConfig object.
In case the run_config is invalid, this method raises a DagsterInvalidConfigError
"""
from dagster.config.validate import process_config
from dagster.core.definitions.executor import ExecutorDefinition
from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition
from dagster.core.definitions.system_storage import SystemStorageDefinition
from .composite_descent import composite_descent
check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition)
run_config = check.opt_dict_param(run_config, "run_config")
check.opt_str_param(mode, "mode")
mode = mode or pipeline_def.get_default_mode_name()
environment_type = create_environment_type(pipeline_def, mode)
config_evr = process_config(environment_type, run_config)
if not config_evr.success:
raise DagsterInvalidConfigError(
"Error in config for pipeline {}".format(pipeline_def.name),
config_evr.errors,
run_config,
)
config_value = config_evr.value
mode_def = pipeline_def.get_mode_definition(mode)
config_mapped_intermediate_storage_configs = config_map_objects(
config_value,
mode_def.intermediate_storage_defs,
"intermediate_storage",
IntermediateStorageDefinition,
"intermediate storage",
)
# TODO: replace this with a simple call to from_dict of config_mapped_intermediate_storage_configs when ready to fully deprecate
# TODO: tracking: https://github.com/dagster-io/dagster/issues/2705
temp_intermed = config_mapped_intermediate_storage_configs
if config_value.get("storage") and temp_intermed is None:
temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}}
config_mapped_execution_configs = config_map_objects(
config_value, mode_def.executor_defs, "execution",
ExecutorDefinition, "executor")
config_mapped_system_storage_configs = config_map_objects(
config_value,
mode_def.system_storage_defs,
"storage",
SystemStorageDefinition,
"system storage",
)
config_mapped_resource_configs = config_map_resources(
pipeline_def, config_value, mode)
config_mapped_logger_configs = config_map_loggers(
pipeline_def, config_value, mode)
solid_config_dict = composite_descent(pipeline_def,
config_value.get("solids", {}))
return EnvironmentConfig(
solids=solid_config_dict,
execution=ExecutionConfig.from_dict(
config_mapped_execution_configs),
storage=StorageConfig.from_dict(
config_mapped_system_storage_configs),
intermediate_storage=IntermediateStorageConfig.from_dict(
temp_intermed),
loggers=config_mapped_logger_configs,
original_config_dict=run_config,
resources=config_mapped_resource_configs,
)
def __init__(
self,
config,
default_value=FIELD_NO_DEFAULT_PROVIDED,
is_optional=None,
is_required=None,
description=None,
):
from .validate import validate_config
from .post_process import resolve_defaults
self.config_type = check.inst(self._resolve_config_arg(config), ConfigType)
self.description = check.opt_str_param(description, 'description')
check.opt_bool_param(is_optional, 'is_optional')
check.opt_bool_param(is_required, 'is_required')
if default_value != FIELD_NO_DEFAULT_PROVIDED:
check.param_invariant(
not (callable(default_value)), 'default_value', 'default_value cannot be a callable'
)
canonical_is_required = canonicalize_backcompat_args(
new_val=is_required,
new_arg='is_required',
old_val=is_optional,
old_arg='is_optional',
coerce_old_to_new=lambda val: not val,
additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.',
)
if canonical_is_required is True:
check.param_invariant(
default_value == FIELD_NO_DEFAULT_PROVIDED,
'default_value',
'required arguments should not specify default values',
)
self._default_value = default_value
# check explicit default value
if self.default_provided:
if self.config_type.kind == ConfigTypeKind.ENUM and is_enum_value(default_value):
raise DagsterInvalidDefinitionError(
(
'You have passed into a python enum value as the default value '
'into of a config enum type {name}. You must pass in the underlying '
'string represention as the default value. One of {value_set}.'
).format(
value_set=[ev.config_value for ev in self.config_type.enum_values],
name=self.config_type.given_name,
)
)
evr = validate_config(self.config_type, default_value)
if not evr.success:
raise DagsterInvalidConfigError(
'Invalid default_value for Field.', evr.errors, default_value,
)
if canonical_is_required is None:
# neither is_required nor is_optional were specified
canonical_is_required = not all_optional_type(self.config_type)
# on implicitly optional - set the default value
# by resolving the defaults of the type
if not canonical_is_required and not self.default_provided:
evr = resolve_defaults(self.config_type, None)
if not evr.success:
raise DagsterInvalidConfigError(
'Unable to resolve implicit default_value for Field.', evr.errors, None,
)
self._default_value = evr.value
self._is_required = canonical_is_required