def __init__( self, config, default_value=FIELD_NO_DEFAULT_PROVIDED, is_required=None, description=None, ): from .validate import validate_config from .post_process import resolve_defaults self.config_type = check.inst(self._resolve_config_arg(config), ConfigType) self.description = check.opt_str_param(description, 'description') check.opt_bool_param(is_required, 'is_required') if default_value != FIELD_NO_DEFAULT_PROVIDED: check.param_invariant(not (callable(default_value)), 'default_value', 'default_value cannot be a callable') if is_required is True: check.param_invariant( default_value == FIELD_NO_DEFAULT_PROVIDED, 'default_value', 'required arguments should not specify default values', ) self._default_value = default_value # check explicit default value if self.default_provided: if self.config_type.kind == ConfigTypeKind.ENUM and is_enum_value( default_value): raise DagsterInvalidDefinitionError(( 'You have passed into a python enum value as the default value ' 'into of a config enum type {name}. You must pass in the underlying ' 'string represention as the default value. One of {value_set}.' ).format( value_set=[ ev.config_value for ev in self.config_type.enum_values ], name=self.config_type.given_name, )) evr = validate_config(self.config_type, default_value) if not evr.success: raise DagsterInvalidConfigError( 'Invalid default_value for Field.', evr.errors, default_value, ) if is_required is None: is_required = not all_optional_type(self.config_type) # on implicitly optional - set the default value # by resolving the defaults of the type if not is_required and not self.default_provided: evr = resolve_defaults(self.config_type, None) if not evr.success: raise DagsterInvalidConfigError( 'Unable to resolve implicit default_value for Field.', evr.errors, None, ) self._default_value = evr.value self._is_required = is_required
def _launch_pipeline_execution_for_created_run(graphene_info, run_id): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.str_param(run_id, 'run_id') # First retrieve the pipeline run instance = graphene_info.context.instance pipeline_run = instance.get_run_by_id(run_id) if not pipeline_run: return graphene_info.schema.type_named('PipelineRunNotFoundError')( run_id) external_pipeline = get_external_pipeline_or_raise( graphene_info, pipeline_run.selector.name, pipeline_run.selector.solid_subset) # Run config validation # If there are any config errors, then inject them into the event log validated_config = ensure_valid_config(external_pipeline, pipeline_run.mode, pipeline_run.environment_dict) if not validated_config.success: # If the config is invalid, we construct a DagsterInvalidConfigError exception and # insert it into the event log. We also return a PipelineConfigValidationInvalid user facing # graphql error. # We currently re-use the engine events machinery to add the error to the event log, but # may need to create a new event type and instance method to handle these errors. invalid_config_exception = DagsterInvalidConfigError( 'Error in config for pipeline {}'.format(external_pipeline.name), validated_config.errors, pipeline_run.environment_dict, ) instance.report_engine_event( str(invalid_config_exception.message), pipeline_run, EngineEventData.engine_error( SerializableErrorInfo( invalid_config_exception.message, [], DagsterInvalidConfigError.__class__.__name__, None, )), ) instance.report_run_failed(pipeline_run) return DauphinPipelineConfigValidationInvalid.for_validation_errors( external_pipeline, validated_config.errors) try: pipeline_run = instance.launch_run(pipeline_run.run_id) except DagsterLaunchFailedError: error = serializable_error_info_from_exc_info(sys.exc_info()) instance.report_engine_event( error.message, pipeline_run, EngineEventData.engine_error(error), ) instance.report_run_failed(pipeline_run) return graphene_info.schema.type_named('LaunchPipelineRunSuccess')( run=graphene_info.schema.type_named('PipelineRun')(pipeline_run))
def get_run_execution_info_for_created_run_or_error( graphene_info, repository_location_name, repository_name, run_id ): ''' Previously created run could either be created in a different process *or* during the launchScheduledRun call where we want to have a record of a run the was created but have invalid configuration ''' check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.str_param(repository_location_name, 'repository_location_name') check.str_param(repository_name, 'repository_name') check.str_param(run_id, 'run_id') instance = graphene_info.context.instance pipeline_run = instance.get_run_by_id(run_id) if not pipeline_run: return graphene_info.schema.type_named('PipelineRunNotFoundError')(run_id) external_pipeline = get_external_pipeline_or_raise( graphene_info, PipelineSelector( location_name=repository_location_name, repository_name=repository_name, pipeline_name=pipeline_run.pipeline_name, solid_selection=list(pipeline_run.solids_to_execute) if pipeline_run.solids_to_execute else None, ), ) validated_config = validate_config_from_snap( external_pipeline.config_schema_snapshot, external_pipeline.root_config_key_for_mode(pipeline_run.mode), pipeline_run.run_config, ) if not validated_config.success: # If the config is invalid, we construct a DagsterInvalidConfigError exception and # insert it into the event log. We also return a PipelineConfigValidationInvalid user facing # graphql error. # We currently re-use the engine events machinery to add the error to the event log, but # may need to create a new event type and instance method to handle these errors. invalid_config_exception = DagsterInvalidConfigError( 'Error in config for pipeline {}'.format(external_pipeline.name), validated_config.errors, pipeline_run.run_config, ) instance.report_engine_event( str(invalid_config_exception.message), pipeline_run, EngineEventData.engine_error( SerializableErrorInfo( invalid_config_exception.message, [], DagsterInvalidConfigError.__class__.__name__, None, ) ), ) instance.report_run_failed(pipeline_run) return DauphinPipelineConfigValidationInvalid.for_validation_errors( external_pipeline, validated_config.errors ) return RunExecutionInfo(external_pipeline, pipeline_run)
def build(pipeline_def, run_config=None, mode=None): """This method validates a given run config against the pipeline config schema. If successful, we instantiate an EnvironmentConfig object. In case the run_config is invalid, this method raises a DagsterInvalidConfigError """ from dagster.config.validate import process_config from dagster.core.definitions.executor import ExecutorDefinition from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition from .composite_descent import composite_descent check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config = check.opt_dict_param(run_config, "run_config") check.opt_str_param(mode, "mode") mode = mode or pipeline_def.get_default_mode_name() environment_type = create_environment_type(pipeline_def, mode) config_evr = process_config( environment_type, run_config_storage_field_backcompat(run_config) ) if not config_evr.success: raise DagsterInvalidConfigError( "Error in config for pipeline {}".format(pipeline_def.name), config_evr.errors, run_config, ) config_value = config_evr.value mode_def = pipeline_def.get_mode_definition(mode) config_mapped_intermediate_storage_configs = config_map_objects( config_value, mode_def.intermediate_storage_defs, "intermediate_storage", IntermediateStorageDefinition, "intermediate storage", ) config_mapped_execution_configs = config_map_objects( config_value, mode_def.executor_defs, "execution", ExecutorDefinition, "executor" ) resource_defs = pipeline_def.get_mode_definition(mode).resource_defs resource_configs = config_value.get("resources", {}) config_mapped_resource_configs = config_map_resources(resource_defs, resource_configs) config_mapped_logger_configs = config_map_loggers(pipeline_def, config_value, mode) solid_config_dict = composite_descent( pipeline_def, config_value.get("solids", {}), mode_def.resource_defs ) return EnvironmentConfig( solids=solid_config_dict, execution=ExecutionConfig.from_dict(config_mapped_execution_configs), intermediate_storage=IntermediateStorageConfig.from_dict( config_mapped_intermediate_storage_configs ), loggers=config_mapped_logger_configs, original_config_dict=run_config, resources=config_mapped_resource_configs, mode=mode, )
def __init__( self, config, default_value=FIELD_NO_DEFAULT_PROVIDED, is_optional=None, is_required=None, description=None, ): from .validate import validate_config from .post_process import post_process_config self.config_type = check.inst(self._resolve_config_arg(config), ConfigType) self.description = check.opt_str_param(description, 'description') check.opt_bool_param(is_optional, 'is_optional') check.opt_bool_param(is_required, 'is_required') canonical_is_required = canonicalize_backcompat_args( new_val=is_required, new_arg='is_required', old_val=is_optional, old_arg='is_optional', coerce_old_to_new=lambda val: not val, additional_warn_txt= '"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.', ) if canonical_is_required is True: check.param_invariant( default_value == FIELD_NO_DEFAULT_PROVIDED, 'default_value', 'required arguments should not specify default values', ) self._default_value = default_value # check explicit default value if self.default_provided: # invoke through property in case it is callable value = self.default_value evr = validate_config(self.config_type, value) if not evr.success: raise DagsterInvalidConfigError( 'Invalid default_value for Field.', evr.errors, default_value, ) if canonical_is_required is None: # neither is_required nor is_optional were specified canonical_is_required = not all_optional_type(self.config_type) # on implicitly optional - set the default value # by resolving the defaults of the type if not canonical_is_required and self._default_value == FIELD_NO_DEFAULT_PROVIDED: evr = post_process_config(self.config_type, None) if not evr.success: raise DagsterInvalidConfigError( 'Unable to resolve implicit default_value for Field.', evr.errors, None, ) self._default_value = evr.value self._is_required = canonical_is_required
def build(pipeline_def, run_config=None, mode=None): """This method validates a given run config against the pipeline config schema. If successful, we instantiate an EnvironmentConfig object. In case the run_config is invalid, this method raises a DagsterInvalidConfigError """ from dagster.config.validate import process_config from dagster.core.definitions.executor import ExecutorDefinition from dagster.core.definitions.intermediate_storage import IntermediateStorageDefinition from dagster.core.definitions.system_storage import SystemStorageDefinition from .composite_descent import composite_descent check.inst_param(pipeline_def, "pipeline_def", PipelineDefinition) run_config = check.opt_dict_param(run_config, "run_config") check.opt_str_param(mode, "mode") mode = mode or pipeline_def.get_default_mode_name() environment_type = create_environment_type(pipeline_def, mode) config_evr = process_config(environment_type, run_config) if not config_evr.success: raise DagsterInvalidConfigError( "Error in config for pipeline {}".format(pipeline_def.name), config_evr.errors, run_config, ) config_value = config_evr.value mode_def = pipeline_def.get_mode_definition(mode) config_mapped_intermediate_storage_configs = config_map_objects( config_value, mode_def.intermediate_storage_defs, "intermediate_storage", IntermediateStorageDefinition, "intermediate storage", ) # TODO: replace this with a simple call to from_dict of config_mapped_intermediate_storage_configs when ready to fully deprecate # TODO: tracking: https://github.com/dagster-io/dagster/issues/2705 temp_intermed = config_mapped_intermediate_storage_configs if config_value.get("storage") and temp_intermed is None: temp_intermed = {EmptyIntermediateStoreBackcompatConfig(): {}} config_mapped_execution_configs = config_map_objects( config_value, mode_def.executor_defs, "execution", ExecutorDefinition, "executor") config_mapped_system_storage_configs = config_map_objects( config_value, mode_def.system_storage_defs, "storage", SystemStorageDefinition, "system storage", ) config_mapped_resource_configs = config_map_resources( pipeline_def, config_value, mode) config_mapped_logger_configs = config_map_loggers( pipeline_def, config_value, mode) solid_config_dict = composite_descent(pipeline_def, config_value.get("solids", {})) return EnvironmentConfig( solids=solid_config_dict, execution=ExecutionConfig.from_dict( config_mapped_execution_configs), storage=StorageConfig.from_dict( config_mapped_system_storage_configs), intermediate_storage=IntermediateStorageConfig.from_dict( temp_intermed), loggers=config_mapped_logger_configs, original_config_dict=run_config, resources=config_mapped_resource_configs, )
def __init__( self, config, default_value=FIELD_NO_DEFAULT_PROVIDED, is_optional=None, is_required=None, description=None, ): from .validate import validate_config from .post_process import resolve_defaults self.config_type = check.inst(self._resolve_config_arg(config), ConfigType) self.description = check.opt_str_param(description, 'description') check.opt_bool_param(is_optional, 'is_optional') check.opt_bool_param(is_required, 'is_required') if default_value != FIELD_NO_DEFAULT_PROVIDED: check.param_invariant( not (callable(default_value)), 'default_value', 'default_value cannot be a callable' ) canonical_is_required = canonicalize_backcompat_args( new_val=is_required, new_arg='is_required', old_val=is_optional, old_arg='is_optional', coerce_old_to_new=lambda val: not val, additional_warn_txt='"is_optional" deprecated in 0.7.0 and will be removed in 0.8.0. Users should use "is_required" instead.', ) if canonical_is_required is True: check.param_invariant( default_value == FIELD_NO_DEFAULT_PROVIDED, 'default_value', 'required arguments should not specify default values', ) self._default_value = default_value # check explicit default value if self.default_provided: if self.config_type.kind == ConfigTypeKind.ENUM and is_enum_value(default_value): raise DagsterInvalidDefinitionError( ( 'You have passed into a python enum value as the default value ' 'into of a config enum type {name}. You must pass in the underlying ' 'string represention as the default value. One of {value_set}.' ).format( value_set=[ev.config_value for ev in self.config_type.enum_values], name=self.config_type.given_name, ) ) evr = validate_config(self.config_type, default_value) if not evr.success: raise DagsterInvalidConfigError( 'Invalid default_value for Field.', evr.errors, default_value, ) if canonical_is_required is None: # neither is_required nor is_optional were specified canonical_is_required = not all_optional_type(self.config_type) # on implicitly optional - set the default value # by resolving the defaults of the type if not canonical_is_required and not self.default_provided: evr = resolve_defaults(self.config_type, None) if not evr.success: raise DagsterInvalidConfigError( 'Unable to resolve implicit default_value for Field.', evr.errors, None, ) self._default_value = evr.value self._is_required = canonical_is_required