def test_correct_env_type_names_for_named(): pipeline_def = define_multi_mode_with_resources_pipeline() mult_type_name = create_environment_type(pipeline_def, 'mult_mode') assert mult_type_name.key == 'MultiModeWithResources.Mode.MultMode.Environment' assert mult_type_name.name == 'MultiModeWithResources.Mode.MultMode.Environment' assert ( mult_type_name.fields['resources'].config_type.key == 'MultiModeWithResources.Mode.MultMode.Resources' ) assert ( mult_type_name.fields['resources'].config_type.name == 'MultiModeWithResources.Mode.MultMode.Resources' ) add_type_name = create_environment_type(pipeline_def, 'add_mode') assert add_type_name.key == 'MultiModeWithResources.Mode.AddMode.Environment' assert add_type_name.name == 'MultiModeWithResources.Mode.AddMode.Environment' assert ( add_type_name.fields['resources'].config_type.key == 'MultiModeWithResources.Mode.AddMode.Resources' ) assert ( add_type_name.fields['resources'].config_type.name == 'MultiModeWithResources.Mode.AddMode.Resources' )
def build(pipeline, environment_dict=None, run_config=None): from dagster.core.types.config.evaluator.composite_descent import composite_descent from dagster.core.types.config.evaluator.validate import process_config check.inst_param(pipeline, 'pipeline', PipelineDefinition) environment_dict = check.opt_dict_param(environment_dict, 'environment_dict') run_config = check.opt_inst_param(run_config, 'run_config', IRunConfig, default=RunConfig()) mode = run_config.mode or pipeline.get_default_mode_name() environment_type = create_environment_type(pipeline, mode) config_evr = process_config(environment_type, environment_dict) if not config_evr.success: raise DagsterInvalidConfigError( 'Error in config for pipeline {}'.format(pipeline.name), config_evr.errors, environment_dict, ) config_value = config_evr.value solid_config_dict = composite_descent(pipeline, config_value.get('solids', {}), run_config) return EnvironmentConfig( solids=solid_config_dict, execution=ExecutionConfig.from_dict(config_value.get('execution')), storage=StorageConfig.from_dict(config_value.get('storage')), loggers=config_value.get('loggers'), original_config_dict=environment_dict, resources=config_value.get('resources'), )
def build(pipeline, environment_dict=None, run_config=None): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment') run_config = check.opt_inst_param(run_config, 'run_config', IRunConfig, default=RunConfig()) mode = run_config.mode or pipeline.get_default_mode_name() environment_type = create_environment_type(pipeline, mode) result = evaluate_config(environment_type, environment_dict, pipeline, run_config) if not result.success: raise DagsterInvalidConfigError(pipeline, result.errors, environment_dict) return EnvironmentConfig.from_config_value(result.value, environment_dict)
def build(pipeline_def, environment_dict=None, mode=None): '''This method validates a given environment dict against the pipeline config schema. If successful, we instantiate an EnvironmentConfig object. In case the environment_dict is invalid, this method raises a DagsterInvalidConfigError ''' from dagster.config.validate import process_config from .composite_descent import composite_descent check.inst_param(pipeline_def, 'pipeline_def', PipelineDefinition) environment_dict = check.opt_dict_param(environment_dict, 'environment_dict') check.opt_str_param(mode, 'mode') mode = mode or pipeline_def.get_default_mode_name() environment_type = create_environment_type(pipeline_def, mode) config_evr = process_config(environment_type, environment_dict) if not config_evr.success: raise DagsterInvalidConfigError( 'Error in config for pipeline {}'.format(pipeline_def.name), config_evr.errors, environment_dict, ) config_value = config_evr.value solid_config_dict = composite_descent(pipeline_def, config_value.get('solids', {})) return EnvironmentConfig( solids=solid_config_dict, execution=ExecutionConfig.from_dict(config_value.get('execution')), storage=StorageConfig.from_dict(config_value.get('storage')), loggers=config_value.get('loggers'), original_config_dict=environment_dict, resources=config_value.get('resources'), )