def test_config_int(): int_inst = resolve_to_config_type(Int) assert evaluate_config(int_inst, 1).success assert not evaluate_config(int_inst, None).success assert not evaluate_config(int_inst, 'r').success assert not int_inst.is_list assert not int_inst.is_nullable assert not (int_inst.is_nullable or int_inst.is_list)
def test_config_any(): any_inst = resolve_to_config_type(None) assert evaluate_config(any_inst, 1).success assert evaluate_config(any_inst, None).success assert evaluate_config(any_inst, 'r').success assert not any_inst.is_list assert not any_inst.is_nullable assert any_inst.is_any
def resolve_is_environment_config_valid(graphene_info, environment_schema, dagster_pipeline, environment_dict): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema) check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) validated_config = evaluate_config(environment_schema.environment_type, environment_dict, dagster_pipeline) dauphin_pipeline = graphene_info.schema.type_named('Pipeline')( dagster_pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return graphene_info.schema.type_named('PipelineConfigValidationValid')( dauphin_pipeline)
def test_int_field(snapshot): config_field = _to_field({'int_field': Int}) assert_config_type_snapshot(snapshot, config_field) assert evaluate_config(config_field.config_type, { 'int_field': 1 }).value == { 'int_field': 1 }
def test_list_nullable_int(): lni = resolve_to_config_type(List[Optional[Int]]) assert evaluate_config(lni, [1]).success assert evaluate_config(lni, [1, 2]).success assert evaluate_config(lni, []).success assert evaluate_config(lni, [None]).success assert evaluate_config(lni, [1, None]).success assert not evaluate_config(lni, None).success assert not evaluate_config(lni, [1, 'absdf']).success
def test_list_int(): list_int = resolve_to_config_type(List[Int]) assert evaluate_config(list_int, [1]).success assert evaluate_config(list_int, [1, 2]).success assert evaluate_config(list_int, []).success assert not evaluate_config(list_int, [None]).success assert not evaluate_config(list_int, [1, None]).success assert not evaluate_config(list_int, None).success assert not evaluate_config(list_int, [1, 'absdf']).success
def dagster_instance_config(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None): overrides = check.opt_dict_param(overrides, 'overrides') dagster_config_dict = merge_dicts( load_yaml_from_globs(os.path.join(base_dir, config_filename)), overrides) dagster_config_type = define_dagster_config_cls() dagster_config = evaluate_config(dagster_config_type, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError(None, dagster_config.errors, dagster_config_dict) return dagster_config.value
def build(pipeline, environment_dict=None, run_config=None): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment') run_config = check.opt_inst_param(run_config, 'run_config', IRunConfig, default=RunConfig()) mode = run_config.mode or pipeline.get_default_mode_name() environment_type = create_environment_type(pipeline, mode) result = evaluate_config(environment_type, environment_dict, pipeline, run_config) if not result.success: raise DagsterInvalidConfigError(pipeline, result.errors, environment_dict) return EnvironmentConfig.from_config_value(result.value, environment_dict)
def rehydrate(self, **constructor_kwargs): from dagster.core.errors import DagsterInvalidConfigError from dagster.core.types.config.evaluator import evaluate_config try: module = importlib.import_module(self.module_name) except seven.ModuleNotFoundError: check.failed( 'Couldn\'t import module {module_name} when attempting to rehydrate the ' 'configurable class {configurable_class}'.format( module_name=self.module_name, configurable_class=self.module_name + '.' + self.class_name, ) ) try: klass = getattr(module, self.class_name) except AttributeError: check.failed( 'Couldn\'t find class {class_name} in module when attempting to rehydrate the ' 'configurable class {configurable_class}'.format( class_name=self.class_name, configurable_class=self.module_name + '.' + self.class_name, ) ) if not issubclass(klass, ConfigurableClass): raise check.CheckError( klass, 'class {class_name} in module {module_name}'.format( class_name=self.class_name, module_name=self.module_name ), ConfigurableClass, ) config_dict = yaml.load(self.config_yaml) result = evaluate_config(klass.config_type(), config_dict) if not result.success: raise DagsterInvalidConfigError(None, result.errors, config_dict) return klass.from_config_value(self, result.value, **constructor_kwargs)
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict, mode): check.str_param(mode, 'mode') pipeline = dauphin_pipeline.get_dagster_pipeline() environment_schema = create_environment_schema(pipeline, mode) validated_config = evaluate_config(environment_schema.environment_type, environment_dict, pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return validated_config
def throwing_evaluate_config_value(config_type, config_value): result = evaluate_config(config_type, config_value) if not result.success: raise DagsterEvaluateConfigValueError(result.errors[0].stack, result.errors[0].message) return result.value
def test_optional_int(): optional_int_inst = resolve_to_config_type(Optional[Int]) assert evaluate_config(optional_int_inst, 1).success assert evaluate_config(optional_int_inst, None).success assert not evaluate_config(optional_int_inst, 'r').success
def eval_config_value_from_dagster_type(dagster_type, value): return evaluate_config(resolve_to_config_type(dagster_type), value)
def assert_eval_failure(config_type, value): assert not evaluate_config(config_type, value).success
def assert_config_value_success(config_type, config_value, expected): result = evaluate_config(config_type, config_value) assert result.success assert result.value == expected
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values { 'create_disposition': 'this is not a valid create disposition' }, 'Value not in enum type BQCreateDisposition', ), ( # Dataset must be of form project_name.dataset_name { 'default_dataset': 'this is not a valid dataset' }, 'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "_Dataset"', ), ( # Table must be of form project_name.dataset_name.table_name { 'destination': 'this is not a valid table' }, 'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "_Table"', ), ( # Priority must match enum values { 'priority': 'this is not a valid priority' }, 'Value not in enum type BQPriority', ), ( # Schema update options must be a list { 'schema_update_options': 'this is not valid schema update options' }, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( { 'schema_update_options': ['this is not valid schema update options'] }, 'Value not in enum type BQSchemaUpdateOption', ), ( { 'write_disposition': 'this is not a valid write disposition' }, 'Value not in enum type BQWriteDisposition', ), ] @pipeline(mode_defs=bq_modes()) def test_config_pipeline(): bq_solid_for_queries(['SELECT 1']).alias('test')() env_type = create_environment_type(test_config_pipeline) for config_fragment, error_message in configs_and_expected_errors: config = { 'solids': { 'test': { 'config': { 'query_job_config': config_fragment } } } } result = evaluate_config(env_type, config, test_config_pipeline) assert result.errors[0].message == error_message