def test_config_int(): int_inst = resolve_to_config_type(Int) assert evaluate_config(int_inst, 1).success assert not evaluate_config(int_inst, None).success assert not evaluate_config(int_inst, 'r').success assert not int_inst.is_list assert not int_inst.is_nullable assert not (int_inst.is_nullable or int_inst.is_list)
def test_config_any(): any_inst = resolve_to_config_type(None) assert evaluate_config(any_inst, 1).success assert evaluate_config(any_inst, None).success assert evaluate_config(any_inst, 'r').success assert not any_inst.is_list assert not any_inst.is_nullable assert any_inst.is_any
def test_resource_def_config_errors(): takes_int_resource_def = ResourceDefinition( resource_fn=lambda: None, config_field=Field(Dict({'required_int': Field(Int)})) ) resource_type = define_resource_cls('Parent', 'takes_int', takes_int_resource_def).inst() assert not evaluate_config(resource_type, 1).success assert not evaluate_config(resource_type, {}).success assert not evaluate_config(resource_type, {'config': {}}).success assert evaluate_config(resource_type, {'config': {'required_int': 2}}).success assert not evaluate_config(resource_type, {'config': {'required_int': 'kdjfkd'}}).success
def resolve_is_environment_config_valid(graphene_info, environment_schema, dagster_pipeline, environment_dict): check.inst_param(graphene_info, 'graphene_info', ResolveInfo) check.inst_param(environment_schema, 'environment_schema', EnvironmentSchema) check.inst_param(dagster_pipeline, 'dagster_pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) validated_config = evaluate_config(environment_schema.environment_type, environment_dict, dagster_pipeline) dauphin_pipeline = graphene_info.schema.type_named('Pipeline')( dagster_pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return graphene_info.schema.type_named('PipelineConfigValidationValid')( dauphin_pipeline)
def get_validated_config(graphene_info, dauphin_pipeline, environment_dict, mode): check.opt_dict_param(environment_dict, 'environment_dict', key_type=str) check.str_param(mode, 'mode') pipeline = dauphin_pipeline.get_dagster_pipeline() environment_schema = create_environment_schema(pipeline, mode) validated_config = evaluate_config(environment_schema.environment_type, environment_dict, pipeline) if not validated_config.success: raise UserFacingGraphQLError( graphene_info.schema.type_named('PipelineConfigValidationInvalid')( pipeline=dauphin_pipeline, errors=[ graphene_info.schema.type_named( 'PipelineConfigValidationError').from_dagster_error( graphene_info, err) for err in validated_config.errors ], )) return validated_config
def test_int_field(snapshot): config_field = Field(Dict({'int_field': Field(Int)})) assert_config_type_snapshot(snapshot, config_field) assert evaluate_config(config_field.config_type, { 'int_field': 1 }).value == { 'int_field': 1 }
def test_int_field(): config_field = Field(Dict({'int_field': Field(Int)})) assert evaluate_config(config_field.config_type, { 'int_field': 1 }).value == { 'int_field': 1 }
def dagster_instance_config(base_dir): dagster_config_dict = load_yaml_from_globs( os.path.join(base_dir, DAGSTER_CONFIG_YAML_FILENAME)) dagster_config_type = define_dagster_config_cls().inst() dagster_config = evaluate_config(dagster_config_type, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError(None, dagster_config.errors, dagster_config_dict) return dagster_config.value
def dagster_instance_config(base_dir, config_filename=DAGSTER_CONFIG_YAML_FILENAME, overrides=None): overrides = check.opt_dict_param(overrides, 'overrides') dagster_config_dict = merge_dicts( load_yaml_from_globs(os.path.join(base_dir, config_filename)), overrides ) dagster_config_type = define_dagster_config_cls().inst() dagster_config = evaluate_config(dagster_config_type, dagster_config_dict) if not dagster_config.success: raise DagsterInvalidConfigError(None, dagster_config.errors, dagster_config_dict) return dagster_config.value
def test_list_nullable_int(): lni = resolve_to_config_type(List[Optional[Int]]) assert evaluate_config(lni, [1]).success assert evaluate_config(lni, [1, 2]).success assert evaluate_config(lni, []).success assert evaluate_config(lni, [None]).success assert evaluate_config(lni, [1, None]).success assert not evaluate_config(lni, None).success assert not evaluate_config(lni, [1, 'absdf']).success
def test_list_int(): list_int = resolve_to_config_type(List[Int]) assert evaluate_config(list_int, [1]).success assert evaluate_config(list_int, [1, 2]).success assert evaluate_config(list_int, []).success assert not evaluate_config(list_int, [None]).success assert not evaluate_config(list_int, [1, None]).success assert not evaluate_config(list_int, None).success assert not evaluate_config(list_int, [1, 'absdf']).success
def create_environment_config(pipeline, environment_dict=None, run_config=None): check.inst_param(pipeline, 'pipeline', PipelineDefinition) check.opt_dict_param(environment_dict, 'environment') run_config = check.opt_inst_param(run_config, 'run_config', RunConfig, default=RunConfig()) mode = run_config.mode or pipeline.get_default_mode_name() environment_type = create_environment_type(pipeline, mode) result = evaluate_config(environment_type, environment_dict, pipeline, run_config) if not result.success: raise DagsterInvalidConfigError(pipeline, result.errors, environment_dict) return EnvironmentConfig.from_config_value(result.value, original_config_dict=environment_dict)
def test_bad_config(): configs_and_expected_errors = [ ( # Create disposition must match enum values {'create_disposition': 'this is not a valid create disposition'}, 'Value not in enum type BQCreateDisposition', ), ( # Dataset must be of form project_name.dataset_name {'default_dataset': 'this is not a valid dataset'}, 'Value at path root:solids:test:config:query_job_config:default_dataset is not valid. Expected "Dataset"', ), ( # Table must be of form project_name.dataset_name.table_name {'destination': 'this is not a valid table'}, 'Value at path root:solids:test:config:query_job_config:destination is not valid. Expected "Table"', ), ( # Priority must match enum values {'priority': 'this is not a valid priority'}, 'Value not in enum type BQPriority', ), ( # Schema update options must be a list {'schema_update_options': 'this is not valid schema update options'}, 'Value at path root:solids:test:config:query_job_config:schema_update_options must be list. Expected: [BQSchemaUpdateOption]', ), ( {'schema_update_options': ['this is not valid schema update options']}, 'Value not in enum type BQSchemaUpdateOption', ), ( {'write_disposition': 'this is not a valid write disposition'}, 'Value not in enum type BQWriteDisposition', ), ] @pipeline(mode_defs=bq_modes()) def test_config_pipeline(): bq_solid_for_queries(['SELECT 1']).alias('test')() # pylint: disable=no-value-for-parameter env_type = create_environment_type(test_config_pipeline) for config_fragment, error_message in configs_and_expected_errors: config = {'solids': {'test': {'config': {'query_job_config': config_fragment}}}} result = evaluate_config(env_type, config, test_config_pipeline) assert result.errors[0].message == error_message
def rehydrate(self, **constructor_kwargs): from dagster.core.errors import DagsterInvalidConfigError from dagster.core.types.evaluator import evaluate_config module = importlib.import_module(self.module_name) klass = getattr(module, self.class_name) check.subclass_param( klass, 'class {class_name} in module {module_name}'.format( class_name=self.class_name, module_name=self.module_name), ConfigurableClass, ) config_dict = yaml.load(self.config_yaml) result = evaluate_config(klass.config_type().inst(), config_dict) if not result.success: raise DagsterInvalidConfigError(None, result.errors, config_dict) constructor_kwargs['inst_data'] = self return klass.from_config_value(result.value, **constructor_kwargs)
def rehydrate(self, **constructor_kwargs): from dagster.core.errors import DagsterInvalidConfigError from dagster.core.types.evaluator import evaluate_config try: module = importlib.import_module(self.module_name) except seven.ModuleNotFoundError: check.invariant( False, 'Couldn\'t import module {module_name} when attempting to rehydrate the ' 'configurable class {configurable_class}'.format( module_name=self.module_name, configurable_class=self.module_name + '.' + self.class_name, ), ) try: klass = getattr(module, self.class_name) except AttributeError: check.invariant( False, 'Couldn\'t find class {class_name} in module when attempting to rehydrate the ' 'configurable class {configurable_class}'.format( class_name=self.class_name, configurable_class=self.module_name + '.' + self.class_name, ), ) check.subclass_param( klass, 'class {class_name} in module {module_name}'.format( class_name=self.class_name, module_name=self.module_name), ConfigurableClass, ) config_dict = yaml.load(self.config_yaml) result = evaluate_config(klass.config_type().inst(), config_dict) if not result.success: raise DagsterInvalidConfigError(None, result.errors, config_dict) constructor_kwargs['inst_data'] = self return klass.from_config_value(result.value, **constructor_kwargs)
def test_optional_int(): optional_int_inst = resolve_to_config_type(Optional[Int]) assert evaluate_config(optional_int_inst, 1).success assert evaluate_config(optional_int_inst, None).success assert not evaluate_config(optional_int_inst, 'r').success
def throwing_evaluate_config_value(config_type, config_value): result = evaluate_config(config_type, config_value) if not result.success: raise DagsterEvaluateConfigValueError(result.errors[0].stack, result.errors[0].message) return result.value
def assert_eval_failure(config_type, value): assert not evaluate_config(config_type, value).success
def eval_config_value_from_dagster_type(dagster_type, value): return evaluate_config(resolve_to_config_type(dagster_type), value)
def assert_config_value_success(config_type, config_value, expected): result = evaluate_config(config_type, config_value) assert result.success assert result.value == expected
def test_config_enums(): assert evaluate_config(define_test_enum_type(), 'VALUE_ONE').success
def test_config_enum_error_none(): assert not evaluate_config(define_test_enum_type(), None).success
def test_config_enum_error_wrong_type(): assert not evaluate_config(define_test_enum_type(), 384934).success
def test_config_enum_error(): assert not evaluate_config(define_test_enum_type(), 'NOT_PRESENT').success