def test_rename_with_both_attributes_present(): schema = S.Dict(schema={ 'foo': { 'rename': 'moo', 'coerce': str }, }, allow_unknown=True) val = {'foo': 1, 'moo': 2} assert normalize_schema(schema, val) == {'moo': '1'} # Yes, we can swap attributes schema = S.Dict(schema={ 'foo': { 'rename': 'moo', 'coerce': str }, 'moo': { 'rename': 'foo', 'coerce': str }, }, allow_unknown=True) val = {'foo': 1, 'moo': 2} assert normalize_schema(schema, val) == {'moo': '1', 'foo': '2'}
def _field_schema(aggr=True, required=True): """Make a field schema that either aggregates or doesn't. """ if aggr: ag = S.String( required=False, allowed=list(aggregations.keys()), default=default_aggregation, nullable=True, ) else: ag = S.String( required=False, allowed=[no_aggregation, None], default=no_aggregation, nullable=True, ) operator = S.Dict({ "operator": S.String(allowed=["+", "-", "/", "*"]), "field": S.String() }) return S.Dict( schema={ "value": S.String(), "aggregation": ag, "ref": S.String(required=False), "condition": "condition", "operators": S.List(schema=operator, required=False), # Performs a dividebyzero safe sql division "divide_by": S.Dict(required=False, schema="aggregated_field"), # Performs casting "as": S.String( required=False, allowed=list(sqlalchemy_datatypes.keys()), coerce=_to_lowercase, ), # Performs coalescing "default": { "anyof": SCALAR_TYPES, "required": False }, # Should the value be used directly in sql "_use_raw_value": S.Boolean(required=False), }, coerce=_coerce_string_into_field, coerce_post=_field_post, allow_unknown=False, required=required, )
def test_disallow_unknown_in_normalize_schema(): with pytest.raises(E.UnknownFields) as ei: normalize_schema(S.Dict(schema=id_int), { 'id': 3, 'foo': 'bar' }, allow_unknown=False)
def test_allow_unknown_in_anyof_schema(): schema = S.Dict( allow_unknown=True, anyof=[S.SubSchema(x=S.String()), S.SubSchema(y=S.String())]) val = {'x': 'foo', 'extra': 'bar'} normalize_schema(schema, val, allow_unknown=False) == val
def _condition_schema(operator, _op, scalar=True, aggr=False, label_required=False): """Build a schema that expresses an (optionally labeled) boolean expression. For instance: condition: field: age label: 'over 80' gt: 80 """ allowed_values = SCALAR_TYPES if scalar else SCALAR_TYPES + [S.List()] field = "aggregated_field" if aggr else "non_aggregated_field" cond_schema = { "field": field, "label": S.String(required=label_required), operator: { "anyof": allowed_values }, } _condition_schema = S.Dict( allow_unknown=False, schema=cond_schema, coerce_post=ConditionPost(operator, _op, scalar), ) return _condition_schema
def test_nested_error(): schema = {'nested': S.Dict(schema={'num': S.Integer()})} with pytest.raises(E.BadType) as ei: normalize_dict(schema, {'nested': {'num': 'three!'}}) assert ei.value.value == 'three!' assert ei.value.type_ == 'integer' assert ei.value.stack == ('nested', 'num')
def test_excludes_single(): schema = S.Dict(schema={'x': S.String(excludes='other')}) with pytest.raises(E.DisallowedField) as ei: normalize_schema(schema, { 'x': 'foo', 'other': 'bar' }, allow_unknown=True)
def test_rename(): schema = S.Dict(schema={ 'foo': { 'rename': 'moo' }, }) val = {'foo': 2} assert normalize_schema(schema, val) == {'moo': 2}
def test_coerce_raises(): """If a coerce raises, it is wrapped in a CoerceUnexpectedError.""" schema = S.Dict( schema={'key': S.String(required=False, coerce=lambda x: 1 / 0)}) with pytest.raises(E.CoerceUnexpectedError) as ei: normalize_schema(schema, {'key': 'hello'}) assert ei.value.value == 'hello' assert type(ei.value.exception) == ZeroDivisionError
def test_rename_with_coerce(): schema = S.Dict(schema={ 'foo': { 'rename': 'moo', 'coerce': str }, }) val = {'foo': 2} assert normalize_schema(schema, val) == {'moo': '2'}
def test_rename_with_maxlength(): schema = S.Dict(schema={ 'foo': { 'rename': 'moo', 'maxlength': 3 }, }) val = {'foo': 'fooob'} with pytest.raises(E.MaxLengthExceeded): assert normalize_schema(schema, val)
def test_default_setter_raises(): """If a default_setter raises, it is wrapped in a DefaultSetterUnexpectedError.""" schema = S.Dict( schema={ 'key': S.String(required=False, default_setter=lambda x: 1 / 0) }) with pytest.raises(E.DefaultSetterUnexpectedError) as ei: normalize_schema(schema, {}) assert ei.value.key == 'key' assert ei.value.value == {} assert type(ei.value.exception) == ZeroDivisionError
def test_validator_raises(): """If a validator raises, it is wrapped in a ValidatorUnexpectedError.""" schema = S.Dict( schema={ 'key': S.String(required=False, validator=lambda f, v, e: 1 / 0) }) with pytest.raises(E.ValidatorUnexpectedError) as ei: normalize_schema(schema, {'key': 'hello'}) assert ei.value.field == 'key' assert ei.value.value == 'hello' assert type(ei.value.exception) == ZeroDivisionError
def test_rename_with_default(): schema = S.Dict(schema={ 'foo': { 'rename': 'moo', 'type': 'boolean', 'default': True }, }) val = {} assert normalize_schema(schema, val) == {'moo': True} val = {'foo': False} assert normalize_schema(schema, val) == {'moo': False}
def test_default_setter_in_starof(): """If a default setter raises inside of a *of-rule, it is treated as the rule not validating """ called = [] def blow_up(x): called.append(True) 1 / 0 anyof = { 'allow_unknown': True, 'anyof': [ S.Dict(required=False, schema={ 'foo': S.String(required=False, default_setter=blow_up) }), S.Dict(required=False, schema={'bar': S.String(required=False)}), ] } assert normalize_schema(anyof, {'bar': 'baz'}) == {'bar': 'baz'} assert called == [True]
def test_default_setter(): old_dict = {'foo': 0} schema = S.Dict( schema={ 'foo': S.Integer(), 'foo-incremented': { 'default_setter': lambda doc: doc['foo'] + 1 } }) new_dict = normalize_schema(schema, old_dict) assert old_dict == {'foo': 0} assert new_dict == {'foo': 0, 'foo-incremented': 1} assert normalize_schema(schema, { 'foo': 0, 'foo-incremented': 5 }) == { 'foo': 0, 'foo-incremented': 5 }
def test_anyof_with_normalization(): """THIS IS THE WHOLE REASON FOR SUREBERUS TO EXIST""" # We want to support # ANY OF: # - {'image': str, 'opacity': {'type': 'integer', 'default': 100}} # - {'gradient': ...} # And when you normalize this, you actually get the `default` applied in the # result, if that rule matches! anyof = S.Dict(anyof=[ S.SubSchema(gradient=S.String()), S.SubSchema(image=S.String(), opacity=S.Integer(default=100)) ]) gfoo = {'gradient': 'foo'} assert normalize_schema(anyof, gfoo) == gfoo ifoo_with_opacity = {'image': 'foo', 'opacity': 99} assert normalize_schema(anyof, ifoo_with_opacity) == ifoo_with_opacity ifoo_with_default = {'image': 'foo'} assert normalize_schema(anyof, ifoo_with_default) == { 'image': 'foo', 'opacity': 100 }
value["_config"] = config return value def add_version(v): # Add version to a parsed ingredient v["_version"] = "2" return v # A field that may OR MAY NOT contain an aggregation. # It will be the transformers responsibility to add an aggregation if one is missing field_schema = S.String(required=True) labeled_condition_schema = S.Dict(schema={ "condition": field_schema, "label": S.String(required=True) }) # A full condition guaranteed to not contain an aggregation named_condition_schema = S.Dict(schema={ "condition": field_schema, "name": S.String(required=True) }) format_schema = S.String(coerce=coerce_format, required=False) metric_schema = S.Dict( schema={ "field": field_schema, "format": format_schema, "quickselects": S.List(required=False,
from sureberus import schema as S from .config_schemas import _field_schema, _full_condition_schema, shelf_schema from .parsed_schemas import shelf_schema as parsed_shelf_schema shelf_schema = S.Dict(choose_schema=S.when_key_is( "_version", { "1": shelf_schema, 1: shelf_schema, "2": parsed_shelf_schema, 2: parsed_shelf_schema, }, default_choice="1", )) # This schema is used with sureberus recipe_schema = S.Dict( schema={ "metrics": S.List(schema=S.String(), required=False), "dimensions": S.List(schema=S.String(), required=False), "filters": S.List(schema={"oneof": [S.String(), "condition"]}, required=False), "order_by": S.List(schema=S.String(), required=False), }, registry={ "aggregated_field": _field_schema(aggr=True, required=True),
def _full_condition_schema(**kwargs): """Conditions can be a field with an operator, like this yaml example condition: field: foo gt: 22 Or conditions can be a list of and-ed and or-ed conditions condition: or: - field: foo gt: 22 - field: foo lt: 0 :param aggr: Build the condition with aggregate fields (default is False) """ label_required = kwargs.get("label_required", False) # Handle conditions where there's an operator_condition = S.Dict( choose_schema=S.when_key_exists({ "gt": _condition_schema("gt", "__gt__", **kwargs), "gte": _condition_schema("gte", "__ge__", **kwargs), "ge": _condition_schema("ge", "__ge__", **kwargs), "lt": _condition_schema("lt", "__lt__", **kwargs), "lte": _condition_schema("lte", "__le__", **kwargs), "le": _condition_schema("le", "__le__", **kwargs), "eq": _condition_schema("eq", "__eq__", **kwargs), "ne": _condition_schema("ne", "__ne__", **kwargs), "like": _condition_schema("like", "like", **kwargs), "ilike": _condition_schema("ilike", "ilike", **kwargs), "in": _condition_schema("in", "in_", scalar=False, **kwargs), "notin": _condition_schema("notin", "notin", scalar=False, **kwargs), "between": _condition_schema("between", "between", scalar=False, **kwargs), "or": S.Dict( schema={ "or": S.List(schema="condition"), "label": S.String(required=label_required), }), "and": S.Dict( schema={ "and": S.List(schema="condition"), "label": S.String(required=label_required), }), # A reference to another condition "ref": S.Dict(schema={"ref": S.String()}), }), required=False, coerce=_coerce_string_into_condition_ref, ) return { "registry": { "condition": operator_condition, "aggregated_field": _field_schema(aggr=True), "non_aggregated_field": _field_schema(aggr=False), }, "schema_ref": "condition", }
def _replace_references(shelf): """Iterate over the shelf and replace and field.value: @ references with the field in another ingredient""" for ingr in shelf.values(): _process_ingredient(ingr, shelf) return shelf condition_schema = _full_condition_schema(aggr=False) quickselect_schema = S.List( required=False, schema=S.Dict(schema={ "condition": "condition", "name": S.String(required=True) }), ) ingredient_schema_choices = { "metric": S.Dict( allow_unknown=True, schema={ "field": "aggregated_field", "divide_by": "optional_aggregated_field", "format": S.String(coerce=coerce_format, required=False), "quickselects": quickselect_schema, }, ), "dimension":
def test_list_normalize(): schema = S.List(schema=S.Dict(schema={'x': S.String(default='')})) result = normalize_schema(schema, [{}]) assert result == [{'x': ''}]
def test_allow_unknown_in_list_schema(): schema = S.List(allow_unknown=True, schema=S.Dict(schema={'x': S.String()})) val = [{'x': 'y', 'extra': 0}] assert normalize_schema(schema, val, allow_unknown=False) == val
def test_allow_unknown_in_dict_schema(): schema = S.Dict(allow_unknown=True, schema={}) assert normalize_schema(schema, {'x': 'y'}, allow_unknown=False) == { 'x': 'y' }
def test_excludes_only_if_exists(): schema = S.Dict( allow_unknown=True, schema={'this': S.String(required=False, excludes='other')}) assert normalize_schema(schema, {'other': 'foo'}) == {'other': 'foo'}