def test_allow_unknown_in_anyof_schema(): schema = S.Dict( allow_unknown=True, anyof=[S.SubSchema(x=S.String()), S.SubSchema(y=S.String())]) val = {'x': 'foo', 'extra': 'bar'} normalize_schema(schema, val, allow_unknown=False) == val
def test_regex(): schema = S.String(regex=r'\d+') assert normalize_schema(schema, '3000') == '3000' with pytest.raises(E.RegexMismatch) as ei: normalize_schema(schema, 'foo') assert ei.value.value == 'foo' assert ei.value.regex == r'\d+'
def _condition_schema(operator, _op, scalar=True, aggr=False, label_required=False): """Build a schema that expresses an (optionally labeled) boolean expression. For instance: condition: field: age label: 'over 80' gt: 80 """ allowed_values = SCALAR_TYPES if scalar else SCALAR_TYPES + [S.List()] field = "aggregated_field" if aggr else "non_aggregated_field" cond_schema = { "field": field, "label": S.String(required=label_required), operator: { "anyof": allowed_values }, } _condition_schema = S.Dict( allow_unknown=False, schema=cond_schema, coerce_post=ConditionPost(operator, _op, scalar), ) return _condition_schema
def test_nullable_with_anyof(): """This is the second reason that sureberus exists.""" anyof = { 'nullable': True, 'anyof': [S.Integer(), S.String()], } assert normalize_schema(anyof, None) == None
def test_coerce_raises(): """If a coerce raises, it is wrapped in a CoerceUnexpectedError.""" schema = S.Dict( schema={'key': S.String(required=False, coerce=lambda x: 1 / 0)}) with pytest.raises(E.CoerceUnexpectedError) as ei: normalize_schema(schema, {'key': 'hello'}) assert ei.value.value == 'hello' assert type(ei.value.exception) == ZeroDivisionError
def test_excludes_single(): schema = S.Dict(schema={'x': S.String(excludes='other')}) with pytest.raises(E.DisallowedField) as ei: normalize_schema(schema, { 'x': 'foo', 'other': 'bar' }, allow_unknown=True)
def _field_schema(aggr=True, required=True): """Make a field schema that either aggregates or doesn't. """ if aggr: ag = S.String( required=False, allowed=list(aggregations.keys()), default=default_aggregation, nullable=True, ) else: ag = S.String( required=False, allowed=[no_aggregation, None], default=no_aggregation, nullable=True, ) operator = S.Dict({ "operator": S.String(allowed=["+", "-", "/", "*"]), "field": S.String() }) return S.Dict( schema={ "value": S.String(), "aggregation": ag, "ref": S.String(required=False), "condition": "condition", "operators": S.List(schema=operator, required=False), # Performs a dividebyzero safe sql division "divide_by": S.Dict(required=False, schema="aggregated_field"), # Performs casting "as": S.String( required=False, allowed=list(sqlalchemy_datatypes.keys()), coerce=_to_lowercase, ), # Performs coalescing "default": { "anyof": SCALAR_TYPES, "required": False }, # Should the value be used directly in sql "_use_raw_value": S.Boolean(required=False), }, coerce=_coerce_string_into_field, coerce_post=_field_post, allow_unknown=False, required=required, )
def test_validator_raises(): """If a validator raises, it is wrapped in a ValidatorUnexpectedError.""" schema = S.Dict( schema={ 'key': S.String(required=False, validator=lambda f, v, e: 1 / 0) }) with pytest.raises(E.ValidatorUnexpectedError) as ei: normalize_schema(schema, {'key': 'hello'}) assert ei.value.field == 'key' assert ei.value.value == 'hello' assert type(ei.value.exception) == ZeroDivisionError
def test_default_setter_raises(): """If a default_setter raises, it is wrapped in a DefaultSetterUnexpectedError.""" schema = S.Dict( schema={ 'key': S.String(required=False, default_setter=lambda x: 1 / 0) }) with pytest.raises(E.DefaultSetterUnexpectedError) as ei: normalize_schema(schema, {}) assert ei.value.key == 'key' assert ei.value.value == {} assert type(ei.value.exception) == ZeroDivisionError
def test_anyof_with_normalization(): """THIS IS THE WHOLE REASON FOR SUREBERUS TO EXIST""" # We want to support # ANY OF: # - {'image': str, 'opacity': {'type': 'integer', 'default': 100}} # - {'gradient': ...} # And when you normalize this, you actually get the `default` applied in the # result, if that rule matches! anyof = S.Dict(anyof=[ S.SubSchema(gradient=S.String()), S.SubSchema(image=S.String(), opacity=S.Integer(default=100)) ]) gfoo = {'gradient': 'foo'} assert normalize_schema(anyof, gfoo) == gfoo ifoo_with_opacity = {'image': 'foo', 'opacity': 99} assert normalize_schema(anyof, ifoo_with_opacity) == ifoo_with_opacity ifoo_with_default = {'image': 'foo'} assert normalize_schema(anyof, ifoo_with_default) == { 'image': 'foo', 'opacity': 100 }
def test_default_setter_in_starof(): """If a default setter raises inside of a *of-rule, it is treated as the rule not validating """ called = [] def blow_up(x): called.append(True) 1 / 0 anyof = { 'allow_unknown': True, 'anyof': [ S.Dict(required=False, schema={ 'foo': S.String(required=False, default_setter=blow_up) }), S.Dict(required=False, schema={'bar': S.String(required=False)}), ] } assert normalize_schema(anyof, {'bar': 'baz'}) == {'bar': 'baz'} assert called == [True]
def _full_condition_schema(**kwargs): """Conditions can be a field with an operator, like this yaml example condition: field: foo gt: 22 Or conditions can be a list of and-ed and or-ed conditions condition: or: - field: foo gt: 22 - field: foo lt: 0 :param aggr: Build the condition with aggregate fields (default is False) """ label_required = kwargs.get("label_required", False) # Handle conditions where there's an operator_condition = S.Dict( choose_schema=S.when_key_exists({ "gt": _condition_schema("gt", "__gt__", **kwargs), "gte": _condition_schema("gte", "__ge__", **kwargs), "ge": _condition_schema("ge", "__ge__", **kwargs), "lt": _condition_schema("lt", "__lt__", **kwargs), "lte": _condition_schema("lte", "__le__", **kwargs), "le": _condition_schema("le", "__le__", **kwargs), "eq": _condition_schema("eq", "__eq__", **kwargs), "ne": _condition_schema("ne", "__ne__", **kwargs), "like": _condition_schema("like", "like", **kwargs), "ilike": _condition_schema("ilike", "ilike", **kwargs), "in": _condition_schema("in", "in_", scalar=False, **kwargs), "notin": _condition_schema("notin", "notin", scalar=False, **kwargs), "between": _condition_schema("between", "between", scalar=False, **kwargs), "or": S.Dict( schema={ "or": S.List(schema="condition"), "label": S.String(required=label_required), }), "and": S.Dict( schema={ "and": S.List(schema="condition"), "label": S.String(required=label_required), }), # A reference to another condition "ref": S.Dict(schema={"ref": S.String()}), }), required=False, coerce=_coerce_string_into_condition_ref, ) return { "registry": { "condition": operator_condition, "aggregated_field": _field_schema(aggr=True), "non_aggregated_field": _field_schema(aggr=False), }, "schema_ref": "condition", }
def test_list_normalize(): schema = S.List(schema=S.Dict(schema={'x': S.String(default='')})) result = normalize_schema(schema, [{}]) assert result == [{'x': ''}]
def test_oneof(): oneof = {'oneof': [S.Integer(), S.String()]} assert normalize_schema(oneof, 3) == 3 assert normalize_schema(oneof, 'three') == 'three' with pytest.raises(E.NoneMatched) as ei: normalize_schema(oneof, object())
def _replace_references(shelf): """Iterate over the shelf and replace and field.value: @ references with the field in another ingredient""" for ingr in shelf.values(): _process_ingredient(ingr, shelf) return shelf condition_schema = _full_condition_schema(aggr=False) quickselect_schema = S.List( required=False, schema=S.Dict(schema={ "condition": "condition", "name": S.String(required=True) }), ) ingredient_schema_choices = { "metric": S.Dict( allow_unknown=True, schema={ "field": "aggregated_field", "divide_by": "optional_aggregated_field", "format": S.String(coerce=coerce_format, required=False), "quickselects": quickselect_schema, }, ), "dimension":
def test_allowed(): schema = S.String(allowed=['2', '3']) assert normalize_schema(schema, '3') == '3' with pytest.raises(E.DisallowedValue) as ei: normalize_schema(schema, '4')
def test_allow_unknown_in_list_schema(): schema = S.List(allow_unknown=True, schema=S.Dict(schema={'x': S.String()})) val = [{'x': 'y', 'extra': 0}] assert normalize_schema(schema, val, allow_unknown=False) == val
def test_excludes_only_if_exists(): schema = S.Dict( allow_unknown=True, schema={'this': S.String(required=False, excludes='other')}) assert normalize_schema(schema, {'other': 'foo'}) == {'other': 'foo'}
shelf_schema = S.Dict(choose_schema=S.when_key_is( "_version", { "1": shelf_schema, 1: shelf_schema, "2": parsed_shelf_schema, 2: parsed_shelf_schema, }, default_choice="1", )) # This schema is used with sureberus recipe_schema = S.Dict( schema={ "metrics": S.List(schema=S.String(), required=False), "dimensions": S.List(schema=S.String(), required=False), "filters": S.List(schema={"oneof": [S.String(), "condition"]}, required=False), "order_by": S.List(schema=S.String(), required=False), }, registry={ "aggregated_field": _field_schema(aggr=True, required=True), "optional_aggregated_field": _field_schema(aggr=True, required=False), "non_aggregated_field": _field_schema(aggr=False, required=True), "condition":
config = deepcopy(value) config.pop("_config", None) config.pop("_neta", None) value["_config"] = config return value def add_version(v): # Add version to a parsed ingredient v["_version"] = "2" return v # A field that may OR MAY NOT contain an aggregation. # It will be the transformers responsibility to add an aggregation if one is missing field_schema = S.String(required=True) labeled_condition_schema = S.Dict(schema={ "condition": field_schema, "label": S.String(required=True) }) # A full condition guaranteed to not contain an aggregation named_condition_schema = S.Dict(schema={ "condition": field_schema, "name": S.String(required=True) }) format_schema = S.String(coerce=coerce_format, required=False) metric_schema = S.Dict(
from copy import copy from datetime import date, datetime from dateutil.relativedelta import relativedelta import dateparser import inspect from sqlalchemy.ext.declarative import DeclarativeMeta from sqlalchemy.sql.base import ImmutableColumnCollection from sureberus import schema as S from recipe.exceptions import InvalidColumnError SCALAR_TYPES = [S.Integer(), S.String(), S.Float(), S.Boolean()] def _chain(*args): """Chain several coercers together""" def fn(value): for arg in args: value = arg(value) return value return fn def _make_sqlalchemy_datatype_lookup(): """ Build a dictionary of the allowed sqlalchemy casts """ from sqlalchemy.sql import sqltypes d = {} for name in dir(sqltypes): sqltype = getattr(sqltypes, name) if name.lower() not in d and name[0] != "_" and name != "NULLTYPE":