def test_allow_unknown_in_anyof_schema():
schema = S.Dict(
allow_unknown=True,
anyof=[S.SubSchema(x=S.String()),
S.SubSchema(y=S.String())])
val = {'x': 'foo', 'extra': 'bar'}
normalize_schema(schema, val, allow_unknown=False) == val
def test_regex():
schema = S.String(regex=r'\d+')
assert normalize_schema(schema, '3000') == '3000'
with pytest.raises(E.RegexMismatch) as ei:
normalize_schema(schema, 'foo')
assert ei.value.value == 'foo'
assert ei.value.regex == r'\d+'
def _condition_schema(operator,
_op,
scalar=True,
aggr=False,
label_required=False):
"""Build a schema that expresses an (optionally labeled) boolean
expression.
For instance:
condition:
field: age
label: 'over 80'
gt: 80
"""
allowed_values = SCALAR_TYPES if scalar else SCALAR_TYPES + [S.List()]
field = "aggregated_field" if aggr else "non_aggregated_field"
cond_schema = {
"field": field,
"label": S.String(required=label_required),
operator: {
"anyof": allowed_values
},
}
_condition_schema = S.Dict(
allow_unknown=False,
schema=cond_schema,
coerce_post=ConditionPost(operator, _op, scalar),
)
return _condition_schema
def test_nullable_with_anyof():
"""This is the second reason that sureberus exists."""
anyof = {
'nullable': True,
'anyof': [S.Integer(), S.String()],
}
assert normalize_schema(anyof, None) == None
def test_coerce_raises():
"""If a coerce raises, it is wrapped in a CoerceUnexpectedError."""
schema = S.Dict(
schema={'key': S.String(required=False, coerce=lambda x: 1 / 0)})
with pytest.raises(E.CoerceUnexpectedError) as ei:
normalize_schema(schema, {'key': 'hello'})
assert ei.value.value == 'hello'
assert type(ei.value.exception) == ZeroDivisionError
def test_excludes_single():
schema = S.Dict(schema={'x': S.String(excludes='other')})
with pytest.raises(E.DisallowedField) as ei:
normalize_schema(schema, {
'x': 'foo',
'other': 'bar'
},
allow_unknown=True)
def _field_schema(aggr=True, required=True):
"""Make a field schema that either aggregates or doesn't. """
if aggr:
ag = S.String(
required=False,
allowed=list(aggregations.keys()),
default=default_aggregation,
nullable=True,
)
else:
ag = S.String(
required=False,
allowed=[no_aggregation, None],
default=no_aggregation,
nullable=True,
)
operator = S.Dict({
"operator": S.String(allowed=["+", "-", "/", "*"]),
"field": S.String()
})
return S.Dict(
schema={
"value":
S.String(),
"aggregation":
ag,
"ref":
S.String(required=False),
"condition":
"condition",
"operators":
S.List(schema=operator, required=False),
# Performs a dividebyzero safe sql division
"divide_by":
S.Dict(required=False, schema="aggregated_field"),
# Performs casting
"as":
S.String(
required=False,
allowed=list(sqlalchemy_datatypes.keys()),
coerce=_to_lowercase,
),
# Performs coalescing
"default": {
"anyof": SCALAR_TYPES,
"required": False
},
# Should the value be used directly in sql
"_use_raw_value":
S.Boolean(required=False),
},
coerce=_coerce_string_into_field,
coerce_post=_field_post,
allow_unknown=False,
required=required,
)
def test_validator_raises():
"""If a validator raises, it is wrapped in a ValidatorUnexpectedError."""
schema = S.Dict(
schema={
'key': S.String(required=False, validator=lambda f, v, e: 1 / 0)
})
with pytest.raises(E.ValidatorUnexpectedError) as ei:
normalize_schema(schema, {'key': 'hello'})
assert ei.value.field == 'key'
assert ei.value.value == 'hello'
assert type(ei.value.exception) == ZeroDivisionError
def test_default_setter_raises():
"""If a default_setter raises, it is wrapped in a DefaultSetterUnexpectedError."""
schema = S.Dict(
schema={
'key': S.String(required=False, default_setter=lambda x: 1 / 0)
})
with pytest.raises(E.DefaultSetterUnexpectedError) as ei:
normalize_schema(schema, {})
assert ei.value.key == 'key'
assert ei.value.value == {}
assert type(ei.value.exception) == ZeroDivisionError
def test_anyof_with_normalization():
"""THIS IS THE WHOLE REASON FOR SUREBERUS TO EXIST"""
# We want to support
# ANY OF:
# - {'image': str, 'opacity': {'type': 'integer', 'default': 100}}
# - {'gradient': ...}
# And when you normalize this, you actually get the `default` applied in the
# result, if that rule matches!
anyof = S.Dict(anyof=[
S.SubSchema(gradient=S.String()),
S.SubSchema(image=S.String(), opacity=S.Integer(default=100))
])
gfoo = {'gradient': 'foo'}
assert normalize_schema(anyof, gfoo) == gfoo
ifoo_with_opacity = {'image': 'foo', 'opacity': 99}
assert normalize_schema(anyof, ifoo_with_opacity) == ifoo_with_opacity
ifoo_with_default = {'image': 'foo'}
assert normalize_schema(anyof, ifoo_with_default) == {
'image': 'foo',
'opacity': 100
}
def test_default_setter_in_starof():
"""If a default setter raises inside of a *of-rule, it is treated as the
rule not validating
"""
called = []
def blow_up(x):
called.append(True)
1 / 0
anyof = {
'allow_unknown':
True,
'anyof': [
S.Dict(required=False,
schema={
'foo': S.String(required=False, default_setter=blow_up)
}),
S.Dict(required=False, schema={'bar': S.String(required=False)}),
]
}
assert normalize_schema(anyof, {'bar': 'baz'}) == {'bar': 'baz'}
assert called == [True]
def _full_condition_schema(**kwargs):
"""Conditions can be a field with an operator, like this yaml example
condition:
field: foo
gt: 22
Or conditions can be a list of and-ed and or-ed conditions
condition:
or:
- field: foo
gt: 22
- field: foo
lt: 0
:param aggr: Build the condition with aggregate fields (default is False)
"""
label_required = kwargs.get("label_required", False)
# Handle conditions where there's an
operator_condition = S.Dict(
choose_schema=S.when_key_exists({
"gt":
_condition_schema("gt", "__gt__", **kwargs),
"gte":
_condition_schema("gte", "__ge__", **kwargs),
"ge":
_condition_schema("ge", "__ge__", **kwargs),
"lt":
_condition_schema("lt", "__lt__", **kwargs),
"lte":
_condition_schema("lte", "__le__", **kwargs),
"le":
_condition_schema("le", "__le__", **kwargs),
"eq":
_condition_schema("eq", "__eq__", **kwargs),
"ne":
_condition_schema("ne", "__ne__", **kwargs),
"like":
_condition_schema("like", "like", **kwargs),
"ilike":
_condition_schema("ilike", "ilike", **kwargs),
"in":
_condition_schema("in", "in_", scalar=False, **kwargs),
"notin":
_condition_schema("notin", "notin", scalar=False, **kwargs),
"between":
_condition_schema("between", "between", scalar=False, **kwargs),
"or":
S.Dict(
schema={
"or": S.List(schema="condition"),
"label": S.String(required=label_required),
}),
"and":
S.Dict(
schema={
"and": S.List(schema="condition"),
"label": S.String(required=label_required),
}),
# A reference to another condition
"ref":
S.Dict(schema={"ref": S.String()}),
}),
required=False,
coerce=_coerce_string_into_condition_ref,
)
return {
"registry": {
"condition": operator_condition,
"aggregated_field": _field_schema(aggr=True),
"non_aggregated_field": _field_schema(aggr=False),
},
"schema_ref": "condition",
}
def test_list_normalize():
schema = S.List(schema=S.Dict(schema={'x': S.String(default='')}))
result = normalize_schema(schema, [{}])
assert result == [{'x': ''}]
def test_oneof():
oneof = {'oneof': [S.Integer(), S.String()]}
assert normalize_schema(oneof, 3) == 3
assert normalize_schema(oneof, 'three') == 'three'
with pytest.raises(E.NoneMatched) as ei:
normalize_schema(oneof, object())
def _replace_references(shelf):
"""Iterate over the shelf and replace and field.value: @ references
with the field in another ingredient"""
for ingr in shelf.values():
_process_ingredient(ingr, shelf)
return shelf
condition_schema = _full_condition_schema(aggr=False)
quickselect_schema = S.List(
required=False,
schema=S.Dict(schema={
"condition": "condition",
"name": S.String(required=True)
}),
)
ingredient_schema_choices = {
"metric":
S.Dict(
allow_unknown=True,
schema={
"field": "aggregated_field",
"divide_by": "optional_aggregated_field",
"format": S.String(coerce=coerce_format, required=False),
"quickselects": quickselect_schema,
},
),
"dimension":
def test_allowed():
schema = S.String(allowed=['2', '3'])
assert normalize_schema(schema, '3') == '3'
with pytest.raises(E.DisallowedValue) as ei:
normalize_schema(schema, '4')
def test_allow_unknown_in_list_schema():
schema = S.List(allow_unknown=True,
schema=S.Dict(schema={'x': S.String()}))
val = [{'x': 'y', 'extra': 0}]
assert normalize_schema(schema, val, allow_unknown=False) == val
def test_excludes_only_if_exists():
schema = S.Dict(
allow_unknown=True,
schema={'this': S.String(required=False, excludes='other')})
assert normalize_schema(schema, {'other': 'foo'}) == {'other': 'foo'}
shelf_schema = S.Dict(choose_schema=S.when_key_is(
"_version",
{
"1": shelf_schema,
1: shelf_schema,
"2": parsed_shelf_schema,
2: parsed_shelf_schema,
},
default_choice="1",
))
# This schema is used with sureberus
recipe_schema = S.Dict(
schema={
"metrics":
S.List(schema=S.String(), required=False),
"dimensions":
S.List(schema=S.String(), required=False),
"filters":
S.List(schema={"oneof": [S.String(), "condition"]}, required=False),
"order_by":
S.List(schema=S.String(), required=False),
},
registry={
"aggregated_field":
_field_schema(aggr=True, required=True),
"optional_aggregated_field":
_field_schema(aggr=True, required=False),
"non_aggregated_field":
_field_schema(aggr=False, required=True),
"condition":
config = deepcopy(value)
config.pop("_config", None)
config.pop("_neta", None)
value["_config"] = config
return value
def add_version(v):
# Add version to a parsed ingredient
v["_version"] = "2"
return v
# A field that may OR MAY NOT contain an aggregation.
# It will be the transformers responsibility to add an aggregation if one is missing
field_schema = S.String(required=True)
labeled_condition_schema = S.Dict(schema={
"condition": field_schema,
"label": S.String(required=True)
})
# A full condition guaranteed to not contain an aggregation
named_condition_schema = S.Dict(schema={
"condition": field_schema,
"name": S.String(required=True)
})
format_schema = S.String(coerce=coerce_format, required=False)
metric_schema = S.Dict(
from copy import copy
from datetime import date, datetime
from dateutil.relativedelta import relativedelta
import dateparser
import inspect
from sqlalchemy.ext.declarative import DeclarativeMeta
from sqlalchemy.sql.base import ImmutableColumnCollection
from sureberus import schema as S
from recipe.exceptions import InvalidColumnError
SCALAR_TYPES = [S.Integer(), S.String(), S.Float(), S.Boolean()]
def _chain(*args):
"""Chain several coercers together"""
def fn(value):
for arg in args:
value = arg(value)
return value
return fn
def _make_sqlalchemy_datatype_lookup():
""" Build a dictionary of the allowed sqlalchemy casts """
from sqlalchemy.sql import sqltypes
d = {}
for name in dir(sqltypes):
sqltype = getattr(sqltypes, name)
if name.lower() not in d and name[0] != "_" and name != "NULLTYPE":
