def test_rename_with_both_attributes_present():
schema = S.Dict(schema={
'foo': {
'rename': 'moo',
'coerce': str
},
},
allow_unknown=True)
val = {'foo': 1, 'moo': 2}
assert normalize_schema(schema, val) == {'moo': '1'}
# Yes, we can swap attributes
schema = S.Dict(schema={
'foo': {
'rename': 'moo',
'coerce': str
},
'moo': {
'rename': 'foo',
'coerce': str
},
},
allow_unknown=True)
val = {'foo': 1, 'moo': 2}
assert normalize_schema(schema, val) == {'moo': '1', 'foo': '2'}
def _field_schema(aggr=True, required=True):
"""Make a field schema that either aggregates or doesn't. """
if aggr:
ag = S.String(
required=False,
allowed=list(aggregations.keys()),
default=default_aggregation,
nullable=True,
)
else:
ag = S.String(
required=False,
allowed=[no_aggregation, None],
default=no_aggregation,
nullable=True,
)
operator = S.Dict({
"operator": S.String(allowed=["+", "-", "/", "*"]),
"field": S.String()
})
return S.Dict(
schema={
"value":
S.String(),
"aggregation":
ag,
"ref":
S.String(required=False),
"condition":
"condition",
"operators":
S.List(schema=operator, required=False),
# Performs a dividebyzero safe sql division
"divide_by":
S.Dict(required=False, schema="aggregated_field"),
# Performs casting
"as":
S.String(
required=False,
allowed=list(sqlalchemy_datatypes.keys()),
coerce=_to_lowercase,
),
# Performs coalescing
"default": {
"anyof": SCALAR_TYPES,
"required": False
},
# Should the value be used directly in sql
"_use_raw_value":
S.Boolean(required=False),
},
coerce=_coerce_string_into_field,
coerce_post=_field_post,
allow_unknown=False,
required=required,
)
def test_disallow_unknown_in_normalize_schema():
with pytest.raises(E.UnknownFields) as ei:
normalize_schema(S.Dict(schema=id_int), {
'id': 3,
'foo': 'bar'
},
allow_unknown=False)
def test_allow_unknown_in_anyof_schema():
schema = S.Dict(
allow_unknown=True,
anyof=[S.SubSchema(x=S.String()),
S.SubSchema(y=S.String())])
val = {'x': 'foo', 'extra': 'bar'}
normalize_schema(schema, val, allow_unknown=False) == val
def _condition_schema(operator,
_op,
scalar=True,
aggr=False,
label_required=False):
"""Build a schema that expresses an (optionally labeled) boolean
expression.
For instance:
condition:
field: age
label: 'over 80'
gt: 80
"""
allowed_values = SCALAR_TYPES if scalar else SCALAR_TYPES + [S.List()]
field = "aggregated_field" if aggr else "non_aggregated_field"
cond_schema = {
"field": field,
"label": S.String(required=label_required),
operator: {
"anyof": allowed_values
},
}
_condition_schema = S.Dict(
allow_unknown=False,
schema=cond_schema,
coerce_post=ConditionPost(operator, _op, scalar),
)
return _condition_schema
def test_nested_error():
schema = {'nested': S.Dict(schema={'num': S.Integer()})}
with pytest.raises(E.BadType) as ei:
normalize_dict(schema, {'nested': {'num': 'three!'}})
assert ei.value.value == 'three!'
assert ei.value.type_ == 'integer'
assert ei.value.stack == ('nested', 'num')
def test_excludes_single():
schema = S.Dict(schema={'x': S.String(excludes='other')})
with pytest.raises(E.DisallowedField) as ei:
normalize_schema(schema, {
'x': 'foo',
'other': 'bar'
},
allow_unknown=True)
def test_rename():
schema = S.Dict(schema={
'foo': {
'rename': 'moo'
},
})
val = {'foo': 2}
assert normalize_schema(schema, val) == {'moo': 2}
def test_coerce_raises():
"""If a coerce raises, it is wrapped in a CoerceUnexpectedError."""
schema = S.Dict(
schema={'key': S.String(required=False, coerce=lambda x: 1 / 0)})
with pytest.raises(E.CoerceUnexpectedError) as ei:
normalize_schema(schema, {'key': 'hello'})
assert ei.value.value == 'hello'
assert type(ei.value.exception) == ZeroDivisionError
def test_rename_with_coerce():
schema = S.Dict(schema={
'foo': {
'rename': 'moo',
'coerce': str
},
})
val = {'foo': 2}
assert normalize_schema(schema, val) == {'moo': '2'}
def test_rename_with_maxlength():
schema = S.Dict(schema={
'foo': {
'rename': 'moo',
'maxlength': 3
},
})
val = {'foo': 'fooob'}
with pytest.raises(E.MaxLengthExceeded):
assert normalize_schema(schema, val)
def test_default_setter_raises():
"""If a default_setter raises, it is wrapped in a DefaultSetterUnexpectedError."""
schema = S.Dict(
schema={
'key': S.String(required=False, default_setter=lambda x: 1 / 0)
})
with pytest.raises(E.DefaultSetterUnexpectedError) as ei:
normalize_schema(schema, {})
assert ei.value.key == 'key'
assert ei.value.value == {}
assert type(ei.value.exception) == ZeroDivisionError
def test_validator_raises():
"""If a validator raises, it is wrapped in a ValidatorUnexpectedError."""
schema = S.Dict(
schema={
'key': S.String(required=False, validator=lambda f, v, e: 1 / 0)
})
with pytest.raises(E.ValidatorUnexpectedError) as ei:
normalize_schema(schema, {'key': 'hello'})
assert ei.value.field == 'key'
assert ei.value.value == 'hello'
assert type(ei.value.exception) == ZeroDivisionError
def test_rename_with_default():
schema = S.Dict(schema={
'foo': {
'rename': 'moo',
'type': 'boolean',
'default': True
},
})
val = {}
assert normalize_schema(schema, val) == {'moo': True}
val = {'foo': False}
assert normalize_schema(schema, val) == {'moo': False}
def test_default_setter_in_starof():
"""If a default setter raises inside of a *of-rule, it is treated as the
rule not validating
"""
called = []
def blow_up(x):
called.append(True)
1 / 0
anyof = {
'allow_unknown':
True,
'anyof': [
S.Dict(required=False,
schema={
'foo': S.String(required=False, default_setter=blow_up)
}),
S.Dict(required=False, schema={'bar': S.String(required=False)}),
]
}
assert normalize_schema(anyof, {'bar': 'baz'}) == {'bar': 'baz'}
assert called == [True]
def test_default_setter():
old_dict = {'foo': 0}
schema = S.Dict(
schema={
'foo': S.Integer(),
'foo-incremented': {
'default_setter': lambda doc: doc['foo'] + 1
}
})
new_dict = normalize_schema(schema, old_dict)
assert old_dict == {'foo': 0}
assert new_dict == {'foo': 0, 'foo-incremented': 1}
assert normalize_schema(schema, {
'foo': 0,
'foo-incremented': 5
}) == {
'foo': 0,
'foo-incremented': 5
}
def test_anyof_with_normalization():
"""THIS IS THE WHOLE REASON FOR SUREBERUS TO EXIST"""
# We want to support
# ANY OF:
# - {'image': str, 'opacity': {'type': 'integer', 'default': 100}}
# - {'gradient': ...}
# And when you normalize this, you actually get the `default` applied in the
# result, if that rule matches!
anyof = S.Dict(anyof=[
S.SubSchema(gradient=S.String()),
S.SubSchema(image=S.String(), opacity=S.Integer(default=100))
])
gfoo = {'gradient': 'foo'}
assert normalize_schema(anyof, gfoo) == gfoo
ifoo_with_opacity = {'image': 'foo', 'opacity': 99}
assert normalize_schema(anyof, ifoo_with_opacity) == ifoo_with_opacity
ifoo_with_default = {'image': 'foo'}
assert normalize_schema(anyof, ifoo_with_default) == {
'image': 'foo',
'opacity': 100
}
value["_config"] = config
return value
def add_version(v):
# Add version to a parsed ingredient
v["_version"] = "2"
return v
# A field that may OR MAY NOT contain an aggregation.
# It will be the transformers responsibility to add an aggregation if one is missing
field_schema = S.String(required=True)
labeled_condition_schema = S.Dict(schema={
"condition": field_schema,
"label": S.String(required=True)
})
# A full condition guaranteed to not contain an aggregation
named_condition_schema = S.Dict(schema={
"condition": field_schema,
"name": S.String(required=True)
})
format_schema = S.String(coerce=coerce_format, required=False)
metric_schema = S.Dict(
schema={
"field": field_schema,
"format": format_schema,
"quickselects": S.List(required=False,
from sureberus import schema as S
from .config_schemas import _field_schema, _full_condition_schema, shelf_schema
from .parsed_schemas import shelf_schema as parsed_shelf_schema
shelf_schema = S.Dict(choose_schema=S.when_key_is(
"_version",
{
"1": shelf_schema,
1: shelf_schema,
"2": parsed_shelf_schema,
2: parsed_shelf_schema,
},
default_choice="1",
))
# This schema is used with sureberus
recipe_schema = S.Dict(
schema={
"metrics":
S.List(schema=S.String(), required=False),
"dimensions":
S.List(schema=S.String(), required=False),
"filters":
S.List(schema={"oneof": [S.String(), "condition"]}, required=False),
"order_by":
S.List(schema=S.String(), required=False),
},
registry={
"aggregated_field":
_field_schema(aggr=True, required=True),
def _full_condition_schema(**kwargs):
"""Conditions can be a field with an operator, like this yaml example
condition:
field: foo
gt: 22
Or conditions can be a list of and-ed and or-ed conditions
condition:
or:
- field: foo
gt: 22
- field: foo
lt: 0
:param aggr: Build the condition with aggregate fields (default is False)
"""
label_required = kwargs.get("label_required", False)
# Handle conditions where there's an
operator_condition = S.Dict(
choose_schema=S.when_key_exists({
"gt":
_condition_schema("gt", "__gt__", **kwargs),
"gte":
_condition_schema("gte", "__ge__", **kwargs),
"ge":
_condition_schema("ge", "__ge__", **kwargs),
"lt":
_condition_schema("lt", "__lt__", **kwargs),
"lte":
_condition_schema("lte", "__le__", **kwargs),
"le":
_condition_schema("le", "__le__", **kwargs),
"eq":
_condition_schema("eq", "__eq__", **kwargs),
"ne":
_condition_schema("ne", "__ne__", **kwargs),
"like":
_condition_schema("like", "like", **kwargs),
"ilike":
_condition_schema("ilike", "ilike", **kwargs),
"in":
_condition_schema("in", "in_", scalar=False, **kwargs),
"notin":
_condition_schema("notin", "notin", scalar=False, **kwargs),
"between":
_condition_schema("between", "between", scalar=False, **kwargs),
"or":
S.Dict(
schema={
"or": S.List(schema="condition"),
"label": S.String(required=label_required),
}),
"and":
S.Dict(
schema={
"and": S.List(schema="condition"),
"label": S.String(required=label_required),
}),
# A reference to another condition
"ref":
S.Dict(schema={"ref": S.String()}),
}),
required=False,
coerce=_coerce_string_into_condition_ref,
)
return {
"registry": {
"condition": operator_condition,
"aggregated_field": _field_schema(aggr=True),
"non_aggregated_field": _field_schema(aggr=False),
},
"schema_ref": "condition",
}
def _replace_references(shelf):
"""Iterate over the shelf and replace and field.value: @ references
with the field in another ingredient"""
for ingr in shelf.values():
_process_ingredient(ingr, shelf)
return shelf
condition_schema = _full_condition_schema(aggr=False)
quickselect_schema = S.List(
required=False,
schema=S.Dict(schema={
"condition": "condition",
"name": S.String(required=True)
}),
)
ingredient_schema_choices = {
"metric":
S.Dict(
allow_unknown=True,
schema={
"field": "aggregated_field",
"divide_by": "optional_aggregated_field",
"format": S.String(coerce=coerce_format, required=False),
"quickselects": quickselect_schema,
},
),
"dimension":
def test_list_normalize():
schema = S.List(schema=S.Dict(schema={'x': S.String(default='')}))
result = normalize_schema(schema, [{}])
assert result == [{'x': ''}]
def test_allow_unknown_in_list_schema():
schema = S.List(allow_unknown=True,
schema=S.Dict(schema={'x': S.String()}))
val = [{'x': 'y', 'extra': 0}]
assert normalize_schema(schema, val, allow_unknown=False) == val
def test_allow_unknown_in_dict_schema():
schema = S.Dict(allow_unknown=True, schema={})
assert normalize_schema(schema, {'x': 'y'}, allow_unknown=False) == {
'x': 'y'
}
def test_excludes_only_if_exists():
schema = S.Dict(
allow_unknown=True,
schema={'this': S.String(required=False, excludes='other')})
assert normalize_schema(schema, {'other': 'foo'}) == {'other': 'foo'}