def _condition_schema(operator,
_op,
scalar=True,
aggr=False,
label_required=False):
"""Build a schema that expresses an (optionally labeled) boolean
expression.
For instance:
condition:
field: age
label: 'over 80'
gt: 80
"""
allowed_values = SCALAR_TYPES if scalar else SCALAR_TYPES + [S.List()]
field = "aggregated_field" if aggr else "non_aggregated_field"
cond_schema = {
"field": field,
"label": S.String(required=label_required),
operator: {
"anyof": allowed_values
},
}
_condition_schema = S.Dict(
allow_unknown=False,
schema=cond_schema,
coerce_post=ConditionPost(operator, _op, scalar),
)
return _condition_schema
def _field_schema(aggr=True, required=True):
"""Make a field schema that either aggregates or doesn't. """
if aggr:
ag = S.String(
required=False,
allowed=list(aggregations.keys()),
default=default_aggregation,
nullable=True,
)
else:
ag = S.String(
required=False,
allowed=[no_aggregation, None],
default=no_aggregation,
nullable=True,
)
operator = S.Dict({
"operator": S.String(allowed=["+", "-", "/", "*"]),
"field": S.String()
})
return S.Dict(
schema={
"value":
S.String(),
"aggregation":
ag,
"ref":
S.String(required=False),
"condition":
"condition",
"operators":
S.List(schema=operator, required=False),
# Performs a dividebyzero safe sql division
"divide_by":
S.Dict(required=False, schema="aggregated_field"),
# Performs casting
"as":
S.String(
required=False,
allowed=list(sqlalchemy_datatypes.keys()),
coerce=_to_lowercase,
),
# Performs coalescing
"default": {
"anyof": SCALAR_TYPES,
"required": False
},
# Should the value be used directly in sql
"_use_raw_value":
S.Boolean(required=False),
},
coerce=_coerce_string_into_field,
coerce_post=_field_post,
allow_unknown=False,
required=required,
)
def test_list_schema():
schema = S.List(schema=S.Integer())
val = [1, 2, 3]
assert normalize_schema(schema, val) == val
with pytest.raises(E.BadType) as ei:
normalize_schema(schema, [1, 'two', object()])
assert ei.value.value == 'two'
assert ei.value.stack == (1, )
"label": S.String(required=True)
})
# A full condition guaranteed to not contain an aggregation
named_condition_schema = S.Dict(schema={
"condition": field_schema,
"name": S.String(required=True)
})
format_schema = S.String(coerce=coerce_format, required=False)
metric_schema = S.Dict(
schema={
"field": field_schema,
"format": format_schema,
"quickselects": S.List(required=False,
schema=labeled_condition_schema),
},
allow_unknown=True,
)
dimension_schema = S.Dict(
schema={
"field":
field_schema,
"extra_fields":
S.List(
required=False,
schema=S.Dict(schema={
"field": field_schema,
"name": S.String(required=True)
}),
shelf_schema = S.Dict(choose_schema=S.when_key_is(
"_version",
{
"1": shelf_schema,
1: shelf_schema,
"2": parsed_shelf_schema,
2: parsed_shelf_schema,
},
default_choice="1",
))
# This schema is used with sureberus
recipe_schema = S.Dict(
schema={
"metrics":
S.List(schema=S.String(), required=False),
"dimensions":
S.List(schema=S.String(), required=False),
"filters":
S.List(schema={"oneof": [S.String(), "condition"]}, required=False),
"order_by":
S.List(schema=S.String(), required=False),
},
registry={
"aggregated_field":
_field_schema(aggr=True, required=True),
"optional_aggregated_field":
_field_schema(aggr=True, required=False),
"non_aggregated_field":
_field_schema(aggr=False, required=True),
"condition":
def _full_condition_schema(**kwargs):
"""Conditions can be a field with an operator, like this yaml example
condition:
field: foo
gt: 22
Or conditions can be a list of and-ed and or-ed conditions
condition:
or:
- field: foo
gt: 22
- field: foo
lt: 0
:param aggr: Build the condition with aggregate fields (default is False)
"""
label_required = kwargs.get("label_required", False)
# Handle conditions where there's an
operator_condition = S.Dict(
choose_schema=S.when_key_exists({
"gt":
_condition_schema("gt", "__gt__", **kwargs),
"gte":
_condition_schema("gte", "__ge__", **kwargs),
"ge":
_condition_schema("ge", "__ge__", **kwargs),
"lt":
_condition_schema("lt", "__lt__", **kwargs),
"lte":
_condition_schema("lte", "__le__", **kwargs),
"le":
_condition_schema("le", "__le__", **kwargs),
"eq":
_condition_schema("eq", "__eq__", **kwargs),
"ne":
_condition_schema("ne", "__ne__", **kwargs),
"like":
_condition_schema("like", "like", **kwargs),
"ilike":
_condition_schema("ilike", "ilike", **kwargs),
"in":
_condition_schema("in", "in_", scalar=False, **kwargs),
"notin":
_condition_schema("notin", "notin", scalar=False, **kwargs),
"between":
_condition_schema("between", "between", scalar=False, **kwargs),
"or":
S.Dict(
schema={
"or": S.List(schema="condition"),
"label": S.String(required=label_required),
}),
"and":
S.Dict(
schema={
"and": S.List(schema="condition"),
"label": S.String(required=label_required),
}),
# A reference to another condition
"ref":
S.Dict(schema={"ref": S.String()}),
}),
required=False,
coerce=_coerce_string_into_condition_ref,
)
return {
"registry": {
"condition": operator_condition,
"aggregated_field": _field_schema(aggr=True),
"non_aggregated_field": _field_schema(aggr=False),
},
"schema_ref": "condition",
}
def _replace_references(shelf):
"""Iterate over the shelf and replace and field.value: @ references
with the field in another ingredient"""
for ingr in shelf.values():
_process_ingredient(ingr, shelf)
return shelf
condition_schema = _full_condition_schema(aggr=False)
quickselect_schema = S.List(
required=False,
schema=S.Dict(schema={
"condition": "condition",
"name": S.String(required=True)
}),
)
ingredient_schema_choices = {
"metric":
S.Dict(
allow_unknown=True,
schema={
"field": "aggregated_field",
"divide_by": "optional_aggregated_field",
"format": S.String(coerce=coerce_format, required=False),
"quickselects": quickselect_schema,
},
),
def test_allow_unknown_in_list_schema():
schema = S.List(allow_unknown=True,
schema=S.Dict(schema={'x': S.String()}))
val = [{'x': 'y', 'extra': 0}]
assert normalize_schema(schema, val, allow_unknown=False) == val
def test_list_normalize():
schema = S.List(schema=S.Dict(schema={'x': S.String(default='')}))
result = normalize_schema(schema, [{}])
assert result == [{'x': ''}]
def test_list():
schema = S.List()
val = [1, 'two', object()]
assert normalize_schema(schema, val) == val
def test_oneof_only_one():
oneof = {'oneof': [{'maxlength': 3}, S.List()]}
with pytest.raises(E.MoreThanOneMatched) as ei:
normalize_schema(oneof, [0])
