def source_topic_fields() -> List[Field]:
"""Mock source topic fields."""
fields = [
Field("time", int),
Field("value", float),
Field("excluded", int),
Field("nonnumeric", bool),
Field("inttype", int),
]
return fields
def make_fields(self) -> List[Field]:
"""Make fields for the example topics.
Returns
-------
fields : `list`
A list of fields mapping field name and type.
"""
# A source topic needs a timestamp field
time = Field(name="time", type=float)
fields = [time]
for n in range(self._nfields):
fields.append(Field(name=f"value{n}", type=float))
return fields
def _create_aggregation_fields(
fields: List[Field],
excluded_field_names: List[str],
operations: List[str],
) -> List[Field]:
"""Create aggregation topic fields based on the source topic fields.
Add the fields `time`, `window_size`, and `count` and fields for the
`min`, `mean`, `stdev`, `median`, and `max` statistics for every
numeric field in the source topic.
Fields in `excluded_field_names` are excluded from aggregation.
Parameters
----------
fields : `list` [`Field`]
List of fields to aggregate.
excluded_field_names : `list`
List of fields excluded from aggregation.
operations : `list`
List of operations to perform.
Returns
-------
aggregation_fields : `list` [`Field`]
List of aggregation fields.
"""
time = Field(name="time", type=float)
window_size = Field(name="window_size", type=float)
count = Field(name="count", type=int)
aggregation_fields = [time, window_size, count]
for field in fields:
if field.name in excluded_field_names:
logger.info(f"Excluding field {field.name}.")
continue
# Only numeric fields are aggregated
if field.type in (int, float):
for operation in operations:
f = Field(
name=f"{operation}_{field.name}",
type=float,
source_field_name=field.name,
operation=operation,
)
aggregation_fields.append(f)
return aggregation_fields
def test_invalid_operation() -> None:
"""Test for invalid operation."""
with pytest.raises(RuntimeError):
Field("field",
int,
source_field_name="source_field",
operation="maximum")
def aggregation_fields() -> List[Field]:
"""Mock aggregation fields."""
fields = [
Field("time", int),
Field("count", int),
Field("window_size", float),
Field("min_value", float, "value", "min"),
Field("mean_value", float, "value", "mean"),
Field("median_value", float, "value", "median"),
Field("stdev_value", float, "value", "stdev"),
Field("max_value", float, "value", "max"),
]
return fields
def test_record_class() -> None:
"""Test Faust Record creation."""
# make a simple Faust Record
Foo = make_record(
cls_name="Foo",
fields=[Field("bar", int)],
doc="Test record",
)
f = Foo(bar=0)
assert f.is_valid()
assert f.asdict() == {"bar": 0}
def test_hash() -> None:
"""Test if Field is hashable.
A Field must be hashable to be used with Faust.
"""
assert hash(
Field(
"field",
int,
source_field_name="source_field",
operation="mean",
))
async def test_get_fields(avro_schema: str) -> None:
"""Test `topic.get_fields()` method returning faust-avro types."""
topic = Topic(name="test-avro-schema",
registry_url="http://localhost:8081")
await topic.register(schema=avro_schema)
fields = await topic.get_fields()
assert Field("int_field", faust_avro.types.int32) in fields
assert Field("long_field", int) in fields
assert Field("float_field", faust_avro.types.float32) in fields
assert Field("double_field", float) in fields
assert Field("bytes_field", bytes) in fields
assert Field("string_field", str) in fields
def test_aggregation_fields(
source_topic_fields: List[Field],
excluded_field_names: List[str],
operations: List[str],
) -> None:
"""Test aggregation fields creation."""
aggregation_fields = Aggregator._create_aggregation_fields(
source_topic_fields, excluded_field_names, operations)
# `time`, `count` and `window_size` are added by the aggregator
assert Field("time", float) in aggregation_fields
assert Field("count", int) in aggregation_fields
assert Field("window_size", float) in aggregation_fields
# if there's `time` field in the source topic it is replaced
assert Field("time", int) not in aggregation_fields
# summary statistic fields added based on the the `value` field
assert Field("min_value", float) in aggregation_fields
assert Field("mean_value", float) in aggregation_fields
assert Field("median_value", float) in aggregation_fields
assert Field("max_value", float) in aggregation_fields
# field names added to the excluded_field_names list are not aggregated
assert Field("excluded", float) not in aggregation_fields
# non numeric fields are excluded
assert Field("nonnumeric", bool) not in aggregation_fields
assert Field("min_nonnumeric", float) not in aggregation_fields
# int type is aggregated as float
assert Field("min_inttype", float) in aggregation_fields