def test_render():
field = fields.Field("first_name", str, fields.NULL, metadata={"desc": "English Language Name"})
expected = {
"name": "first_name",
"type": ["string", fields.NULL],
"default": fields.NULL,
"desc": "English Language Name",
}
assert expected == field.render()
field = fields.Field("engine_name", str, fields.NULL)
expected = {
"name": "engine_name",
"type": ["string", fields.NULL],
"default": fields.NULL,
}
assert expected == field.render()
field = fields.Field(
"breed_name", str, fields.NULL, metadata={"encoding": "some_exotic_encoding", "doc": "Official Breed Name"},
)
expected = {
"name": "breed_name",
"type": ["string", fields.NULL],
"default": fields.NULL,
"encoding": "some_exotic_encoding",
"doc": "Official Breed Name",
}
assert expected == field.render()
def test_sequence_with_logical_type(sequence, python_primitive_type,
python_type_str, value):
"""
When the type is List, the Avro field type should be array
with the items attribute present.
"""
name = "an_array_field"
python_type = sequence[python_primitive_type]
field = fields.Field(name, python_type, dataclasses.MISSING)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": python_type_str
},
}
assert expected == field.to_dict()
field = fields.Field(name, python_type, None)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": python_type_str
},
"default": [],
}
assert expected == field.to_dict()
values = [value]
field = fields.Field(name,
python_type,
default=dataclasses.MISSING,
default_factory=lambda: values)
expected = {
"name":
name,
"type": {
"type": "array",
"name": name,
"items": python_type_str
},
"default": [
fields.LOGICAL_TYPES_FIELDS_CLASSES[python_primitive_type].
to_logical_type(value) for value in values
],
}
assert expected == field.to_dict()
def test_mapping_logical_type(mapping, python_primitive_type, python_type_str,
value):
"""
When the type is Dict, the Avro field type should be map
with the values attribute present. The keys are always string type.
"""
name = "a_map_field"
python_type = mapping[str, python_primitive_type]
field = fields.Field(name, python_type, dataclasses.MISSING)
expected = {
"name": name,
"type": {
"type": "map",
"name": name,
"values": python_type_str
},
}
assert expected == field.to_dict()
field = fields.Field(name, python_type, None)
expected = {
"name": name,
"type": {
"type": "map",
"name": name,
"values": python_type_str
},
"default": {},
}
assert expected == field.to_dict()
values = {"key": value}
field = fields.Field(name,
python_type,
default=dataclasses.MISSING,
default_factory=lambda: values)
expected = {
"name": name,
"type": {
"type": "map",
"name": name,
"values": python_type_str
},
"default": {
key: fields.LOGICAL_TYPES_FIELDS_CLASSES[python_primitive_type].
to_logical_type(value)
for key, value in values.items()
},
}
assert expected == field.to_dict()
def test_mapping_type(mapping, python_primitive_type, python_type_str):
"""
When the type is Dict, the Avro field type should be map
with the values attribute present. The keys are always string type.
"""
name = "a_map_field"
python_type = mapping[str, python_primitive_type]
field = fields.Field(name, python_type, dataclasses.MISSING)
expected = {
"name": name,
"type": {
"type": "map",
"name": name,
"values": python_type_str
},
}
assert expected == field.to_dict()
field = fields.Field(name, python_type, None)
expected = {
"name": name,
"type": {
"type": "map",
"name": name,
"values": python_type_str
},
"default": {},
}
assert expected == field.to_dict()
value = faker.pydict(2, True, python_primitive_type)
field = fields.Field(name,
python_type,
default=dataclasses.MISSING,
default_factory=lambda: value)
expected = {
"name": name,
"type": {
"type": "map",
"name": name,
"values": python_type_str
},
"default": value,
}
assert expected == field.to_dict()
def test_sequence_type(sequence, python_primitive_type, python_type_str):
"""
When the type is List, the Avro field type should be array
with the items attribute present.
"""
name = "an_array_field"
python_type = sequence[python_primitive_type]
field = fields.Field(name, python_type, dataclasses.MISSING)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": python_type_str
},
}
assert expected == field.to_dict()
field = fields.Field(name, python_type, None)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": python_type_str
},
"default": [],
}
assert expected == field.to_dict()
values = faker.pylist(2, True, python_primitive_type)
field = fields.Field(name,
python_type,
default=dataclasses.MISSING,
default_factory=lambda: values)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": python_type_str
},
"default": values,
}
assert expected == field.to_dict()
def test_invalid_type_container_field():
python_type = typing.Set
name = "test_field"
msg = f"Invalid Type for field {name}. Accepted types are list, tuple, dict or typing.Union"
with pytest.raises(ValueError, match=msg):
fields.Field(name, python_type, dataclasses.MISSING)
def test_invalid_default_values(logical_type, invalid_default, msg):
name = "a_field"
field = fields.Field(name, logical_type, invalid_default)
msg = msg or f"Invalid default type. Default should be {logical_type}"
with pytest.raises(AssertionError, match=msg):
field.to_dict()
def test_invalid_default_values(primitive_type, invalid_default):
name = "a_field"
field = fields.Field(name, primitive_type, invalid_default)
msg = f"Invalid default type. Default should be {primitive_type}"
with pytest.raises(AssertionError, match=msg):
field.to_dict()
def test_logical_type_time_with_default():
name = "a time"
python_type = datetime.time
time = consts.now.time()
field = fields.Field(name, python_type, time)
hour, minutes, seconds, microseconds = (
time.hour,
time.minute,
time.second,
time.microsecond,
)
miliseconds = int((((hour * 60 + minutes) * 60 + seconds) * 1000) +
(microseconds / 1000))
expected = {
"name": name,
"type": {
"type": fields.INT,
"logicalType": fields.TIME_MILLIS
},
"default": miliseconds,
}
assert expected == field.to_dict()
def test_logical_types(python_type, avro_type, logical_type):
name = "a logical type"
python_type = python_type
field = fields.Field(name, python_type)
expected = {"name": name, "type": {"type": avro_type, "logicalType": logical_type}}
assert expected == field.to_dict()
def test_primitive_types_with_default_value(primitive_type, default):
name = "a_field"
field = fields.Field(name, primitive_type, default)
avro_type = [fields.PYTHON_TYPE_TO_AVRO[primitive_type], fields.NULL]
assert {
"name": name,
"type": avro_type,
"default": default
} == field.to_dict()
def parse_fields(self):
return [
fields.Field(
dataclass_field.name,
dataclass_field.type,
dataclass_field.default,
dataclass_field.default_factory,
dataclass_field.metadata,
) for dataclass_field in dataclasses.fields(self.klass)
]
def parse_fields(self):
return [
fields.Field(
dataclass_field.name,
dataclass_field.type,
dataclass_field.default,
dataclass_field.default_factory,
)
for dataclass_field in dataclasses.fields(self.klass_or_instance)
]
def test_union_type(args):
primitive_types, avro_types = args[0], args[1]
name = "an_union_field"
python_type = typing.Union[primitive_types]
field = fields.Field(name, python_type)
expected = {"name": name, "type": [*avro_types]}
assert expected == field.to_dict()
def test_sequence_with_union_type(union, items, default):
name = "an_array_field"
python_type = typing.List[union]
field = fields.Field(name, python_type, default=dataclasses.MISSING)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": items
}
}
assert expected == field.to_dict()
field = fields.Field(name, python_type, default_factory=lambda: default)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": items
},
"default": default,
}
assert expected == field.to_dict()
field = fields.Field(name, python_type, default=None)
items.insert(0, fields.NULL)
expected = {
"name": name,
"type": {
"type": "array",
"name": name,
"items": items
},
"default": [],
}
assert expected == field.to_dict()
def test_render_metadata():
field = fields.Field("first_name", str, fields.NULL, metadata={"desc": "English Language Name"})
expected = [("desc", "English Language Name")]
assert expected == field.get_metadata()
field = fields.Field("engine_name", str, fields.NULL)
expected = []
assert expected == field.get_metadata()
field = fields.Field(
"breed_name", str, fields.NULL, metadata={"encoding": "some_exotic_encoding", "doc": "Official Breed Name"},
)
expected = [("encoding", "some_exotic_encoding"), ("doc", "Official Breed Name")]
assert expected == field.get_metadata()
def test_logical_types_with_null_as_default(python_type, avro_type, logical_type):
name = "a logical type"
python_type = python_type
field = fields.Field(name, python_type, None)
expected = {
"name": name,
"type": {"type": avro_type, "logicalType": logical_type},
"default": fields.NULL,
}
assert expected == field.to_dict()
def test_logical_type_uuid_with_default():
name = "a uuid"
python_type = uuid.uuid4
default = uuid.uuid4()
field = fields.Field(name, python_type, default)
expected = {
"name": name,
"type": {"type": fields.STRING, "logicalType": fields.UUID},
"default": str(default),
}
assert expected == field.to_dict()
def test_logical_type_datetime_with_default():
name = "a datetime"
python_type = datetime.datetime
field = fields.Field(name, python_type, consts.now)
ts = (consts.now - datetime.datetime(1970, 1, 1)).total_seconds()
expected = {
"name": name,
"type": {"type": fields.LONG, "logicalType": fields.TIMESTAMP_MILLIS},
"default": ts * 1000,
}
assert expected == field.to_dict()
def test_logical_type_date_with_default():
name = "a date"
python_type = datetime.date
field = fields.Field(name, python_type, consts.now.date())
date_time = datetime.datetime.combine(consts.now, datetime.datetime.min.time())
ts = (date_time - datetime.datetime(1970, 1, 1)).total_seconds()
expected = {
"name": name,
"type": {"type": fields.INT, "logicalType": fields.DATE},
"default": ts / (3600 * 24),
}
assert expected == field.to_dict()
def test_tuple_type():
"""
When the type is Tuple, the Avro field type should be enum
with the symbols attribute present.
"""
name = "an_enum_field"
default = ("BLUE", "YELOW", "RED")
python_type = typing.Tuple
field = fields.Field(name, python_type, default)
expected = {
"name": name,
"type": {
"type": "enum",
"name": name,
"symbols": list(default)
},
}
assert expected == field.to_dict()
def parse_faust_record_fields(self) -> typing.List["fields.Field"]:
schema_fields = []
for dataclass_field in dataclasses.fields(self.klass):
faust_field = dataclass_field.default
if faust_field.required:
default = dataclasses.MISSING
default_factory = dataclasses.MISSING
else:
default = faust_field.default
default_factory = dataclasses.MISSING
if isinstance(default, dataclasses.Field):
default_factory = default.default_factory
default = dataclasses.MISSING
schema_fields.append(
fields.Field(dataclass_field.name, dataclass_field.type,
default, default_factory))
return schema_fields
def test_union_type_with_records():
class User:
"******"
first_name: str
class Car:
"Car"
engine_name: str
name = "an_union_field"
python_type = typing.Union[User, Car]
field = fields.Field(name, python_type)
expected = {
"name":
name,
"type": [
{
"name": "User",
"type": "record",
"doc": "User",
"fields": [{
"name": "first_name",
"type": "string"
}],
},
{
"name": "Car",
"type": "record",
"doc": "Car",
"fields": [{
"name": "engine_name",
"type": "string"
}],
},
],
}
assert expected == field.to_dict()
def test_fixed_type():
"""
When the type is types.Fixed, the Avro field type should be fixed
with size attribute present.
"""
name = "a_fixed_field"
namespace = "md5"
aliases = ["md5", "hash"]
default = types.Fixed(16, namespace=namespace, aliases=aliases)
python_type = types.Fixed
field = fields.Field(name, python_type, default)
expected = {
"name": name,
"type": {
"type": "fixed",
"name": name,
"size": default.size,
"namespace": namespace,
"aliases": aliases,
},
}
assert expected == field.to_dict()
def test_union_type_with_record_default():
class User:
"******"
first_name: str
class Car:
"Car"
engine_name: str
name = "an_union_field"
python_type = typing.Union[User, Car]
field = fields.Field(name, python_type, None)
expected = {
"name":
name,
"type": [
fields.NULL,
{
"name": "User",
"type": "record",
"doc": "User",
"fields": [{
"name": "first_name",
"type": "string"
}],
},
{
"name": "Car",
"type": "record",
"doc": "Car",
"fields": [{
"name": "engine_name",
"type": "string"
}],
},
],
"default":
fields.NULL,
}
assert expected == field.to_dict()
field = fields.Field(
name,
python_type,
default=dataclasses.MISSING,
default_factory=lambda: {"first_name": "a name"},
)
expected = {
"name":
name,
"type": [
{
"name": "User",
"type": "record",
"doc": "User",
"fields": [{
"name": "first_name",
"type": "string"
}],
},
{
"name": "Car",
"type": "record",
"doc": "Car",
"fields": [{
"name": "engine_name",
"type": "string"
}],
},
],
"default": {
"first_name": "a name"
},
}
assert expected == field.to_dict()
def test_primitive_types(primitive_type):
name = "a_field"
field = fields.Field(name, primitive_type, dataclasses.MISSING)
avro_type = fields.PYTHON_TYPE_TO_AVRO[primitive_type]
assert {"name": name, "type": avro_type} == field.to_dict()