def roundtrip(schema, records):
new_file = StringIO()
json_writer(new_file, schema, records)
new_file.seek(0)
new_records = list(json_reader(new_file, schema))
return new_records
def writerows(self, rows):
"""
Write rows in bulk
:param rows: multiple rows
:type rows: list(dict)
"""
json_writer(self.file_obj, self.parsed_schema, rows)
def encode(self, records: List, version: str = None) -> EncodedMessage:
schema_response = self._schema_response(version)
string_writer = StringIO()
fastavro.json_writer(string_writer, self._schema(schema_response),
records)
return EncodedMessage(body=string_writer.getvalue().encode(),
version=str(
self._schema_version(schema_response)))
def writerow(self, row):
"""
Write sincle row
:param rows: row
:type rows: dict
"""
json_writer(self.file_obj, self.parsed_schema, [row])
def prepare_message(self, message, subject):
write_schema_response = self._schema_registry.get_latest_schema(
subject)
write_schema_obj = json.loads(
write_schema_response[RESPONSE_KEY_SCHEMA])
write_schema = parse_schema(write_schema_obj)
string_writer = StringIO()
json_writer(string_writer, write_schema, message)
properties = BasicProperties(
headers={
MESSAGE_PROPERTY_SUBJECT:
subject,
MESSAGE_PROPERTY_VERSION:
write_schema_response[RESPONSE_KEY_VERSION],
})
return {
MESSAGE_KEY_PROPERTIES: properties,
MESSAGE_KEY_BODY: string_writer.getvalue()
}
def serialize(payload: typing.Dict,
schema: typing.Dict,
serialization_type: str = "avro") -> bytes:
if serialization_type == "avro":
file_like_output: typing.Union[io.BytesIO, io.StringIO] = io.BytesIO()
fastavro.schemaless_writer(file_like_output, schema, payload)
value = file_like_output.getvalue()
elif serialization_type == "avro-json":
file_like_output = io.StringIO()
fastavro.json_writer(file_like_output, schema, [payload])
value = file_like_output.getvalue().encode("utf-8")
else:
raise ValueError(
f"Serialization type should be `avro` or `avro-json`, not {serialization_type}"
)
file_like_output.flush()
return value # type: ignore
def to_json(self):
"""
:return: JSON serialized data
:rtype: str
"""
if not self._ok or self._json_data:
return self._json_data
if self._schema is None:
try:
self._json_data = json.dumps(self._object_data,
default=AvroTools._json_converter)
except Exception as e:
self._last_error = f'JSON serialization error: {e}'
else:
out = io.StringIO()
json_writer(
out, self._schema, self._object_data if isinstance(
self._object_data, list) else [self._object_data])
out.flush()
out.seek(0)
self._json_data = out.read()
return self._json_data
def test_json_writer_with_validation():
"""https://github.com/fastavro/fastavro/issues/580"""
schema = {
"doc": "A weather reading.",
"name": "Weather",
"namespace": "test",
"type": "record",
"fields": [
{"name": "station", "type": "string"},
{"name": "time", "type": "long"},
{"name": "temp", "type": "int"},
],
}
records = [
{"station": "011990-99999", "temp": 0, "time": 1433269388},
{"station": "011990-99999", "temp": 22, "time": "last day"},
{"station": "011990-99999", "temp": -11, "time": 1433273379},
{"station": "012650-99999", "temp": 111.9, "time": 1433275478},
]
new_file = StringIO()
with pytest.raises(ValidationError):
json_writer(new_file, schema, records, validator=True)
def test_encoded_union_output():
schema = {
"type":
"record",
"name":
"Test",
"namespace":
"test",
"fields": [{
"name":
"union",
"type": [
"null",
"int",
{
"type": "record",
"name": "union_record",
"fields": [{
"name": "union_record_field",
"type": "string",
}],
},
],
}],
}
# A null value is encoded as just null
records = [{"union": None}]
new_file = StringIO()
json_writer(new_file, schema, records)
assert new_file.getvalue().strip() == json.dumps({"union": None})
# A non-null, non-named type is encoded as an object with a key for the
# type
records = [{"union": 321}]
new_file = StringIO()
json_writer(new_file, schema, records)
assert new_file.getvalue().strip() == json.dumps({"union": {"int": 321}})
# A non-null, named type is encoded as an object with a key for the name
records = [{"union": {"union_record_field": "union_field"}}]
new_file = StringIO()
json_writer(new_file, schema, records)
expected = json.dumps({
"union": {
"test.union_record": {
"union_record_field": "union_field"
}
}
})
assert new_file.getvalue().strip() == expected
def test_union_output_without_type():
"""https://github.com/fastavro/fastavro/issues/420"""
schema = {
"type": "record",
"name": "Test",
"namespace": "test",
"fields": [
{
"name": "union",
"type": [
"null",
"int",
{
"type": "record",
"name": "union_record",
"fields": [{"name": "union_record_field", "type": "string"}],
},
],
}
],
}
# A null value is encoded as just null
records = [{"union": None}]
new_file = StringIO()
json_writer(new_file, schema, records, write_union_type=False)
assert new_file.getvalue().strip() == json.dumps({"union": None})
# A non-null, non-named type is encoded as just the value
records = [{"union": 321}]
new_file = StringIO()
json_writer(new_file, schema, records, write_union_type=False)
assert new_file.getvalue().strip() == json.dumps({"union": 321})
# A non-null, named type is encoded as an object
records = [{"union": {"union_record_field": "union_field"}}]
new_file = StringIO()
json_writer(new_file, schema, records, write_union_type=False)
expected = json.dumps({"union": {"union_record_field": "union_field"}})
assert new_file.getvalue().strip() == expected
