def test_file(filename):
with open(filename, 'rb') as fo:
reader = fastavro.reader(fo)
assert hasattr(reader, 'schema'), 'no schema on file'
if basename(filename) in NO_DATA:
return
records = list(reader)
assert len(records) > 0, 'no records found'
new_file = MemoryIO()
fastavro.writer(new_file, reader.schema, records, reader.codec)
new_file_bytes = new_file.getvalue()
new_file = NoSeekMemoryIO(new_file_bytes)
new_reader = fastavro.reader(new_file)
assert hasattr(new_reader, 'schema'), "schema wasn't written"
assert new_reader.schema == reader.schema
assert new_reader.codec == reader.codec
new_records = list(new_reader)
assert new_records == records
# Test schema migration with the same schema
new_file = NoSeekMemoryIO(new_file_bytes)
schema_migration_reader = fastavro.reader(new_file, reader.schema)
assert schema_migration_reader.reader_schema == reader.schema
new_records = list(schema_migration_reader)
assert new_records == records
def check(filename):
with open(filename, 'rb') as fo:
reader = fastavro.reader(fo)
assert hasattr(reader, 'schema'), 'no schema on file'
if basename(filename) in NO_DATA:
return
records = list(reader)
assert len(records) > 0, 'no records found'
new_file = MemoryIO()
fastavro.writer(new_file, reader.schema, records, reader.codec)
new_file_bytes = new_file.getvalue()
new_file = NoSeekMemoryIO(new_file_bytes)
new_reader = fastavro.reader(new_file)
assert hasattr(new_reader, 'schema'), "schema wasn't written"
assert new_reader.schema == reader.schema
assert new_reader.codec == reader.codec
new_records = list(new_reader)
assert new_records == records
# Test schema migration with the same schema
new_file = NoSeekMemoryIO(new_file_bytes)
schema_migration_reader = fastavro.reader(new_file, reader.schema)
assert schema_migration_reader.reader_schema == reader.schema
new_records = list(schema_migration_reader)
assert new_records == records
def test_user_specified_sync():
"""https://github.com/fastavro/fastavro/issues/300"""
schema = {
"type": "record",
"name": "test_user_specified_sync",
"fields": []
}
file1 = MemoryIO()
file2 = MemoryIO()
records = [{}]
fastavro.writer(file1, schema, records, sync_marker=b'sync')
fastavro.writer(file2, schema, records, sync_marker=b'sync')
assert file1.getvalue() == file2.getvalue()
def test_unsupported_codec():
schema = {
"doc":
"A weather reading.",
"name":
"Weather",
"namespace":
"test",
"type":
"record",
"fields": [
{
"name": "station",
"type": "string"
},
{
"name": "time",
"type": "long"
},
{
"name": "temp",
"type": "int"
},
],
}
records = [
{
"station": "011990-99999",
"temp": 0,
"time": 1433269388
},
{
"station": "011990-99999",
"temp": 22,
"time": 1433270389
},
{
"station": "011990-99999",
"temp": -11,
"time": 1433273379
},
{
"station": "012650-99999",
"temp": 111,
"time": 1433275478
},
]
file = MemoryIO()
with pytest.raises(ValueError, match="unrecognized codec"):
fastavro.writer(file, schema, records, codec="unsupported")
file = MemoryIO()
fastavro.writer(file, schema, records, codec="deflate")
# Change the avro binary to act as if it were written with a codec called
# `unsupported`
modified_avro = file.getvalue().replace(b"\x0edeflate", b"\x16unsupported")
modified_file = MemoryIO(modified_avro)
with pytest.raises(ValueError, match="Unrecognized codec"):
list(fastavro.reader(modified_file))