def test_boolean_roundtrip():
schema = {"type": "record", "fields": [{"name": "field", "type": "boolean"}]}
record = {"field": True}
new_file = MemoryIO()
fastavro.schemaless_writer(new_file, schema, record)
new_file.seek(0)
new_record = fastavro.schemaless_reader(new_file, schema)
assert record == new_record
record = {"field": False}
new_file = MemoryIO()
fastavro.schemaless_writer(new_file, schema, record)
new_file.seek(0)
new_record = fastavro.schemaless_reader(new_file, schema)
assert record == new_record
def _avro_rows(block, avro_schema):
"""Parse all rows in a stream block.
Args:
block ( \
~google.cloud.bigquery_storage_v1beta1.types.ReadRowsResponse \
):
A block containing Avro bytes to parse into rows.
avro_schema (fastavro.schema):
A parsed Avro schema, used to deserialized the bytes in the
block.
Returns:
Iterable[Mapping]:
A sequence of rows, represented as dictionaries.
"""
blockio = six.BytesIO(block.avro_rows.serialized_binary_rows)
while True:
# Loop in a while loop because schemaless_reader can only read
# a single record.
try:
# TODO: Parse DATETIME into datetime.datetime (no timezone),
# instead of as a string.
yield fastavro.schemaless_reader(blockio, avro_schema)
except StopIteration:
break # Finished with block
def deserialize(schema, binary):
bytes_writer = BytesIO()
bytes_writer.write(binary)
bytes_writer.seek(0)
res = fastavro.schemaless_reader(bytes_writer, schema)
return res
def _deserialize(avro_schema, data):
"""Deserialize an Avro datum with the specified schema string
:param dict avro_schema: The schema JSON snippet
:param str data: The Avro datum to deserialize
:rtype: dict
"""
return fastavro.schemaless_reader(io.BytesIO(data), avro_schema)
def read_schemaless(iostream, schema, num_records, runs=1):
times = []
schema = parse_schema(schema)
for _ in range(runs):
for _ in range(num_records):
iostream.seek(0)
start = time.time()
record = schemaless_reader(iostream, schema)
end = time.time()
times.append(end - start)
print('... {0} runs averaged {1} seconds'.format(runs, (sum(times) / runs)))
return records
def test_schemaless_writer_and_reader():
schema = {
"type": "record",
"name": "Test",
"namespace": "test",
"fields": [{"name": "field", "type": {"type": "string"}}],
}
record = {"field": "test"}
new_file = MemoryIO()
fastavro.schemaless_writer(new_file, schema, record)
new_file.seek(0)
new_record = fastavro.schemaless_reader(new_file, schema)
assert record == new_record
def test_default_values_in_reader():
writer_schema = {
'name': 'name1',
'type': 'record',
'namespace': 'namespace1',
'fields': [{
'doc': 'test',
'type': 'int',
'name': 'good_field'
}],
'doc': 'test'
}
reader_schema = {
'name': 'name1',
'doc': 'test',
'namespace': 'namespace1',
'fields': [{
'name': 'good_field',
'doc': 'test',
'type': 'int'
}, {
'name': 'good_compatible_field',
'doc': 'test',
'default': 1,
'type': 'int'
}],
'type': 'record'
}
record = {'good_field': 1}
new_file = MemoryIO()
fastavro.schemaless_writer(new_file, writer_schema, record)
new_file.seek(0)
new_record = fastavro.schemaless_reader(
new_file,
writer_schema,
reader_schema,
)
assert new_record == {'good_field': 1, 'good_compatible_field': 1}
def decoder_integer(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(io_stream, self.int_schema)
def decoder_points_list(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(
io_stream,
self.points_list_dict_schema,
)
def decoder_struct_map(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(io_stream, self.struct_map_schema)
def decoder_array(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(io_stream, self.array_schema)
def decoder(msg):
bytes_reader = io.BytesIO(msg)
bytes_reader.seek(0)
user = schemaless_reader(bytes_reader, schema)
return user
def decoder_boolean(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(io_stream, self.bool_schema)
def roundtrip(schema, record):
new_file = BytesIO()
fastavro.schemaless_writer(new_file, schema, record)
new_file.seek(0)
new_record = fastavro.schemaless_reader(new_file, schema)
return new_record
def test_schemaless_writer_and_reader_with_union():
"""Testing basic functionality of reader with union when option to
return_record_name is true.
"""
schema = {
"name":
"Message",
"type":
"record",
"namespace":
"test",
"fields": [{
"name": "id",
"type": "long"
}, {
"name":
"payload",
"type": [
{
"name":
"ApplicationCreated",
"type":
"record",
"fields": [{
"name": "applicationId",
"type": "string"
}, {
"name": "data",
"type": "string"
}]
},
{
"name":
"ApplicationSubmitted",
"type":
"record",
"fields": [{
"name": "applicationId",
"type": "string"
}, {
"name": "data",
"type": "string"
}]
},
]
}]
}
record = {
"id":
123,
"payload": ("test.ApplicationSubmitted", {
"applicationId": "123456789UT",
"data": "..."
})
}
new_file = BytesIO()
fastavro.schemaless_writer(new_file, schema, record)
new_file.seek(0)
new_record = fastavro.schemaless_reader(new_file,
schema,
return_record_name=True)
assert record == new_record
def deserialize(self, record):
bytes_reader = io.BytesIO(record)
dict_record = schemaless_reader(bytes_reader, self.schema)
return dict_record
def deserialize(schema, binary):
bytes_writer = BytesIO()
bytes_writer.write(binary)
bytes_writer.seek(0)
data = schemaless_reader(bytes_writer, schema)
return data
def deserialize(self, binary):
bytes_writer = BytesIO()
bytes_writer.write(binary)
bytes_writer.seek(0)
return fastavro.schemaless_reader(bytes_writer, self.schema)
def decoder_float(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(io_stream, self.float_schema)
def read_report(b: bytes) -> SiteReport:
buf = io.BytesIO(b)
d = fastavro.schemaless_reader(buf, schema)
return SiteReport(**d)
def decoder_null(self, data):
io_stream = io.BytesIO(data)
return fastavro.schemaless_reader(io_stream, self.null_schema)
# engine = create_engine(SQL_URL, echo=True)
engine = create_engine(SQL_URL)
Session = sessionmaker(bind=engine)
session = Session()
schema = json.loads(open(SCHEMA_PATH, 'r').read())
signal_session = session.query(SensorDatum).filter(
SensorDatum.sensor_id == SENSOR_ID,
SensorDatum.create_time.between(START_TIME, END_TIME)).all()
signal = []
for i in signal_session:
# signal_dict = i.to_dict()
with BytesIO(i.to_dict()['data']) as byte_io:
# signal.append(fastavro.schemaless_reader(byte_io, schema)['data']) # multiple dict object in a list
signal.extend(fastavro.schemaless_reader(byte_io, schema)['data'])
signal = numpy.array(signal) # transfer to numpy.ndarray
L = len(signal)
hx = fftpack.hilbert(signal)
signal_baoluopu = numpy.sqrt(signal**2 + hx**2)
signal_qubaoluopu = (2 * signal - signal_baoluopu) / signal_baoluopu * 0.5
# PL = abs(numpy.fft.fft(signal_baoluopu / L))[: int(L / 2)]
PL = abs(numpy.fft.fft(signal_qubaoluopu / L))[:int(L / 2)]
PL[0] = 0
f = numpy.fft.fftfreq(L, 1 / FS)[:int(L / 2)]
plt.plot(f, PL)
def fastavro(self):
schema = fastavro.schema.parse_schema(json.loads(SCHEMA))
for value in self.values:
input_buf = BytesIO(value)
fastavro.schemaless_reader(input_buf, schema)
def decode(self, data):
buffer = io.BytesIO(data)
d = fastavro.schemaless_reader(buffer, self._schema)
return self._record_cls(**d)
def decode(self, encoded):
with BytesIO(encoded) as buf:
return AvroRecord(schemaless_reader(buf, self.parsed_schema))
def fast_avro_decode(schema, encoded_message):
stringio = io.BytesIO(encoded_message)
return fastavro.schemaless_reader(stringio, schema)
