def test_boolean_roundtrip(): schema = {"type": "record", "fields": [{"name": "field", "type": "boolean"}]} record = {"field": True} new_file = MemoryIO() fastavro.schemaless_writer(new_file, schema, record) new_file.seek(0) new_record = fastavro.schemaless_reader(new_file, schema) assert record == new_record record = {"field": False} new_file = MemoryIO() fastavro.schemaless_writer(new_file, schema, record) new_file.seek(0) new_record = fastavro.schemaless_reader(new_file, schema) assert record == new_record
def _avro_rows(block, avro_schema): """Parse all rows in a stream block. Args: block ( \ ~google.cloud.bigquery_storage_v1beta1.types.ReadRowsResponse \ ): A block containing Avro bytes to parse into rows. avro_schema (fastavro.schema): A parsed Avro schema, used to deserialized the bytes in the block. Returns: Iterable[Mapping]: A sequence of rows, represented as dictionaries. """ blockio = six.BytesIO(block.avro_rows.serialized_binary_rows) while True: # Loop in a while loop because schemaless_reader can only read # a single record. try: # TODO: Parse DATETIME into datetime.datetime (no timezone), # instead of as a string. yield fastavro.schemaless_reader(blockio, avro_schema) except StopIteration: break # Finished with block
def deserialize(schema, binary): bytes_writer = BytesIO() bytes_writer.write(binary) bytes_writer.seek(0) res = fastavro.schemaless_reader(bytes_writer, schema) return res
def _deserialize(avro_schema, data): """Deserialize an Avro datum with the specified schema string :param dict avro_schema: The schema JSON snippet :param str data: The Avro datum to deserialize :rtype: dict """ return fastavro.schemaless_reader(io.BytesIO(data), avro_schema)
def read_schemaless(iostream, schema, num_records, runs=1): times = [] schema = parse_schema(schema) for _ in range(runs): for _ in range(num_records): iostream.seek(0) start = time.time() record = schemaless_reader(iostream, schema) end = time.time() times.append(end - start) print('... {0} runs averaged {1} seconds'.format(runs, (sum(times) / runs))) return records
def test_schemaless_writer_and_reader(): schema = { "type": "record", "name": "Test", "namespace": "test", "fields": [{"name": "field", "type": {"type": "string"}}], } record = {"field": "test"} new_file = MemoryIO() fastavro.schemaless_writer(new_file, schema, record) new_file.seek(0) new_record = fastavro.schemaless_reader(new_file, schema) assert record == new_record
def test_default_values_in_reader(): writer_schema = { 'name': 'name1', 'type': 'record', 'namespace': 'namespace1', 'fields': [{ 'doc': 'test', 'type': 'int', 'name': 'good_field' }], 'doc': 'test' } reader_schema = { 'name': 'name1', 'doc': 'test', 'namespace': 'namespace1', 'fields': [{ 'name': 'good_field', 'doc': 'test', 'type': 'int' }, { 'name': 'good_compatible_field', 'doc': 'test', 'default': 1, 'type': 'int' }], 'type': 'record' } record = {'good_field': 1} new_file = MemoryIO() fastavro.schemaless_writer(new_file, writer_schema, record) new_file.seek(0) new_record = fastavro.schemaless_reader( new_file, writer_schema, reader_schema, ) assert new_record == {'good_field': 1, 'good_compatible_field': 1}
def decoder_integer(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader(io_stream, self.int_schema)
def decoder_points_list(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader( io_stream, self.points_list_dict_schema, )
def decoder_struct_map(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader(io_stream, self.struct_map_schema)
def decoder_array(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader(io_stream, self.array_schema)
def decoder(msg): bytes_reader = io.BytesIO(msg) bytes_reader.seek(0) user = schemaless_reader(bytes_reader, schema) return user
def decoder_boolean(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader(io_stream, self.bool_schema)
def roundtrip(schema, record): new_file = BytesIO() fastavro.schemaless_writer(new_file, schema, record) new_file.seek(0) new_record = fastavro.schemaless_reader(new_file, schema) return new_record
def test_schemaless_writer_and_reader_with_union(): """Testing basic functionality of reader with union when option to return_record_name is true. """ schema = { "name": "Message", "type": "record", "namespace": "test", "fields": [{ "name": "id", "type": "long" }, { "name": "payload", "type": [ { "name": "ApplicationCreated", "type": "record", "fields": [{ "name": "applicationId", "type": "string" }, { "name": "data", "type": "string" }] }, { "name": "ApplicationSubmitted", "type": "record", "fields": [{ "name": "applicationId", "type": "string" }, { "name": "data", "type": "string" }] }, ] }] } record = { "id": 123, "payload": ("test.ApplicationSubmitted", { "applicationId": "123456789UT", "data": "..." }) } new_file = BytesIO() fastavro.schemaless_writer(new_file, schema, record) new_file.seek(0) new_record = fastavro.schemaless_reader(new_file, schema, return_record_name=True) assert record == new_record
def deserialize(self, record): bytes_reader = io.BytesIO(record) dict_record = schemaless_reader(bytes_reader, self.schema) return dict_record
def deserialize(schema, binary): bytes_writer = BytesIO() bytes_writer.write(binary) bytes_writer.seek(0) data = schemaless_reader(bytes_writer, schema) return data
def deserialize(self, binary): bytes_writer = BytesIO() bytes_writer.write(binary) bytes_writer.seek(0) return fastavro.schemaless_reader(bytes_writer, self.schema)
def decoder_float(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader(io_stream, self.float_schema)
def read_report(b: bytes) -> SiteReport: buf = io.BytesIO(b) d = fastavro.schemaless_reader(buf, schema) return SiteReport(**d)
def decoder_null(self, data): io_stream = io.BytesIO(data) return fastavro.schemaless_reader(io_stream, self.null_schema)
# engine = create_engine(SQL_URL, echo=True) engine = create_engine(SQL_URL) Session = sessionmaker(bind=engine) session = Session() schema = json.loads(open(SCHEMA_PATH, 'r').read()) signal_session = session.query(SensorDatum).filter( SensorDatum.sensor_id == SENSOR_ID, SensorDatum.create_time.between(START_TIME, END_TIME)).all() signal = [] for i in signal_session: # signal_dict = i.to_dict() with BytesIO(i.to_dict()['data']) as byte_io: # signal.append(fastavro.schemaless_reader(byte_io, schema)['data']) # multiple dict object in a list signal.extend(fastavro.schemaless_reader(byte_io, schema)['data']) signal = numpy.array(signal) # transfer to numpy.ndarray L = len(signal) hx = fftpack.hilbert(signal) signal_baoluopu = numpy.sqrt(signal**2 + hx**2) signal_qubaoluopu = (2 * signal - signal_baoluopu) / signal_baoluopu * 0.5 # PL = abs(numpy.fft.fft(signal_baoluopu / L))[: int(L / 2)] PL = abs(numpy.fft.fft(signal_qubaoluopu / L))[:int(L / 2)] PL[0] = 0 f = numpy.fft.fftfreq(L, 1 / FS)[:int(L / 2)] plt.plot(f, PL)
def fastavro(self): schema = fastavro.schema.parse_schema(json.loads(SCHEMA)) for value in self.values: input_buf = BytesIO(value) fastavro.schemaless_reader(input_buf, schema)
def decode(self, data): buffer = io.BytesIO(data) d = fastavro.schemaless_reader(buffer, self._schema) return self._record_cls(**d)
def decode(self, encoded): with BytesIO(encoded) as buf: return AvroRecord(schemaless_reader(buf, self.parsed_schema))
def fast_avro_decode(schema, encoded_message): stringio = io.BytesIO(encoded_message) return fastavro.schemaless_reader(stringio, schema)