def fromKey(self, key, avroType): bytes = io.BytesIO(base64.b64decode(key)) reader = DatumReader(avroType.schema) return reader.read(BinaryDecoder(bytes))
def deserialize(self, message): avro_reader = DatumReader(self._schema) bytesio = BytesIO(message) decoder = BinaryDecoder(bytesio) return avro_reader.read(decoder)
def value_deserializer(msg): bytes_reader = BytesIO(msg) decoder = BinaryDecoder(bytes_reader) reader = DatumReader(schema) return reader.read(decoder)
def deserialize(thing): reader = DatumReader(writers_schema=SCHEMA, readers_schema=SCHEMA) buf = StringIO.StringIO(thing) data = reader.read(BinaryDecoder(buf)) buf.close() return data
from io import BytesIO from itertools import repeat from time import time from avro.datafile import DataFileReader, DataFileWriter from avro.io import DatumReader, DatumWriter, BinaryEncoder, BinaryDecoder import sys LOOPS = 1 with open(sys.argv[1]) as reader: datum_reader = DatumReader() file_reader = DataFileReader(reader, datum_reader) SCHEMA = datum_reader.writers_schema BUFS = [] datum_writer = DatumWriter(SCHEMA) for record in file_reader: buf = BytesIO() encoder = BinaryEncoder(buf) datum_writer.write(record, encoder) BUFS.append(buf) datum_reader = DatumReader(SCHEMA) start = time() n = 0 for _ in repeat(None, LOOPS): for buf in BUFS: n += 1 buf.seek(0) record = datum_reader.read(BinaryDecoder(buf)) print 1000. * (time() - start) / n
def deserialize(myschema, mydata): buf = io.BytesIO(mydata) decoder = BinaryDecoder(buf) reader = DatumReader(writer_schema=myschema) return (reader.read(decoder))
def deserialize(self, rec_bytes): return self.reader.read(BinaryDecoder(StringIO(rec_bytes)))
from io import BytesIO from avro.schema import parse from avro.io import DatumReader, BinaryDecoder schema = parse("""{ "namespace": "org.buildroot.package.python_avro", "type": "record", "name": "Developer", "fields": [ {"name": "email", "type": "string"}, {"name": "maintainer_of", "type": "string"} ] }""") example = b'<[email protected]\x16python_avro' reader = DatumReader(schema) deserialized = reader.read(BinaryDecoder(BytesIO(example))) assert deserialized == { 'email': '*****@*****.**', 'maintainer_of': 'python_avro', }
def deserialize(item, schema): return DatumReader(schema).read(BinaryDecoder(io.BytesIO(item)))
def decode(msg_value): message_bytes = io.BytesIO(msg_value) decoder = BinaryDecoder(message_bytes) event_dict = reader.read(decoder) return event_dict
#!/usr/bin/env python from kafka import KafkaConsumer import avro.schema from avro.io import BinaryDecoder, DatumReader import io import os consumer = KafkaConsumer('test.avro', bootstrap_servers=['localhost:9092']) cwd = os.getcwd() schema_path = os.path.join(cwd, 'avro/schemas/user.avsc') schema = avro.schema.Parse(open(schema_path).read()) try: print("Listening to the `test.avro` topic") for msg in consumer: bytes_reader = io.BytesIO(msg.value) decoder = BinaryDecoder(bytes_reader) reader = DatumReader(schema) user = reader.read(decoder) print(user) finally: consumer.close()
def get_input_value(self): # FIXME reuse, reuse, reuse sys.stderr.write('value: %r\n' % self._value) f = StringIO(self._value) dec = BinaryDecoder(f) return self.datum_reader.read(dec)
def decode_result(self, byte_mem): return self._result_reader.read(BinaryDecoder(BytesIO(byte_mem)))
def decode_param(self, byte_mem): return self._param_reader.read(BinaryDecoder(BytesIO(byte_mem)))