def fromKey(self, key, avroType):
bytes = io.BytesIO(base64.b64decode(key))
reader = DatumReader(avroType.schema)
return reader.read(BinaryDecoder(bytes))
def deserialize(self, message):
avro_reader = DatumReader(self._schema)
bytesio = BytesIO(message)
decoder = BinaryDecoder(bytesio)
return avro_reader.read(decoder)
def value_deserializer(msg):
bytes_reader = BytesIO(msg)
decoder = BinaryDecoder(bytes_reader)
reader = DatumReader(schema)
return reader.read(decoder)
def deserialize(thing):
reader = DatumReader(writers_schema=SCHEMA, readers_schema=SCHEMA)
buf = StringIO.StringIO(thing)
data = reader.read(BinaryDecoder(buf))
buf.close()
return data
from io import BytesIO
from itertools import repeat
from time import time
from avro.datafile import DataFileReader, DataFileWriter
from avro.io import DatumReader, DatumWriter, BinaryEncoder, BinaryDecoder
import sys
LOOPS = 1
with open(sys.argv[1]) as reader:
datum_reader = DatumReader()
file_reader = DataFileReader(reader, datum_reader)
SCHEMA = datum_reader.writers_schema
BUFS = []
datum_writer = DatumWriter(SCHEMA)
for record in file_reader:
buf = BytesIO()
encoder = BinaryEncoder(buf)
datum_writer.write(record, encoder)
BUFS.append(buf)
datum_reader = DatumReader(SCHEMA)
start = time()
n = 0
for _ in repeat(None, LOOPS):
for buf in BUFS:
n += 1
buf.seek(0)
record = datum_reader.read(BinaryDecoder(buf))
print 1000. * (time() - start) / n
def deserialize(myschema, mydata):
buf = io.BytesIO(mydata)
decoder = BinaryDecoder(buf)
reader = DatumReader(writer_schema=myschema)
return (reader.read(decoder))
def deserialize(self, rec_bytes):
return self.reader.read(BinaryDecoder(StringIO(rec_bytes)))
from io import BytesIO
from avro.schema import parse
from avro.io import DatumReader, BinaryDecoder
schema = parse("""{
"namespace": "org.buildroot.package.python_avro",
"type": "record",
"name": "Developer",
"fields": [
{"name": "email", "type": "string"},
{"name": "maintainer_of", "type": "string"}
]
}""")
example = b'<[email protected]\x16python_avro'
reader = DatumReader(schema)
deserialized = reader.read(BinaryDecoder(BytesIO(example)))
assert deserialized == {
'email': '*****@*****.**',
'maintainer_of': 'python_avro',
}
def deserialize(item, schema):
return DatumReader(schema).read(BinaryDecoder(io.BytesIO(item)))
def decode(msg_value):
message_bytes = io.BytesIO(msg_value)
decoder = BinaryDecoder(message_bytes)
event_dict = reader.read(decoder)
return event_dict
#!/usr/bin/env python
from kafka import KafkaConsumer
import avro.schema
from avro.io import BinaryDecoder, DatumReader
import io
import os
consumer = KafkaConsumer('test.avro',
bootstrap_servers=['localhost:9092'])
cwd = os.getcwd()
schema_path = os.path.join(cwd, 'avro/schemas/user.avsc')
schema = avro.schema.Parse(open(schema_path).read())
try:
print("Listening to the `test.avro` topic")
for msg in consumer:
bytes_reader = io.BytesIO(msg.value)
decoder = BinaryDecoder(bytes_reader)
reader = DatumReader(schema)
user = reader.read(decoder)
print(user)
finally:
consumer.close()
def get_input_value(self):
# FIXME reuse, reuse, reuse
sys.stderr.write('value: %r\n' % self._value)
f = StringIO(self._value)
dec = BinaryDecoder(f)
return self.datum_reader.read(dec)
def decode_result(self, byte_mem):
return self._result_reader.read(BinaryDecoder(BytesIO(byte_mem)))
def decode_param(self, byte_mem):
return self._param_reader.read(BinaryDecoder(BytesIO(byte_mem)))
