def json2avro2json(data):
"""
Function which reads json file, generates avro screma, convert json to
avro and convert it back using the schema. All operations are done in
memory (via io.BytesIO)
"""
# parse schema into schema object
arec = gen_schema(data)
schema = avro.schema.parse(json.dumps(arec))
# setup avro writer with given schema
writer = avro.io.DatumWriter(schema)
bytes_writer = io.BytesIO()
encoder = avro.io.BinaryEncoder(bytes_writer)
writer.write(data, encoder)
# our data now is in a bytes_writer, let's get it
avro_data = bytes_writer.getvalue()
# read back data into JSON record
bytes_reader = io.BytesIO(avro_data)
decoder = avro.io.BinaryDecoder(bytes_reader)
reader = avro.io.DatumReader(schema)
json_data = reader.read(decoder)
return json_data
def setUp(self):
self.tdir = tempfile.mkdtemp()
data = {"int":1, "float":1.2, "list":[1,2,3],
"dict":{"dname": "foo", "dval":1},
"listdict":[{"lname":"foo"}], "str":"string"}
self.bare_data = dict(data)
data['wmaid'] = wmaHash(data)
data['stype'] = 'avroio'
self.data = data
schema = gen_schema(self.data)
sname = os.path.join(self.tdir, 'schema.avsc')
with open(sname, 'w') as ostream:
ostream.write(json.dumps(schema))
self.mgr = AvroStorage('avroio:%s' % sname)
def setUp(self):
self.tdir = tempfile.mkdtemp()
data = {
"int": 1,
"float": 1.2,
"list": [1, 2, 3],
"dict": {
"dname": "foo",
"dval": 1
},
"listdict": [{
"lname": "foo"
}],
"str": "string"
}
self.bare_data = dict(data)
data['wmaid'] = wmaHash(data)
data['stype'] = 'avroio'
self.data = data
schema = gen_schema(self.data)
sname = os.path.join(self.tdir, 'schema.avsc')
with open(sname, 'w') as ostream:
ostream.write(json.dumps(schema))
self.mgr = AvroStorage('avroio:%s' % sname)