def roundtrip(schema, records): new_file = StringIO() json_writer(new_file, schema, records) new_file.seek(0) new_records = list(json_reader(new_file, schema)) return new_records
def writerows(self, rows): """ Write rows in bulk :param rows: multiple rows :type rows: list(dict) """ json_writer(self.file_obj, self.parsed_schema, rows)
def encode(self, records: List, version: str = None) -> EncodedMessage: schema_response = self._schema_response(version) string_writer = StringIO() fastavro.json_writer(string_writer, self._schema(schema_response), records) return EncodedMessage(body=string_writer.getvalue().encode(), version=str( self._schema_version(schema_response)))
def writerow(self, row): """ Write sincle row :param rows: row :type rows: dict """ json_writer(self.file_obj, self.parsed_schema, [row])
def prepare_message(self, message, subject): write_schema_response = self._schema_registry.get_latest_schema( subject) write_schema_obj = json.loads( write_schema_response[RESPONSE_KEY_SCHEMA]) write_schema = parse_schema(write_schema_obj) string_writer = StringIO() json_writer(string_writer, write_schema, message) properties = BasicProperties( headers={ MESSAGE_PROPERTY_SUBJECT: subject, MESSAGE_PROPERTY_VERSION: write_schema_response[RESPONSE_KEY_VERSION], }) return { MESSAGE_KEY_PROPERTIES: properties, MESSAGE_KEY_BODY: string_writer.getvalue() }
def serialize(payload: typing.Dict, schema: typing.Dict, serialization_type: str = "avro") -> bytes: if serialization_type == "avro": file_like_output: typing.Union[io.BytesIO, io.StringIO] = io.BytesIO() fastavro.schemaless_writer(file_like_output, schema, payload) value = file_like_output.getvalue() elif serialization_type == "avro-json": file_like_output = io.StringIO() fastavro.json_writer(file_like_output, schema, [payload]) value = file_like_output.getvalue().encode("utf-8") else: raise ValueError( f"Serialization type should be `avro` or `avro-json`, not {serialization_type}" ) file_like_output.flush() return value # type: ignore
def to_json(self): """ :return: JSON serialized data :rtype: str """ if not self._ok or self._json_data: return self._json_data if self._schema is None: try: self._json_data = json.dumps(self._object_data, default=AvroTools._json_converter) except Exception as e: self._last_error = f'JSON serialization error: {e}' else: out = io.StringIO() json_writer( out, self._schema, self._object_data if isinstance( self._object_data, list) else [self._object_data]) out.flush() out.seek(0) self._json_data = out.read() return self._json_data
def test_json_writer_with_validation(): """https://github.com/fastavro/fastavro/issues/580""" schema = { "doc": "A weather reading.", "name": "Weather", "namespace": "test", "type": "record", "fields": [ {"name": "station", "type": "string"}, {"name": "time", "type": "long"}, {"name": "temp", "type": "int"}, ], } records = [ {"station": "011990-99999", "temp": 0, "time": 1433269388}, {"station": "011990-99999", "temp": 22, "time": "last day"}, {"station": "011990-99999", "temp": -11, "time": 1433273379}, {"station": "012650-99999", "temp": 111.9, "time": 1433275478}, ] new_file = StringIO() with pytest.raises(ValidationError): json_writer(new_file, schema, records, validator=True)
def test_encoded_union_output(): schema = { "type": "record", "name": "Test", "namespace": "test", "fields": [{ "name": "union", "type": [ "null", "int", { "type": "record", "name": "union_record", "fields": [{ "name": "union_record_field", "type": "string", }], }, ], }], } # A null value is encoded as just null records = [{"union": None}] new_file = StringIO() json_writer(new_file, schema, records) assert new_file.getvalue().strip() == json.dumps({"union": None}) # A non-null, non-named type is encoded as an object with a key for the # type records = [{"union": 321}] new_file = StringIO() json_writer(new_file, schema, records) assert new_file.getvalue().strip() == json.dumps({"union": {"int": 321}}) # A non-null, named type is encoded as an object with a key for the name records = [{"union": {"union_record_field": "union_field"}}] new_file = StringIO() json_writer(new_file, schema, records) expected = json.dumps({ "union": { "test.union_record": { "union_record_field": "union_field" } } }) assert new_file.getvalue().strip() == expected
def test_union_output_without_type(): """https://github.com/fastavro/fastavro/issues/420""" schema = { "type": "record", "name": "Test", "namespace": "test", "fields": [ { "name": "union", "type": [ "null", "int", { "type": "record", "name": "union_record", "fields": [{"name": "union_record_field", "type": "string"}], }, ], } ], } # A null value is encoded as just null records = [{"union": None}] new_file = StringIO() json_writer(new_file, schema, records, write_union_type=False) assert new_file.getvalue().strip() == json.dumps({"union": None}) # A non-null, non-named type is encoded as just the value records = [{"union": 321}] new_file = StringIO() json_writer(new_file, schema, records, write_union_type=False) assert new_file.getvalue().strip() == json.dumps({"union": 321}) # A non-null, named type is encoded as an object records = [{"union": {"union_record_field": "union_field"}}] new_file = StringIO() json_writer(new_file, schema, records, write_union_type=False) expected = json.dumps({"union": {"union_record_field": "union_field"}}) assert new_file.getvalue().strip() == expected