def _deserialize_array(self, schema, annotated_datum):
"""
Array is deserialized into array.
Every element is deserialized recursively according to `items` schema.
:param schema: Avro schema of `annotated_datum`
:param datum: Data with Avro json annotation to deserialize
:return: deserialized array (list)
"""
if not isinstance(annotated_datum, list):
raise AvroTypeException(schema, annotated_datum)
if annotated_datum is None:
raise AvroTypeException(schema, annotated_datum)
deserialize = functools.partial(self._deserialize_data, schema.items)
return list(map(deserialize, annotated_datum))
def _serialize_union(self, schema, datum):
"""
With union schema has multiple possible schemas.
We iterate over possible schemas and see which one fits `datum` passed.
Union serialization:
if null:
"null"
else:
{"<type>": value}
Then used one that matches to serialize `datum`
:param schema: Avro schema for this union
:param datum: Data to serialize
:return: dict {"type": value} or "null"
"""
for candidate_schema in schema.schemas:
if validate(candidate_schema, datum):
if candidate_schema.type == "null":
return self._process_null()
else:
field_type_name = self._union_name(candidate_schema)
return {
field_type_name:
self._process_data(candidate_schema, datum)
}
raise AvroTypeException(schema, datum)
def _deserialize_union(self, schema, annotated_datum):
"""
With union schema has multiple possible schemas.
We iterate over possible schemas and see which one matches the key that
was in the json.
Union serialization:
if null:
None
else:
value
Then used one that matches to deserialize `annotated_datum`
:param schema: Avro schema for this union
:param annotated_datum: Data with Avro json annotation to deserialize
:return: dict {"type": value} or "null"
"""
if not annotated_datum:
return self._deserialize_null()
if not isinstance(annotated_datum, dict):
raise AvroTypeException(schema, annotated_datum)
key = list(annotated_datum.keys())[0]
for candidate_schema in schema.schemas:
if isinstance(candidate_schema, avro.schema.NamedSchema):
if candidate_schema.name == key:
return self._deserialize_data(candidate_schema,
annotated_datum[key])
else:
if candidate_schema.type == key:
return self._deserialize_data(candidate_schema,
annotated_datum[key])
raise schema.AvroTypeException(schema, datum)
def __init__(self, value):
if isinstance(value, self.__class__):
super().__setattr__("_value", value._value)
else:
if not self.validate(value):
raise AvroTypeException(self.__schema__, value)
super().__setattr__("_value", value)
def from_json(self, json_str):
args = ()
kwargs = {"schema": self._avro_schema}
annotated_datum = json.loads(json_str,
cls=self._json_decoder,
*args,
**kwargs)
datum = self._deserialize_data(self._avro_schema, annotated_datum)
if not validate(self._avro_schema, datum):
raise AvroTypeException(schema, datum)
return datum
def _serialize_array(self, schema, datum):
"""
Array is serialized into array.
Every element is serialized recursively according to `items` schema.
:param schema: Avro schema of `datum`
:param datum: Data to serialize
:return: serialized array (list)
"""
if datum is None:
raise AvroTypeException(schema, datum)
serialize = functools.partial(self._serialize_data, schema.items)
return list(map(serialize, datum))
def _serialize_map(self, schema, datum):
"""
Map is serialized into a map.
Every value is serialized recursively according to `values` schema.
:param schema: Avro schema of `datum`
:param datum: Data to serialize.
:return: map with serialized values
"""
if datum is None:
raise AvroTypeException(schema, datum)
serialize = functools.partial(self._serialize_data, schema.values)
return dict((key, serialize(value)) for key, value in datum.iteritems())
def write_union(self, writer_schema, datum, encoder):
# resolve union
index_of_schema = -1
for i, candidate_schema in enumerate(writer_schema.schemas):
if Validate(candidate_schema, datum):
index_of_schema = i
if index_of_schema < 0:
raise AvroTypeException(writer_schema, datum)
# write data
encoder.write_int(index_of_schema)
self.write_data(writer_schema.schemas[index_of_schema], datum, encoder)
def _deserialize_map(self, schema, annotated_datum):
"""
Map is serialized into a map.
Every value is deserialized recursively according to `values` schema.
:param schema: Avro schema of `annotated_datum`
:param datum: Data with Avro json annotation to deserialize.
:return: map with deserialized values
"""
if not isinstance(annotated_datum, dict):
raise AvroTypeException(schema, annotated_datum)
deserialize = functools.partial(self._deserialize_data, schema.values)
return dict((key, deserialize(value))
for key, value in six.iteritems(annotated_datum))
def validate_entity_payload(project_Schema, payload):
# Use avro.io to validate payload against the linke schema
schema_str = ''
if type(project_Schema.schema.definition) == list:
project_Schema.schema.definition[0][
'name'] = project_Schema.schema.name
schema_str = json.dumps(project_Schema.schema.definition[0])
else:
project_Schema.schema.definition['name'] = project_Schema.schema.name
schema_str = json.dumps(project_Schema.schema.definition)
avro_schema = schema.Parse(schema_str)
if not Validate(avro_schema, payload):
raise AvroTypeException(avro_schema, payload)
def _serialize_data(self, schema, datum):
"""
Non-specific serialize function.
It checks type in the schema and calls correct serialization.
:param schema: Avro schema of the `datum`
:param datum: Data to serialize
"""
if not validate(schema, datum):
raise AvroTypeException(schema, datum)
if schema.type in AvroJsonSerializer.PRIMITIVE_CONVERTERS:
return datum
if schema.type in AvroJsonSerializer.COMPLEX_CONVERTERS:
return self.COMPLEX_CONVERTERS[schema.type](self, schema, datum)
raise avro.schema.AvroException("Unknown type: %s" % schema.type)
def _deserialize_record(self, schema, annotated_datum):
"""
Records are deserialized into dicts.
Every field value is deserialized based on it's schema.
If a value is None, remove the key from the dict.
:param schema: Avro schema of this record
:param annotated_datum: Data with Avro json annotation to deserialize
"""
if not isinstance(annotated_datum, dict):
raise AvroTypeException(schema, annotated_datum)
result = {}
for field in schema.fields:
val = self._deserialize_data(field.type,
annotated_datum.get(field.name))
if val:
result[field.name] = val
return result
def __init__(self, value):
dict_value = _recursive_to_dict(value)
if not self.validate(dict_value):
raise AvroTypeException(self.__schema__, value)
super().__setattr__("_value", dict_value)