def _create_array_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
# It seems there's bug when array item record contains only one field, which throws
# java.lang.ClassCastException: required ... is not a group when reading
record_schema = """
{
"type": "record",
"name": "test",
"fields": [
{
"name": "array",
"type": {
"type": "array",
"items": {
"type": "record",
"name": "item",
"fields": [
{ "name": "int", "type": "int" },
{ "name": "double", "type": "double" }
]
}
}
}
]
}
"""
schema = AvroSchema.parse_string(record_schema)
records = [
_create_array_avro_record(schema, [(1, 2.), (3, 4.)]),
_create_array_avro_record(schema, [(5, 6.), (7, 8.)])
]
return schema, records
def _create_map_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
record_schema = """
{
"type": "record",
"name": "test",
"fields": [
{
"name": "map",
"type": {
"type": "map",
"values": "long"
}
}
]
}
"""
schema = AvroSchema.parse_string(record_schema)
records = [
_create_map_avro_record(schema, {
'a': 1,
'b': 2
}),
_create_map_avro_record(schema, {
'c': 3,
'd': 4
})
]
return schema, records
def _create_enum_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
schema = AvroSchema.parse_string(ENUM_SCHEMA)
records = [
_create_enum_avro_record(schema, 'SPADES'),
_create_enum_avro_record(schema, 'DIAMONDS')
]
return schema, records
def _create_array_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
schema = AvroSchema.parse_string(ARRAY_SCHEMA)
records = [
_create_array_avro_record(schema, [(1, 2.), (3, 4.)]),
_create_array_avro_record(schema, [(5, 6.), (7, 8.)])
]
return schema, records
def _create_basic_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
schema = AvroSchema.parse_string(BASIC_SCHEMA)
records = [
_create_basic_avro_record(schema, True, 0, 1, 2, 3, 's1'),
_create_basic_avro_record(schema, False, 4, 5, 6, 7, 's2')
]
return schema, records
def _create_union_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
schema = AvroSchema.parse_string(UNION_SCHEMA)
records = [
_create_union_avro_record(schema, 1),
_create_union_avro_record(schema, 2.),
_create_union_avro_record(schema, None)
]
return schema, records
def _create_enum_avro_schema_and_py_objects() -> Tuple[AvroSchema, List[dict]]:
schema = AvroSchema.parse_string(ENUM_SCHEMA)
records = [
{
'suit': 'SPADES'
},
{
'suit': 'DIAMONDS'
},
]
return schema, records
def _create_array_avro_record(schema, item_values: list):
jvm = get_gateway().jvm
j_record = jvm.GenericData.Record(schema._j_schema)
item_schema = AvroSchema(
schema._j_schema.getField('array').schema().getElementType())
j_array = jvm.java.util.ArrayList()
for idx, item_value in enumerate(item_values):
j_item = jvm.GenericData.Record(item_schema._j_schema)
j_item.put('int', item_value[0])
j_item.put('double', item_value[1])
j_array.add(j_item)
j_record.put('array', j_array)
return j_record
def _create_map_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
schema = AvroSchema.parse_string(MAP_SCHEMA)
records = [
_create_map_avro_record(schema, {
'a': 1,
'b': 2
}),
_create_map_avro_record(schema, {
'c': 3,
'd': 4
})
]
return schema, records
def _create_union_avro_schema_and_py_objects(
) -> Tuple[AvroSchema, List[dict]]:
schema = AvroSchema.parse_string(UNION_SCHEMA)
records = [
{
'union': 1
},
{
'union': 2.
},
{
'union': None
},
]
return schema, records
def _create_map_avro_schema_and_py_objects() -> Tuple[AvroSchema, List[dict]]:
schema = AvroSchema.parse_string(MAP_SCHEMA)
records = [
{
'map': {
'a': 1,
'b': 2
}
},
{
'map': {
'c': 3,
'd': 4
}
},
]
return schema, records
def _create_union_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
record_schema = """
{
"type": "record",
"name": "test",
"fields": [
{
"name": "union",
"type": [ "int", "double", "null" ]
}
]
}
"""
schema = AvroSchema.parse_string(record_schema)
records = [
_create_union_avro_record(schema, 1),
_create_union_avro_record(schema, 2.),
_create_union_avro_record(schema, None)
]
return schema, records
def _create_basic_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
record_schema = """
{
"type": "record",
"name": "test",
"fields": [
{ "name": "null", "type": "null" },
{ "name": "boolean", "type": "boolean" },
{ "name": "int", "type": "int" },
{ "name": "long", "type": "long" },
{ "name": "float", "type": "float" },
{ "name": "double", "type": "double" },
{ "name": "string", "type": "string" }
]
}
"""
schema = AvroSchema.parse_string(record_schema)
records = [
_create_basic_avro_record(schema, True, 0, 1, 2, 3, 's1'),
_create_basic_avro_record(schema, False, 4, 5, 6, 7, 's2')
]
return schema, records
def _create_enum_avro_schema_and_records(
) -> Tuple[AvroSchema, List[JavaObject]]:
record_schema = """
{
"type": "record",
"name": "test",
"fields": [
{
"name": "suit",
"type": {
"type": "enum",
"name": "Suit",
"symbols" : ["SPADES", "HEARTS", "DIAMONDS", "CLUBS"]
}
}
]
}
"""
schema = AvroSchema.parse_string(record_schema)
records = [
_create_enum_avro_record(schema, 'SPADES'),
_create_enum_avro_record(schema, 'DIAMONDS')
]
return schema, records
def _create_array_avro_schema_and_py_objects(
) -> Tuple[AvroSchema, List[dict]]:
schema = AvroSchema.parse_string(ARRAY_SCHEMA)
records = [
{
'array': [{
'int': 1,
'double': 2.
}, {
'int': 3,
'double': 4.
}]
},
{
'array': [{
'int': 5,
'double': 6.
}, {
'int': 7,
'double': 8.
}]
},
]
return schema, records
def _create_basic_avro_schema_and_py_objects(
) -> Tuple[AvroSchema, List[dict]]:
schema = AvroSchema.parse_string(BASIC_SCHEMA)
objects = [
{
'null': None,
'boolean': True,
'int': 0,
'long': 1,
'float': 2.,
'double': 3.,
'string': 's1'
},
{
'null': None,
'boolean': False,
'int': 4,
'long': 5,
'float': 6.,
'double': 7.,
'string': 's2'
},
]
return schema, objects