def test_presto_process_array_data(self):
data = [
{"array_column": [[1], [2]], "int_column": 3},
{"array_column": [[11], [22]], "int_column": 33},
]
all_columns = [
{"name": "array_column", "type": "ARRAY"},
{"name": "array_column.nested_row", "type": "BIGINT"},
{"name": "int_column", "type": "BIGINT"},
]
array_column_hierarchy = {
"array_column": {"type": "ARRAY", "children": ["array_column.nested_row"]}
}
actual_array_data = PrestoEngineSpec._process_array_data(
data, all_columns, array_column_hierarchy
)
expected_array_data = {
0: [
{"array_column": [[1], [2]], "array_column.nested_row": 1},
{"array_column": "", "array_column.nested_row": 2, "int_column": ""},
],
1: [
{"array_column": [[11], [22]], "array_column.nested_row": 11},
{"array_column": "", "array_column.nested_row": 22, "int_column": ""},
],
}
self.assertEqual(actual_array_data, expected_array_data)
def test_presto_process_array_data(self):
data = [
{
'array_column': [[1], [2]],
'int_column': 3
},
{
'array_column': [[11], [22]],
'int_column': 33
},
]
all_columns = [
{
'name': 'array_column',
'type': 'ARRAY'
},
{
'name': 'array_column.nested_row',
'type': 'BIGINT'
},
{
'name': 'int_column',
'type': 'BIGINT'
},
]
array_column_hierarchy = {
'array_column': {
'type': 'ARRAY',
'children': ['array_column.nested_row'],
},
}
actual_array_data = PrestoEngineSpec._process_array_data(
data, all_columns, array_column_hierarchy)
expected_array_data = {
0: [
{
'array_column': [[1], [2]],
'array_column.nested_row': 1
},
{
'array_column': '',
'array_column.nested_row': 2,
'int_column': ''
},
],
1: [
{
'array_column': [[11], [22]],
'array_column.nested_row': 11
},
{
'array_column': '',
'array_column.nested_row': 22,
'int_column': ''
},
],
}
self.assertEqual(actual_array_data, expected_array_data)
