def test_transform_array_struct_nested_type(self) -> None: column = ColumnMetadata('col1', None, 'array<struct<nest1:int,nest2:int>>', 0) column.set_column_key(self.column_key) array_type = ArrayTypeMetadata( name='col1', parent=column, type_str='array<struct<nest1:int,nest2:int>>') inner_struct = StructTypeMetadata( name='_inner_', parent=array_type, type_str='struct<nest1:int,nest2:int>') inner_scalar_nest1 = ScalarTypeMetadata(name='nest1', parent=inner_struct, type_str='int') inner_scalar_nest2 = ScalarTypeMetadata(name='nest2', parent=inner_struct, type_str='int') array_type.array_inner_type = inner_struct inner_struct.struct_items = { 'nest1': inner_scalar_nest1, 'nest2': inner_scalar_nest2 } inner_scalar_nest1.sort_order = 0 inner_scalar_nest2.sort_order = 1 actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, array_type)
def test_transform_map_struct_nested_type(self) -> None: column = ColumnMetadata('col1', None, 'map<string,struct<nest1:int,nest2:int>>', 0) column.set_column_key(self.column_key) map_type = MapTypeMetadata( name='col1', parent=column, type_str='map<string,struct<nest1:int,nest2:int>>') map_key = ScalarTypeMetadata(name='_map_key', parent=map_type, type_str='string') inner_struct = StructTypeMetadata( name='_map_value', parent=map_type, type_str='struct<nest1:int,nest2:int>') inner_scalar_nest1 = ScalarTypeMetadata(name='nest1', parent=inner_struct, type_str='int') inner_scalar_nest2 = ScalarTypeMetadata(name='nest2', parent=inner_struct, type_str='int') map_type.map_key_type = map_key map_type.map_value_type = inner_struct inner_struct.struct_items = { 'nest1': inner_scalar_nest1, 'nest2': inner_scalar_nest2 } inner_scalar_nest1.sort_order = 0 inner_scalar_nest2.sort_order = 1 actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, map_type)
def test_transform_union_as_nested_type(self) -> None: column = ColumnMetadata( 'col1', None, 'struct<nest1:uniontype<string,struct<c1:int,c2:string>>,' 'nest2:uniontype<string,int>>', 0) column.set_column_key(self.column_key) struct_type = StructTypeMetadata( name='col1', parent=column, type_str='struct<nest1:uniontype<string,struct<c1:int,c2:string>>,' 'nest2:uniontype<string,int>>') inner_scalar_nest1 = ScalarTypeMetadata( name='nest1', parent=struct_type, type_str='uniontype<string,struct<c1:int,c2:string>>') inner_scalar_nest2 = ScalarTypeMetadata( name='nest2', parent=struct_type, type_str='uniontype<string,int>') struct_type.struct_items = { 'nest1': inner_scalar_nest1, 'nest2': inner_scalar_nest2 } inner_scalar_nest1.sort_order = 0 inner_scalar_nest2.sort_order = 1 actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, struct_type)
def test_transform_struct_map_array_nested_type(self) -> None: column = ColumnMetadata( 'col1', None, 'struct<nest1:map<string,array<int>>,nest2:array<string>>', 0) column.set_column_key(self.column_key) struct_type = StructTypeMetadata( name='col1', parent=column, type_str='struct<nest1:map<string,array<int>>,nest2:array<string>>' ) inner_map = MapTypeMetadata(name='nest1', parent=struct_type, type_str='map<string,array<int>>') inner_map_key = ScalarTypeMetadata(name='_map_key', parent=inner_map, type_str='string') inner_map_array = ArrayTypeMetadata(name='_map_value', parent=inner_map, type_str='array<int>') inner_struct_array = ArrayTypeMetadata(name='nest2', parent=struct_type, type_str='array<string>') struct_type.struct_items = { 'nest1': inner_map, 'nest2': inner_struct_array } inner_map.map_key_type = inner_map_key inner_map.map_value_type = inner_map_array inner_map.sort_order = 0 inner_struct_array.sort_order = 1 actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, struct_type)
def test_transform_map_type(self) -> None: column = ColumnMetadata('col1', None, 'map<string,map<string,int>>', 0) column.set_column_key(self.column_key) map_type = MapTypeMetadata(name='col1', parent=column, type_str='map<string,map<string,int>>') map_key = ScalarTypeMetadata(name='_map_key', parent=map_type, type_str='string') map_value = MapTypeMetadata(name='_map_value', parent=map_type, type_str='map<string,int>') inner_map_key = ScalarTypeMetadata(name='_map_key', parent=map_value, type_str='string') inner_scalar = ScalarTypeMetadata(name='_map_value', parent=map_value, type_str='int') map_type.map_key_type = map_key map_type.map_value_type = map_value map_value.map_key_type = inner_map_key map_value.map_value_type = inner_scalar actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, map_type)
def test_transform_invalid_struct_inner_type(self) -> None: column = ColumnMetadata( 'col1', None, 'struct<nest1:varchar(256)å,' 'nest2:<derived from deserializer>>', 0) column.set_column_key(self.column_key) with self.assertRaises(ParseException): parse_hive_type(column.type, column.name, column)
def test_transform_no_complex_type(self) -> None: column = ColumnMetadata('col1', None, 'int', 0) column.set_column_key(self.column_key) scalar_type = ScalarTypeMetadata(name='col1', parent=column, type_str='int') actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, scalar_type)
def test_transform_non_alpha_only_types(self) -> None: column = ColumnMetadata( 'col1', None, 'struct<nest1:decimal(10,2),nest2:double precision,' 'nest3:varchar(32),nest4:map<varchar(32),decimal(10,2)>,' 'nest5:interval_day_time>', 0) column.set_column_key(self.column_key) struct_type = StructTypeMetadata( name='col1', parent=column, type_str='struct<nest1:decimal(10,2),nest2:double precision,' 'nest3:varchar(32),nest4:map<varchar(32),decimal(10,2)>,' 'nest5:interval_day_time>') inner_scalar_nest1 = ScalarTypeMetadata(name='nest1', parent=struct_type, type_str='decimal(10,2)') inner_scalar_nest2 = ScalarTypeMetadata(name='nest2', parent=struct_type, type_str='double precision') inner_scalar_nest3 = ScalarTypeMetadata(name='nest3', parent=struct_type, type_str='varchar(32)') inner_map_nest4 = MapTypeMetadata( name='nest4', parent=struct_type, type_str='map<varchar(32),decimal(10,2)>') inner_map_nest4_key = ScalarTypeMetadata(name='_map_key', parent=inner_map_nest4, type_str='varchar(32)') inner_map_nest4_value = ScalarTypeMetadata(name='_map_value', parent=inner_map_nest4, type_str='decimal(10,2)') inner_scalar_nest5 = ScalarTypeMetadata(name='nest5', parent=struct_type, type_str='interval_day_time') struct_type.struct_items = { 'nest1': inner_scalar_nest1, 'nest2': inner_scalar_nest2, 'nest3': inner_scalar_nest3, 'nest4': inner_map_nest4, 'nest5': inner_scalar_nest5 } inner_map_nest4.map_key_type = inner_map_nest4_key inner_map_nest4.map_value_type = inner_map_nest4_value inner_scalar_nest1.sort_order = 0 inner_scalar_nest2.sort_order = 1 inner_scalar_nest3.sort_order = 2 inner_map_nest4.sort_order = 3 inner_scalar_nest5.sort_order = 4 actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, struct_type)
def test_transform_union_as_scalar_type(self) -> None: column = ColumnMetadata('col1', None, 'uniontype<string,struct<c1:int,c2:string>>', 0) column.set_column_key(self.column_key) struct_type = ScalarTypeMetadata( name='col1', parent=column, type_str='uniontype<string,struct<c1:int,c2:string>>') actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, struct_type)
def test_transform_array_type(self) -> None: column = ColumnMetadata('col1', None, 'array<array<int>>', 0) column.set_column_key(self.column_key) array_type = ArrayTypeMetadata(name='col1', parent=column, type_str='array<array<int>>') inner_array = ArrayTypeMetadata(name='_inner_', parent=array_type, type_str='array<int>') array_type.array_inner_type = inner_array actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, array_type)
def setUp(self) -> None: super(TestTableMetadata, self).setUp() TableMetadata.serialized_nodes_keys = set() TableMetadata.serialized_rels_keys = set() column_with_type_metadata = ColumnMetadata( 'has_nested_type', 'column with nested types', 'array<array<array<string>>>', 6) column_with_type_metadata.set_column_key( 'hive://gold.test_schema1/test_table1/has_nested_type') column_with_type_metadata.set_type_metadata( self._set_up_type_metadata(column_with_type_metadata)) self.table_metadata = TableMetadata( 'hive', 'gold', 'test_schema1', 'test_table1', 'test_table1', [ ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0), ColumnMetadata('test_id2', 'description of test_id2', 'bigint', 1), ColumnMetadata('is_active', None, 'boolean', 2), ColumnMetadata('source', 'description of source', 'varchar', 3), ColumnMetadata('etl_created_at', 'description of etl_created_at', 'timestamp', 4), ColumnMetadata('ds', None, 'varchar', 5), column_with_type_metadata ]) self.table_metadata2 = TableMetadata( 'hive', 'gold', 'test_schema1', 'test_table1', 'test_table1', [ ColumnMetadata('test_id1', 'description of test_table1', 'bigint', 0), ColumnMetadata('test_id2', 'description of test_id2', 'bigint', 1), ColumnMetadata('is_active', None, 'boolean', 2), ColumnMetadata('source', 'description of source', 'varchar', 3), ColumnMetadata('etl_created_at', 'description of etl_created_at', 'timestamp', 4), ColumnMetadata('ds', None, 'varchar', 5), column_with_type_metadata ])
def test_transform_array_map_nested_type(self) -> None: column = ColumnMetadata('col1', None, 'array<map<string,int>>', 0) column.set_column_key(self.column_key) array_type = ArrayTypeMetadata(name='col1', parent=column, type_str='array<map<string,int>>') inner_map = MapTypeMetadata(name='_inner_', parent=array_type, type_str='map<string,int>') inner_map_key = ScalarTypeMetadata(name='_map_key', parent=inner_map, type_str='string') inner_scalar = ScalarTypeMetadata(name='_map_value', parent=inner_map, type_str='int') array_type.array_inner_type = inner_map inner_map.map_key_type = inner_map_key inner_map.map_value_type = inner_scalar actual = parse_hive_type(column.type, column.name, column) self.assertEqual(actual, array_type)
def test_transform_invalid_array_inner_type(self) -> None: column = ColumnMetadata('col1', None, 'array<array<int*>>', 0) column.set_column_key(self.column_key) with self.assertRaises(ParseException): parse_hive_type(column.type, column.name, column)
def test_serialize_struct_type_metadata(self) -> None: column = ColumnMetadata( 'col1', None, 'struct<c1:struct<c2:struct<c3:string,c4:string>>,c5:string>', 0) column.set_column_key(self.column_key) struct_type_metadata = StructTypeMetadata( name='col1', parent=column, type_str= 'struct<c1:struct<c2:struct<c3:string,c4:string>>,c5:string>') nested_struct_type_metadata_level1 = StructTypeMetadata( name='c1', parent=struct_type_metadata, type_str='struct<c2:struct<c3:string,c4:string>>') nested_struct_type_metadata_level2 = StructTypeMetadata( name='c2', parent=nested_struct_type_metadata_level1, type_str='struct<c3:string,c4:string>') nested_scalar_type_metadata_c3 = ScalarTypeMetadata( name='c3', parent=nested_struct_type_metadata_level2, type_str='string', description='description of c3') nested_scalar_type_metadata_c4 = ScalarTypeMetadata( name='c4', parent=nested_struct_type_metadata_level2, type_str='string') nested_scalar_type_metadata_c5 = ScalarTypeMetadata( name='c5', parent=struct_type_metadata, type_str='string', description='description of c5') struct_type_metadata.struct_items = { 'c1': nested_struct_type_metadata_level1, 'c5': nested_scalar_type_metadata_c5 } nested_struct_type_metadata_level1.struct_items = { 'c2': nested_struct_type_metadata_level2 } nested_struct_type_metadata_level2.struct_items = { 'c3': nested_scalar_type_metadata_c3, 'c4': nested_scalar_type_metadata_c4 } nested_struct_type_metadata_level1.sort_order = 0 nested_scalar_type_metadata_c5.sort_order = 1 nested_struct_type_metadata_level2.sort_order = 0 nested_scalar_type_metadata_c3.sort_order = 0 nested_scalar_type_metadata_c4.sort_order = 1 expected_nodes = [{ 'kind': 'struct', 'name': 'col1', 'data_type': 'struct<c1:struct<c2:struct<c3:string,c4:string>>,c5:string>', 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1' }, { 'kind': 'struct', 'name': 'c1', 'data_type': 'struct<c2:struct<c3:string,c4:string>>', 'LABEL': 'Type_Metadata', 'sort_order:UNQUOTED': 0, 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1' }, { 'kind': 'struct', 'name': 'c2', 'data_type': 'struct<c3:string,c4:string>', 'LABEL': 'Type_Metadata', 'sort_order:UNQUOTED': 0, 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2' }, { 'kind': 'scalar', 'name': 'c3', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'sort_order:UNQUOTED': 0, 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c3' }, { 'description': 'description of c3', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c3/_description', 'LABEL': 'Description', 'description_source': 'description' }, { 'kind': 'scalar', 'name': 'c4', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'sort_order:UNQUOTED': 1, 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c4' }, { 'kind': 'scalar', 'name': 'c5', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'sort_order:UNQUOTED': 1, 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c5' }, { 'description': 'description of c5', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c5/_description', 'LABEL': 'Description', 'description_source': 'description' }] expected_rels = [{ 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Column', 'TYPE': 'TYPE_METADATA', 'REVERSE_TYPE': 'TYPE_METADATA_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c3', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c3/_description', 'START_LABEL': 'Type_Metadata', 'END_LABEL': 'Description', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c3', 'TYPE': 'DESCRIPTION', 'REVERSE_TYPE': 'DESCRIPTION_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2/c4', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c1/c2', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c5', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c5/_description', 'START_LABEL': 'Type_Metadata', 'END_LABEL': 'Description', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/c5', 'TYPE': 'DESCRIPTION', 'REVERSE_TYPE': 'DESCRIPTION_OF' }] node_row = struct_type_metadata.next_node() actual = [] while node_row: node_row_serialized = neo4_serializer.serialize_node(node_row) actual.append(node_row_serialized) node_row = struct_type_metadata.next_node() for i in range(0, len(expected_nodes)): self.assertEqual(actual[i], expected_nodes[i]) relation_row = struct_type_metadata.next_relation() actual = [] while relation_row: relation_row_serialized = neo4_serializer.serialize_relationship( relation_row) actual.append(relation_row_serialized) relation_row = struct_type_metadata.next_relation() for i in range(0, len(expected_rels)): self.assertEqual(actual[i], expected_rels[i])
def test_serialize_map_struct_type_metadata(self) -> None: column = ColumnMetadata( 'col1', None, 'map<string,struct<c1:map<string,string>,c2:string>>', 0) column.set_column_key(self.column_key) map_type_metadata = MapTypeMetadata( name='col1', parent=column, type_str='map<string,struct<c1:map<string,string>,c2:string>>') map_key = ScalarTypeMetadata(name='_map_key', parent=map_type_metadata, type_str='string') nested_struct_type_metadata_level1 = StructTypeMetadata( name='_map_value', parent=map_type_metadata, type_str='struct<c1:map<string,string>,c2:string>') nested_map_type_metadata_level2 = MapTypeMetadata( name='c1', parent=nested_struct_type_metadata_level1, type_str='map<string,string>') nested_map_key = ScalarTypeMetadata( name='_map_key', parent=nested_map_type_metadata_level2, type_str='string') nested_scalar_type_metadata_level3 = ScalarTypeMetadata( name='_map_value', parent=nested_map_type_metadata_level2, type_str='string') nested_scalar_type_metadata_level2 = ScalarTypeMetadata( name='c2', parent=nested_struct_type_metadata_level1, type_str='string') map_type_metadata.map_key_type = map_key map_type_metadata.map_value_type = nested_struct_type_metadata_level1 nested_struct_type_metadata_level1.struct_items = { 'c1': nested_map_type_metadata_level2, 'c2': nested_scalar_type_metadata_level2 } nested_map_type_metadata_level2.map_key_type = nested_map_key nested_map_type_metadata_level2.map_value_type = nested_scalar_type_metadata_level3 nested_map_type_metadata_level2.sort_order = 0 nested_scalar_type_metadata_level2.sort_order = 1 expected_nodes = [{ 'kind': 'map', 'name': 'col1', 'data_type': 'map<string,struct<c1:map<string,string>,c2:string>>', 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1' }, { 'kind': 'scalar', 'name': '_map_key', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_key' }, { 'kind': 'struct', 'name': '_map_value', 'data_type': 'struct<c1:map<string,string>,c2:string>', 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value' }, { 'kind': 'map', 'name': 'c1', 'data_type': 'map<string,string>', 'sort_order:UNQUOTED': 0, 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1' }, { 'kind': 'scalar', 'name': '_map_key', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1/_map_key' }, { 'kind': 'scalar', 'name': '_map_value', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1/_map_value' }, { 'kind': 'scalar', 'name': 'c2', 'data_type': 'string', 'LABEL': 'Type_Metadata', 'sort_order:UNQUOTED': 1, 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c2' }] expected_rels = [{ 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Column', 'TYPE': 'TYPE_METADATA', 'REVERSE_TYPE': 'TYPE_METADATA_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_key', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1/_map_key', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1/_map_value', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value/c2', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_map_value', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }] node_row = map_type_metadata.next_node() actual = [] while node_row: node_row_serialized = neo4_serializer.serialize_node(node_row) actual.append(node_row_serialized) node_row = map_type_metadata.next_node() for i in range(0, len(expected_nodes)): self.assertEqual(actual[i], expected_nodes[i]) relation_row = map_type_metadata.next_relation() actual = [] while relation_row: relation_row_serialized = neo4_serializer.serialize_relationship( relation_row) actual.append(relation_row_serialized) relation_row = map_type_metadata.next_relation() for i in range(0, len(expected_rels)): self.assertEqual(actual[i], expected_rels[i])
def test_serialize_array_type_metadata(self) -> None: column = ColumnMetadata('col1', None, 'array<array<array<string>>>', 0) column.set_column_key(self.column_key) array_type_metadata = ArrayTypeMetadata( name='col1', parent=column, type_str='array<array<array<string>>>') nested_array_type_metadata_level1 = ArrayTypeMetadata( name='_inner_', parent=array_type_metadata, type_str='array<array<string>>') nested_array_type_metadata_level2 = ArrayTypeMetadata( name='_inner_', parent=nested_array_type_metadata_level1, type_str='array<string>') nested_scalar_type_metadata_level3 = ScalarTypeMetadata( name='_inner_', parent=nested_array_type_metadata_level2, type_str='string') array_type_metadata.array_inner_type = nested_array_type_metadata_level1 nested_array_type_metadata_level1.array_inner_type = nested_array_type_metadata_level2 nested_array_type_metadata_level2.array_inner_type = nested_scalar_type_metadata_level3 expected_nodes = [{ 'kind': 'array', 'name': 'col1', 'LABEL': 'Type_Metadata', 'data_type': 'array<array<array<string>>>', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1' }, { 'kind': 'array', 'name': '_inner_', 'LABEL': 'Type_Metadata', 'data_type': 'array<array<string>>', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_inner_' }, { 'kind': 'array', 'name': '_inner_', 'LABEL': 'Type_Metadata', 'data_type': 'array<string>', 'KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_inner_/_inner_' }] expected_rels = [{ 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Column', 'TYPE': 'TYPE_METADATA', 'REVERSE_TYPE': 'TYPE_METADATA_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_inner_', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }, { 'END_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_inner_/_inner_', 'START_KEY': 'hive://gold.test_schema1/test_table1/col1/type/col1/_inner_', 'END_LABEL': 'Type_Metadata', 'START_LABEL': 'Type_Metadata', 'TYPE': 'SUBTYPE', 'REVERSE_TYPE': 'SUBTYPE_OF' }] node_row = array_type_metadata.next_node() actual = [] while node_row: node_row_serialized = neo4_serializer.serialize_node(node_row) actual.append(node_row_serialized) node_row = array_type_metadata.next_node() for i in range(0, len(expected_nodes)): self.assertEqual(actual[i], expected_nodes[i]) relation_row = array_type_metadata.next_relation() actual = [] while relation_row: relation_row_serialized = neo4_serializer.serialize_relationship( relation_row) actual.append(relation_row_serialized) relation_row = array_type_metadata.next_relation() for i in range(0, len(expected_rels)): self.assertEqual(actual[i], expected_rels[i])