def test__unflatten_one_column(self):
columns = [{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
}]
expected = {
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
}
]
}
}
}
result = generate.unflatten(columns)
self.assertEqual(result, expected)
def test__unflatten_one_column(self):
columns = [{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
}]
expected = {
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': {
'id': {
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
},
'stats': {
'has_stats': {
'id': 'has_stats',
'label': 'Has Stats?',
'value': False,
'description': 'Indicates whether there are statistics for this table',
'include': False,
},
},
}
}
}
result = generate.unflatten(columns)
self.assertEqual(result, expected)
def unflatten(columns):
"""Given a list of column dictionaries following this layout:
[{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
}]
unflatten will convert them into a dict with this nested structure:
{
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': {
"id": {
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
}
}
}
}
}
Required keys in each column: table_schema, table_name, column_index
Keys prefixed with 'column_' end up in per-column data and keys prefixed
with 'table_' end up in table metadata. Keys without either prefix are
ignored.
"""
structured = {}
for entry in columns:
schema_name = entry['table_schema']
table_name = entry['table_name']
if schema_name not in structured:
structured[schema_name] = {}
schema = structured[schema_name]
if table_name not in schema:
metadata = get_stripped_prefix(entry, 'table_')
stats = get_stripped_prefix(entry, 'stats:')
stats_dict = format_stats(stats)
schema[table_name] = {
'metadata': metadata,
'stats': stats_dict,
'columns': {}
}
table = schema[table_name]
column = get_stripped_prefix(entry, 'column_')
# the index should really never be that big so it's ok to end up
# serializing this to JSON (2^53 is the max safe value there)
column['index'] = bigint(column['index'])
table['columns'][column['name']] = column
return structured
def unflatten(columns):
"""Given a list of column dictionaries following this layout:
[{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
}]
unflatten will convert them into a dict with this nested structure:
{
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': {
"id": {
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
}
}
}
}
}
Required keys in each column: table_schema, table_name, column_index
Keys prefixed with 'column_' end up in per-column data and keys prefixed
with 'table_' end up in table metadata. Keys without either prefix are
ignored.
"""
structured = {}
for entry in columns:
schema_name = entry['table_schema']
table_name = entry['table_name']
if schema_name not in structured:
structured[schema_name] = {}
schema = structured[schema_name]
if table_name not in schema:
metadata = get_stripped_prefix(entry, 'table_')
stats = get_stripped_prefix(entry, 'stats:')
stats_dict = format_stats(stats)
schema[table_name] = {
'metadata': metadata,
'stats': stats_dict,
'columns': {}
}
table = schema[table_name]
column = get_stripped_prefix(entry, 'column_')
# the index should really never be that big so it's ok to end up
# serializing this to JSON (2^53 is the max safe value there)
column['index'] = bigint(column['index'])
table['columns'][column['name']] = column
return structured
def test__unflatten_multiple_schemas(self):
columns = [
{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
},
{
'column_comment': None,
'column_index': Decimal('2'),
'column_name': 'name',
'column_type': 'text',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
},
{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'other_test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE',
},
{
'column_comment': None,
'column_index': Decimal('2'),
'column_name': 'email',
'column_type': 'character varying',
'table_comment': None,
'table_name': 'other_test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE',
},
{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'other_test_schema',
'table_type': 'BASE TABLE'
},
{
'column_comment': None,
'column_index': Decimal('2'),
'column_name': 'name',
'column_type': 'text',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'other_test_schema',
'table_type': 'BASE TABLE'
},
]
expected = {
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
{
'type': 'text',
'comment': None,
'index': Decimal('2'),
'name': 'name',
}
],
},
'other_test_table': {
'metadata': {
'comment': None,
'name': 'other_test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
{
'type': 'character varying',
'comment': None,
'index': Decimal('2'),
'name': 'email',
}
]
}
},
'other_test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'other_test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
{
'type': 'text',
'comment': None,
'index': Decimal('2'),
'name': 'name',
}
],
},
}
}
result = generate.unflatten(columns)
self.assertEqual(result, expected)