def test__unflatten_one_column(self):
columns = [{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
}]
expected = {
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
}
]
}
}
}
result = generate.unflatten(columns)
self.assertEqual(result, expected)
def test__unflatten_one_column(self):
columns = [{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
}]
expected = {
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': {
'id': {
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
},
'stats': {
'has_stats': {
'id': 'has_stats',
'label': 'Has Stats?',
'value': False,
'description': 'Indicates whether there are statistics for this table',
'include': False,
},
},
}
}
}
result = generate.unflatten(columns)
self.assertEqual(result, expected)
def run(self):
single_file = self.args.single_file
write_files = self.args.write_files
schemas = self.args.schemas
logger.info("Bootstrapping the following schemas:")
for schema in schemas:
logger.info("- {}".format(schema))
# Look up all of the relations in the DB
manifest = self._get_manifest()
adapter = dbt.adapters.factory.get_adapter(self.config)
all_relations = adapter.get_catalog(manifest)
selected_relations = all_relations.where(
lambda row: row["table_schema"] in schemas)
zipped_relations = [
dict(zip(selected_relations.column_names, row))
for row in selected_relations
]
relations_to_design = unflatten(zipped_relations)
if len(relations_to_design) == 0:
logger.info(
dbt.ui.printer.yellow(
"Warning: No relations found in selected schemas: {}."
"\nAborting.".format(schemas)))
return {}
for schema, relations in relations_to_design.items():
schema_path = os.path.join(self.config.source_paths[0], schema)
if not write_files:
pass
elif os.path.isdir(schema_path):
pass
else:
os.mkdir(schema_path)
all_models = []
for relation, meta_data in relations.items():
relation_dict = self.prep_metadata(meta_data)
all_models.append(relation_dict)
if not single_file:
if not write_files:
logger.info("-" * 20)
logger.info("Design for relation: {}.{}".format(
schema, relation))
logger.info("-" * 20)
yml = self.render_relations([relation_dict])
self.print_relation(yml)
else:
design_file_name = "{}.yml".format(relation)
design_file_path = os.path.join(
schema_path, design_file_name)
yml = self.render_relations([relation_dict])
self.write_relation(design_file_path, yml)
if single_file:
if not write_files:
logger.info("-" * 20)
logger.info("Design for schmea: {}".format(schema))
logger.info("-" * 20)
yml = self.render_relations(all_models)
self.print_relation(yml)
else:
design_file_name = "{}.yml".format(schema)
design_file_path = os.path.join(schema_path,
design_file_name)
yml = self.render_relations([all_models])
self.write_relation(design_file_path, yml)
return all_models
def test__unflatten_empty(self):
columns = {}
expected = {}
result = generate.unflatten(columns)
self.assertEqual(result, expected)
def test__unflatten_multiple_schemas(self):
columns = [
{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
},
{
'column_comment': None,
'column_index': Decimal('2'),
'column_name': 'name',
'column_type': 'text',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE'
},
{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'other_test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE',
},
{
'column_comment': None,
'column_index': Decimal('2'),
'column_name': 'email',
'column_type': 'character varying',
'table_comment': None,
'table_name': 'other_test_table',
'table_schema': 'test_schema',
'table_type': 'BASE TABLE',
},
{
'column_comment': None,
'column_index': Decimal('1'),
'column_name': 'id',
'column_type': 'integer',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'other_test_schema',
'table_type': 'BASE TABLE'
},
{
'column_comment': None,
'column_index': Decimal('2'),
'column_name': 'name',
'column_type': 'text',
'table_comment': None,
'table_name': 'test_table',
'table_schema': 'other_test_schema',
'table_type': 'BASE TABLE'
},
]
expected = {
'test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
{
'type': 'text',
'comment': None,
'index': Decimal('2'),
'name': 'name',
}
],
},
'other_test_table': {
'metadata': {
'comment': None,
'name': 'other_test_table',
'type': 'BASE TABLE',
'schema': 'test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
{
'type': 'character varying',
'comment': None,
'index': Decimal('2'),
'name': 'email',
}
]
}
},
'other_test_schema': {
'test_table': {
'metadata': {
'comment': None,
'name': 'test_table',
'type': 'BASE TABLE',
'schema': 'other_test_schema',
},
'columns': [
{
'type': 'integer',
'comment': None,
'index': bigint(1),
'name': 'id'
},
{
'type': 'text',
'comment': None,
'index': Decimal('2'),
'name': 'name',
}
],
},
}
}
result = generate.unflatten(columns)
self.assertEqual(result, expected)