def test__unflatten_one_column(self): columns = [{ 'column_comment': None, 'column_index': Decimal('1'), 'column_name': 'id', 'column_type': 'integer', 'table_comment': None, 'table_name': 'test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE' }] expected = { 'test_schema': { 'test_table': { 'metadata': { 'comment': None, 'name': 'test_table', 'type': 'BASE TABLE', 'schema': 'test_schema', }, 'columns': [ { 'type': 'integer', 'comment': None, 'index': bigint(1), 'name': 'id' } ] } } } result = generate.unflatten(columns) self.assertEqual(result, expected)
def test__unflatten_one_column(self): columns = [{ 'column_comment': None, 'column_index': Decimal('1'), 'column_name': 'id', 'column_type': 'integer', 'table_comment': None, 'table_name': 'test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE' }] expected = { 'test_schema': { 'test_table': { 'metadata': { 'comment': None, 'name': 'test_table', 'type': 'BASE TABLE', 'schema': 'test_schema', }, 'columns': { 'id': { 'type': 'integer', 'comment': None, 'index': bigint(1), 'name': 'id' }, }, 'stats': { 'has_stats': { 'id': 'has_stats', 'label': 'Has Stats?', 'value': False, 'description': 'Indicates whether there are statistics for this table', 'include': False, }, }, } } } result = generate.unflatten(columns) self.assertEqual(result, expected)
def run(self): single_file = self.args.single_file write_files = self.args.write_files schemas = self.args.schemas logger.info("Bootstrapping the following schemas:") for schema in schemas: logger.info("- {}".format(schema)) # Look up all of the relations in the DB manifest = self._get_manifest() adapter = dbt.adapters.factory.get_adapter(self.config) all_relations = adapter.get_catalog(manifest) selected_relations = all_relations.where( lambda row: row["table_schema"] in schemas) zipped_relations = [ dict(zip(selected_relations.column_names, row)) for row in selected_relations ] relations_to_design = unflatten(zipped_relations) if len(relations_to_design) == 0: logger.info( dbt.ui.printer.yellow( "Warning: No relations found in selected schemas: {}." "\nAborting.".format(schemas))) return {} for schema, relations in relations_to_design.items(): schema_path = os.path.join(self.config.source_paths[0], schema) if not write_files: pass elif os.path.isdir(schema_path): pass else: os.mkdir(schema_path) all_models = [] for relation, meta_data in relations.items(): relation_dict = self.prep_metadata(meta_data) all_models.append(relation_dict) if not single_file: if not write_files: logger.info("-" * 20) logger.info("Design for relation: {}.{}".format( schema, relation)) logger.info("-" * 20) yml = self.render_relations([relation_dict]) self.print_relation(yml) else: design_file_name = "{}.yml".format(relation) design_file_path = os.path.join( schema_path, design_file_name) yml = self.render_relations([relation_dict]) self.write_relation(design_file_path, yml) if single_file: if not write_files: logger.info("-" * 20) logger.info("Design for schmea: {}".format(schema)) logger.info("-" * 20) yml = self.render_relations(all_models) self.print_relation(yml) else: design_file_name = "{}.yml".format(schema) design_file_path = os.path.join(schema_path, design_file_name) yml = self.render_relations([all_models]) self.write_relation(design_file_path, yml) return all_models
def test__unflatten_empty(self): columns = {} expected = {} result = generate.unflatten(columns) self.assertEqual(result, expected)
def test__unflatten_multiple_schemas(self): columns = [ { 'column_comment': None, 'column_index': Decimal('1'), 'column_name': 'id', 'column_type': 'integer', 'table_comment': None, 'table_name': 'test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE' }, { 'column_comment': None, 'column_index': Decimal('2'), 'column_name': 'name', 'column_type': 'text', 'table_comment': None, 'table_name': 'test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE' }, { 'column_comment': None, 'column_index': Decimal('1'), 'column_name': 'id', 'column_type': 'integer', 'table_comment': None, 'table_name': 'other_test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE', }, { 'column_comment': None, 'column_index': Decimal('2'), 'column_name': 'email', 'column_type': 'character varying', 'table_comment': None, 'table_name': 'other_test_table', 'table_schema': 'test_schema', 'table_type': 'BASE TABLE', }, { 'column_comment': None, 'column_index': Decimal('1'), 'column_name': 'id', 'column_type': 'integer', 'table_comment': None, 'table_name': 'test_table', 'table_schema': 'other_test_schema', 'table_type': 'BASE TABLE' }, { 'column_comment': None, 'column_index': Decimal('2'), 'column_name': 'name', 'column_type': 'text', 'table_comment': None, 'table_name': 'test_table', 'table_schema': 'other_test_schema', 'table_type': 'BASE TABLE' }, ] expected = { 'test_schema': { 'test_table': { 'metadata': { 'comment': None, 'name': 'test_table', 'type': 'BASE TABLE', 'schema': 'test_schema', }, 'columns': [ { 'type': 'integer', 'comment': None, 'index': bigint(1), 'name': 'id' }, { 'type': 'text', 'comment': None, 'index': Decimal('2'), 'name': 'name', } ], }, 'other_test_table': { 'metadata': { 'comment': None, 'name': 'other_test_table', 'type': 'BASE TABLE', 'schema': 'test_schema', }, 'columns': [ { 'type': 'integer', 'comment': None, 'index': bigint(1), 'name': 'id' }, { 'type': 'character varying', 'comment': None, 'index': Decimal('2'), 'name': 'email', } ] } }, 'other_test_schema': { 'test_table': { 'metadata': { 'comment': None, 'name': 'test_table', 'type': 'BASE TABLE', 'schema': 'other_test_schema', }, 'columns': [ { 'type': 'integer', 'comment': None, 'index': bigint(1), 'name': 'id' }, { 'type': 'text', 'comment': None, 'index': Decimal('2'), 'name': 'name', } ], }, } } result = generate.unflatten(columns) self.assertEqual(result, expected)