def _catalog_filter_schemas(cls, manifest: Manifest) -> Callable[[agate.Row], bool]: schemas = frozenset((None, s.lower()) for d, s in manifest.get_used_schemas()) def _(row: agate.Row) -> bool: table_database = _expect_row_value("table_database", row) table_schema = _expect_row_value("table_schema", row) if table_schema is None: return False return (table_database, table_schema.lower()) in schemas return _
def get_catalog(self, manifest: Manifest) -> agate.Table: schemas = manifest.get_used_schemas() columns = [] for database, schema in schemas: relations = self.list_relations(database, schema) for relation in relations: logger.debug("Getting table schema for relation {}", relation) columns.extend( self._massage_column_for_catalog(col) for col in self.get_columns_in_relation(relation)) return agate.Table.from_object(columns)
def get_catalog(self, manifest: Manifest) -> agate.Table: schemas = manifest.get_used_schemas() columns = [] for (database_name, schema_name) in schemas: relations = self.list_relations(database_name, schema_name) for relation in relations: logger.debug("Getting table schema for relation {}", relation) columns += [ col.to_dict() for col in self.get_columns_in_relation(relation) ] return agate.Table.from_object(columns)
def get_catalog(self, manifest: Manifest) -> agate.Table: schemas = manifest.get_used_schemas() def to_dict(d: any) -> Dict: return d.__dict__ columns = [] for (database_name, schema_name) in schemas: relations = self.list_relations(database_name, schema_name) for relation in relations: logger.debug("Getting table schema for relation {}", relation) columns += list( map(to_dict, self.get_columns_in_relation(relation)) ) return agate.Table.from_object(columns)
def _catalog_filter_schemas(manifest: Manifest) -> Callable[[agate.Row], bool]: """Return a function that takes a row and decides if the row should be included in the catalog output. """ schemas = frozenset((d.lower(), s.lower()) for d, s in manifest.get_used_schemas()) def test(row: agate.Row) -> bool: table_database = _expect_row_value('table_database', row) table_schema = _expect_row_value('table_schema', row) # the schema may be present but None, which is not an error and should # be filtered out if table_schema is None: return False return (table_database.lower(), table_schema.lower()) in schemas return test
def get_catalog(self, manifest: Manifest) -> agate.Table: schemas = manifest.get_used_schemas() columns = [] for database, schema in schemas: relations = self.list_relations(database, schema) for relation in relations: properties = self.get_properties(relation) logger.debug("Getting table schema for relation {}".format( relation)) # noqa table_columns = self.get_columns_in_relation(relation) rel_type = self.get_relation_type(relation) columns += self._parse_relation(relation, table_columns, rel_type, properties) return table_from_data(columns, SparkAdapter.COLUMN_NAMES)