def _catalog_filter_schemas(cls, manifest: Manifest) -> Callable[[agate.Row], bool]:
schemas = frozenset((None, s.lower()) for d, s in manifest.get_used_schemas())
def _(row: agate.Row) -> bool:
table_database = _expect_row_value("table_database", row)
table_schema = _expect_row_value("table_schema", row)
if table_schema is None:
return False
return (table_database, table_schema.lower()) in schemas
return _
def get_catalog(self, manifest: Manifest) -> agate.Table:
schemas = manifest.get_used_schemas()
columns = []
for database, schema in schemas:
relations = self.list_relations(database, schema)
for relation in relations:
logger.debug("Getting table schema for relation {}", relation)
columns.extend(
self._massage_column_for_catalog(col)
for col in self.get_columns_in_relation(relation))
return agate.Table.from_object(columns)
def get_catalog(self, manifest: Manifest) -> agate.Table:
schemas = manifest.get_used_schemas()
columns = []
for (database_name, schema_name) in schemas:
relations = self.list_relations(database_name, schema_name)
for relation in relations:
logger.debug("Getting table schema for relation {}", relation)
columns += [
col.to_dict()
for col in self.get_columns_in_relation(relation)
]
return agate.Table.from_object(columns)
def get_catalog(self, manifest: Manifest) -> agate.Table:
schemas = manifest.get_used_schemas()
def to_dict(d: any) -> Dict:
return d.__dict__
columns = []
for (database_name, schema_name) in schemas:
relations = self.list_relations(database_name, schema_name)
for relation in relations:
logger.debug("Getting table schema for relation {}", relation)
columns += list(
map(to_dict, self.get_columns_in_relation(relation))
)
return agate.Table.from_object(columns)
def _catalog_filter_schemas(manifest: Manifest) -> Callable[[agate.Row], bool]:
"""Return a function that takes a row and decides if the row should be
included in the catalog output.
"""
schemas = frozenset((d.lower(), s.lower())
for d, s in manifest.get_used_schemas())
def test(row: agate.Row) -> bool:
table_database = _expect_row_value('table_database', row)
table_schema = _expect_row_value('table_schema', row)
# the schema may be present but None, which is not an error and should
# be filtered out
if table_schema is None:
return False
return (table_database.lower(), table_schema.lower()) in schemas
return test
def get_catalog(self, manifest: Manifest) -> agate.Table:
schemas = manifest.get_used_schemas()
columns = []
for database, schema in schemas:
relations = self.list_relations(database, schema)
for relation in relations:
properties = self.get_properties(relation)
logger.debug("Getting table schema for relation {}".format(
relation)) # noqa
table_columns = self.get_columns_in_relation(relation)
rel_type = self.get_relation_type(relation)
columns += self._parse_relation(relation, table_columns,
rel_type, properties)
return table_from_data(columns, SparkAdapter.COLUMN_NAMES)
