def test_scrape_tables(self) -> None:
table = Table(name="test_table1",
database="test_schema1",
description=None,
tableType="delta",
isTemporary=False)
actual = self.dExtractor.scrape_table(table)
expected = ScrapedTableMetadata(schema="test_schema1",
table="test_table1")
expected.set_columns([
ScrapedColumnMetadata(name="a",
description=None,
data_type="string",
sort_order=0),
ScrapedColumnMetadata(name="b",
description=None,
data_type="int",
sort_order=1)
])
if actual is not None:
self.assertEqual(expected.schema, actual.schema)
self.assertEqual(expected.table, actual.table)
self.assertEqual(expected.columns, actual.columns)
self.assertEqual(expected.failed_to_scrape,
actual.failed_to_scrape)
self.assertEqual(expected.is_view, actual.is_view)
self.assertIsNotNone(actual.table_detail)
else:
self.assertIsNotNone(actual)
def test_fetch_partitioned_delta_columns(self) -> None:
actual = self.dExtractor.fetch_columns("test_schema1", "test_table3")
partition_column = ScrapedColumnMetadata(name="c", description=None, data_type="boolean", sort_order=0)
partition_column.set_is_partition(True)
expected = [partition_column,
ScrapedColumnMetadata(name="d", description=None, data_type="float", sort_order=1)]
for a, b in zip(actual, expected):
self.assertEqual(a, b)
def test_create_table_metadata(self) -> None:
scraped = ScrapedTableMetadata(schema="test_schema1", table="test_table1")
scraped.set_columns([ScrapedColumnMetadata(name="a", description=None, data_type="string", sort_order=0),
ScrapedColumnMetadata(name="b", description=None, data_type="int", sort_order=1)])
created_metadata = self.dExtractor.create_table_metadata(scraped)
expected = TableMetadata("test_database", "test_cluster", "test_schema1", "test_table1", description=None,
columns=[ColumnMetadata("a", None, "string", 0),
ColumnMetadata("b", None, "int", 1)])
self.assertEqual(str(expected), str(created_metadata))
def test_fetch_delta_columns(self) -> None:
actual = self.dExtractor.fetch_columns("test_schema1", "test_table1")
expected = [
ScrapedColumnMetadata(name="a",
description=None,
data_type="string",
sort_order=0),
ScrapedColumnMetadata(name="b",
description=None,
data_type="int",
sort_order=1)
]
for a, b in zip(actual, expected):
self.assertEqual(a, b)