def data_sources_enroll_iterator(
client: "ImmutaClient",
schema_table_mapping: Dict[str, List[Dict[str, str]]],
schema_obj: Dict[str, str],
config: Dict[str, Any],
) -> Iterator[Tuple[DataSource, Handler]]:
LOGGER.info("Processing schema_prefix: %s", schema_obj["schema_prefix"])
matches_prefix = partial(fnmatch.fnmatch, pat=schema_obj["schema_prefix"])
for schema, tables in keyfilter(matches_prefix, schema_table_mapping).items():
LOGGER.info("Processing schema: %s", schema)
for table in tables:
if not fnmatch.fnmatch(table["tableName"], schema_obj["table_prefix"]):
continue
LOGGER.info("Processing table: %s.%s", schema, table["tableName"])
handler = make_handler_metadata(
config=config, table=table["tableName"], schema=schema
)
columns = client.get_column_types(
config=config, data_source_type=config["handler_type"], handler=handler
)
data_source, handler = to_immuta_objects(
schema=schema, table=table["tableName"], columns=columns, config=config
)
yield (data_source, handler)
def test_to_immuta_objects(
db_keys: Dict[str, str],
handler_type: str,
expected_type: Any,
columns: List[ds.DataSourceColumn],
query_engine_target_schema: str,
prefix_query_engine_names_with_schema: bool,
prefix_query_engine_names_with_handler: bool,
):
base_config = {
"username": "******",
"password": "******",
"database": "baz",
"owner_profile_id": 0,
}
config = {**base_config, **db_keys}
config["handler_type"] = handler_type
source, handler, schema_evolution = ds.to_immuta_objects(
table="foo",
schema="bar",
config=config,
columns=columns,
bodata_schema_name=query_engine_target_schema,
prefix_query_engine_names_with_schema=
prefix_query_engine_names_with_schema,
prefix_query_engine_names_with_handler=
prefix_query_engine_names_with_handler,
)
assert source.name == ds.make_immuta_datasource_name(
table="foo",
schema="bar",
handler_type=handler_type,
user_prefix="",
)
assert source.sqlTableName == ds.make_postgres_table_name(
table="foo",
schema="bar" if prefix_query_engine_names_with_schema else "",
handler_type=handler_type
if prefix_query_engine_names_with_handler else "",
user_prefix="",
)
assert source.blobHandlerType == handler_type
assert handler.metadata.bodataSchemaName == query_engine_target_schema
assert isinstance(handler, ds.Handler)
assert isinstance(handler.metadata, expected_type)
assert isinstance(schema_evolution, ds.SchemaEvolutionMetadata)
def test_to_immuta_objects(
db_keys: Dict[str, str],
handler_type: str,
expected_type: Any,
columns: List[ds.DataSourceColumn],
):
base_config = {"username": "******", "password": "******", "database": "baz"}
config = {**base_config, **db_keys}
config["handler_type"] = handler_type
source, handler = ds.to_immuta_objects(
table="foo", schema="bar", config=config, columns=columns
)
assert source.name == ds.make_immuta_table_name(
table="foo", schema="bar", handler_type=handler_type, user_prefix=""
)
assert source.sqlTableName == ds.make_postgres_table_name(
table="foo", schema="bar", handler_type=handler_type, user_prefix=""
)
assert source.blobHandlerType == handler_type
assert isinstance(handler, ds.Handler)
assert isinstance(handler.metadata, expected_type)