def test_entity_uri_parse_glob_failure(self):
""" """
entity_uri = "dataplex://projects/project-id/locations/us-central1/lakes" \
"/lake-id/zones/zone-id/entities/test_entity_*"
# This should be supported eventually
with pytest.raises(NotImplementedError):
EntityUri.from_uri(entity_uri)
def test_entity_uri_parse_asset_id_failure(self):
""" """
entity_uri = "dataplex://projects/project-id/locations/us-central1/lakes" \
"/lake-id/zones/zone-id/assets/asset-id"
with pytest.raises(ValueError):
EntityUri.from_uri(entity_uri)
def test_entity_uri_parse_dataplex_uri_without_default_configs(self):
""" """
entity_uri = "dataplex://projects/project-id/locations/us-central1/lakes" \
"/lake-id/zones/zone-id/entities/entity-id"
parsed_uri = EntityUri.from_uri(entity_uri)
expected_entity_dict = {
"uri": "dataplex://projects/project-id/locations/us-central1/lakes"
"/lake-id/zones/zone-id/entities/entity-id",
"scheme": "DATAPLEX",
"entity_id": "entity-id",
"db_primary_key":
"projects/project-id/locations/us-central1/lakes/lake-id/zones/zone-id/entities/entity-id", # noqa: E501
"configs": {
"projects": "project-id",
"locations": "us-central1",
"lakes": "lake-id",
"zones": "zone-id",
"entities": "entity-id",
}
}
assert parsed_uri.scheme == expected_entity_dict["scheme"]
assert parsed_uri.uri_configs_string == (
"projects/project-id/locations/us-central1/lakes"
"/lake-id/zones/zone-id/entities/entity-id")
assert parsed_uri.default_configs is None
assert parsed_uri.complete_uri_string == expected_entity_dict["uri"]
assert parsed_uri.get_entity_id() == expected_entity_dict["entity_id"]
assert parsed_uri.configs_dict == expected_entity_dict["configs"]
assert parsed_uri.get_db_primary_key(
) == expected_entity_dict["db_primary_key"]
assert parsed_uri.to_dict() == expected_entity_dict
def test_entity_uri_parse_bigquery_uri_without_default_configs(self):
""" """
bigquery_uri = "bigquery://projects/project-id/datasets/dataset-id/tables/table-id"
parsed_uri = EntityUri.from_uri(bigquery_uri)
print(parsed_uri)
expected_entity_dict = {
"uri":
"bigquery://projects/project-id/datasets/dataset-id/tables/table-id",
"scheme": "BIGQUERY",
"entity_id":
"projects/project-id/datasets/dataset-id/tables/table-id",
"db_primary_key":
"projects/project-id/datasets/dataset-id/tables/table-id",
"configs": {
"projects": "project-id",
"datasets": "dataset-id",
"tables": "table-id",
}
}
assert parsed_uri.scheme == expected_entity_dict["scheme"]
assert parsed_uri.uri_configs_string == "projects/project-id/datasets/dataset-id/tables/table-id"
assert parsed_uri.default_configs is None
assert parsed_uri.complete_uri_string == expected_entity_dict["uri"]
assert parsed_uri.get_entity_id() == expected_entity_dict["entity_id"]
assert parsed_uri.get_db_primary_key(
) == expected_entity_dict["db_primary_key"]
assert parsed_uri.configs_dict == expected_entity_dict["configs"]
assert parsed_uri.to_dict() == expected_entity_dict
def test_entity_uri_parse_override_project_lake_id_failure(self):
""" """
entity_uri = "dataplex://projects/project-id-2/zones/zone-id/entities/entity-id"
default_configs = {
"projects": "project-id-1",
"locations": "us-central1",
"lakes": "lake-id",
}
# This should fail without metadata_defaults
with pytest.raises(ValueError):
EntityUri.from_uri(entity_uri)
parsed_uri = EntityUri.from_uri(uri_string=entity_uri,
default_configs=default_configs)
assert parsed_uri.complete_uri_string == entity_uri
assert parsed_uri.get_db_primary_key() == (
"projects/project-id-2/locations/us-central1/lakes"
"/lake-id/zones/zone-id/entities/entity-id")
def test_dq_entity_parse_bigquery_uri(self,
gcp_project_id,
test_dataplex_metadata_defaults_configs,
test_bigquery_client,):
bq_entity_uri_string = f"bigquery://projects/{gcp_project_id}/datasets/" \
f"austin_311/tables/contact_details_partitioned"
bq_entity_uri = EntityUri.from_uri(
uri_string=bq_entity_uri_string,
default_configs=test_dataplex_metadata_defaults_configs,
)
clouddq_entity = DqEntity.from_bq_entity_uri(
entity_uri=bq_entity_uri,
bigquery_client=test_bigquery_client)
clouddq_entity_expected_dict = {bq_entity_uri_string.upper().split("://")[1]: {
'source_database': 'BIGQUERY',
'table_name': 'contact_details_partitioned',
'database_name': 'austin_311',
'instance_name': f'{gcp_project_id}',
'columns': {
'ROW_ID': {'name': 'row_id', 'data_type': 'STRING'},
'CONTACT_TYPE': {'name': 'contact_type', 'data_type': 'STRING'},
'VALUE': {'name': 'value', 'data_type': 'STRING'},
'TS': {'name': 'ts', 'data_type': 'TIMESTAMP'}
},
'resource_type': 'BIGQUERY',
'partition_fields': [{
'name': 'ts',
'type': 'TIMESTAMP',
'partitioning_type': 'DAY'
}],
'dataset_name': 'austin_311',
'project_name': f'{gcp_project_id}'
}}
assert clouddq_entity.to_dict() == clouddq_entity_expected_dict
def test_entity_uri_parse_failure(self, entity_uri, error_type):
""" """
with pytest.raises(error_type):
EntityUri.from_uri(entity_uri)
def from_dict(
cls: DqRuleBinding,
rule_binding_id: str,
kwargs: dict,
default_configs: dict | None = None,
validate_uri: bool = True,
) -> DqRuleBinding:
"""
Args:
cls: DqRuleBinding:
rule_binding_id: typing.Union[str, str]:
kwargs: typing.Dict:
Returns:
"""
entity_config: dict = get_keys_from_dict_and_assert_oneof(
config_id=rule_binding_id,
kwargs=kwargs,
keys=["entity_uri", "entity_id"],
)
if "entity_id" in entity_config:
entity_id = entity_config["entity_id"]
entity_uri = None
if "entity_uri" in entity_config:
parsed_entity_uri = EntityUri.from_uri(
entity_config["entity_uri"],
default_configs=default_configs,
validate_uri=validate_uri,
)
entity_id = parsed_entity_uri.get_entity_id()
entity_uri = parsed_entity_uri
if entity_id:
entity_id.upper()
column_id: str = get_from_dict_and_assert(
config_id=rule_binding_id,
kwargs=kwargs,
key="column_id",
)
if column_id:
column_id.upper()
row_filter_id: str = get_from_dict_and_assert(
config_id=rule_binding_id,
kwargs=kwargs,
key="row_filter_id",
)
if row_filter_id:
row_filter_id.upper()
rule_ids: list[str] = get_from_dict_and_assert(
config_id=rule_binding_id,
kwargs=kwargs,
key="rule_ids",
assertion=lambda x: type(x) == list,
error_msg=
f"Rule Binding ID: '{rule_binding_id}' must have defined value "
f"'rule_ids' of type 'list'.",
)
incremental_time_filter_column_id: str | None = kwargs.get(
"incremental_time_filter_column_id", None)
if incremental_time_filter_column_id:
incremental_time_filter_column_id.upper()
metadata: dict | None = kwargs.get("metadata", dict())
if type(metadata) != dict:
raise ValueError(
f"Rule Binding ID: '{rule_binding_id}' has invalid "
f"metadata field with type {type(metadata)} and values: {metadata}\n"
"'metadata' must be of type dictionary.")
return DqRuleBinding(
rule_binding_id=str(rule_binding_id).upper(),
entity_id=entity_id,
entity_uri=entity_uri,
column_id=column_id,
row_filter_id=row_filter_id,
incremental_time_filter_column_id=incremental_time_filter_column_id,
rule_ids=rule_ids,
metadata=metadata,
)