def test_bq_timezone_validation():
with pytest.raises(pydantic.ValidationError, match="UTC"):
BigQueryUsageConfig.parse_obj(
dict(
project_id="sample-bigquery-project-name-1234",
start_time="2021-07-20T00:00:00",
))
def test_bq_usage_config():
config = BigQueryUsageConfig.parse_obj(
dict(
project_id="sample-bigquery-project-name-1234",
bucket_duration="HOUR",
end_time="2021-07-20T00:00:00Z",
))
assert (config.end_time - config.start_time) == timedelta(hours=1)
assert config.projects == ["sample-bigquery-project-name-1234"]
def test_bq_usage_config():
config = BigQueryUsageConfig.parse_obj(
dict(
project_id="sample-bigquery-project-name-1234",
bucket_duration="HOUR",
end_time="2021-07-20T00:00:00Z",
table_pattern={"allow": ["test-regex", "test-regex-1"], "deny": []},
)
)
assert config.get_allow_pattern_string() == "test-regex|test-regex-1"
assert config.get_deny_pattern_string() == ""
assert (config.end_time - config.start_time) == timedelta(hours=1)
assert config.projects == ["sample-bigquery-project-name-1234"]
def test_bigquery_uri_with_credential():
from datahub.ingestion.source.usage.bigquery_usage import BigQueryUsageConfig
expected_credential_json = {
"auth_provider_x509_cert_url":
"https://www.googleapis.com/oauth2/v1/certs",
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
"client_email": "*****@*****.**",
"client_id": "test_client-id",
"client_x509_cert_url":
"https://www.googleapis.com/robot/v1/metadata/x509/[email protected]",
"private_key": "random_private_key",
"private_key_id": "test-private-key",
"project_id": "test-project",
"token_uri": "https://oauth2.googleapis.com/token",
"type": "service_account",
}
config = BigQueryUsageConfig.parse_obj({
"project_id": "test-project",
"credential": {
"project_id": "test-project",
"private_key_id": "test-private-key",
"private_key": "random_private_key",
"client_email": "*****@*****.**",
"client_id": "test_client-id",
},
})
try:
assert config._credentials_path
with open(config._credentials_path) as jsonFile:
json_credential = json.load(jsonFile)
jsonFile.close()
credential = json.dumps(json_credential, sort_keys=True)
expected_credential = json.dumps(expected_credential_json,
sort_keys=True)
assert expected_credential == credential
except AssertionError as e:
if config._credentials_path:
os.unlink(str(config._credentials_path))
raise e