def test_get_iam_policy(self):
# Setup Expected Response
version = 351608024
etag = b"21"
expected_response = {"version": version, "etag": etag}
expected_response = policy_pb2.Policy(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
resource = client.tag_template_path("[PROJECT]", "[LOCATION]",
"[TAG_TEMPLATE]")
response = client.get_iam_policy(resource)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = iam_policy_pb2.GetIamPolicyRequest(
resource=resource)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_create_tag(self):
# Setup Expected Response
name = "name3373707"
template = "template-1321546630"
template_display_name = "templateDisplayName-532252787"
column = "column-1354837162"
expected_response = {
"name": name,
"template": template,
"template_display_name": template_display_name,
"column": column,
}
expected_response = tags_pb2.Tag(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
parent = client.entry_path("[PROJECT]", "[LOCATION]", "[ENTRY_GROUP]",
"[ENTRY]")
tag = {}
response = client.create_tag(parent, tag)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.CreateTagRequest(parent=parent,
tag=tag)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_test_iam_permissions(self):
# Setup Expected Response
expected_response = {}
expected_response = iam_policy_pb2.TestIamPermissionsResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
resource = client.tag_template_path("[PROJECT]", "[LOCATION]",
"[TAG_TEMPLATE]")
permissions = []
response = client.test_iam_permissions(resource, permissions)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = iam_policy_pb2.TestIamPermissionsRequest(
resource=resource, permissions=permissions)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_update_tag_template_field(self):
# Setup Expected Response
display_name = "displayName1615086568"
expected_response = {"display_name": display_name}
expected_response = tags_pb2.TagTemplateField(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
name = client.field_path("[PROJECT]", "[LOCATION]", "[TAG_TEMPLATE]",
"[FIELD]")
tag_template_field = {}
response = client.update_tag_template_field(name, tag_template_field)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.UpdateTagTemplateFieldRequest(
name=name, tag_template_field=tag_template_field)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_tag_template(self):
# Setup Expected Response
name_2 = "name2-1052831874"
display_name = "displayName1615086568"
expected_response = {"name": name_2, "display_name": display_name}
expected_response = tags_pb2.TagTemplate(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
name = client.tag_template_path("[PROJECT]", "[LOCATION]",
"[TAG_TEMPLATE]")
response = client.get_tag_template(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.GetTagTemplateRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def sample_get_entry(project_id, location_id, entry_group_id, entry_id):
"""
Get Entry
Args:
project_id Your Google Cloud project ID
location_id Google Cloud region, e.g. us-central1
entry_group_id ID of the Entry Group, e.g. @bigquery, @pubsub, my_entry_group
entry_id ID of the Entry
"""
client = datacatalog_v1beta1.DataCatalogClient()
# project_id = '[Google Cloud Project ID]'
# location_id = '[Google Cloud Location ID]'
# entry_group_id = '[Entry Group ID]'
# entry_id = '[Entry ID]'
name = client.entry_path(project_id, location_id, entry_group_id, entry_id)
response = client.get_entry(request={'name': name})
entry = response
print(u"Entry name: {}".format(entry.name))
print(u"Entry type: {}".format(
datacatalog_v1beta1.EntryType(entry.type).name))
print(u"Linked resource: {}".format(entry.linked_resource))
def test_search_catalog(self):
# Setup Expected Response
next_page_token = ""
results_element = {}
results = [results_element]
expected_response = {
"next_page_token": next_page_token,
"results": results
}
expected_response = datacatalog_pb2.SearchCatalogResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
scope = {}
query = "query107944136"
paged_list_response = client.search_catalog(scope, query)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.results[0] == resources[0]
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.SearchCatalogRequest(scope=scope,
query=query)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_update_entry_group(self):
# Setup Expected Response
name = "name3373707"
display_name = "displayName1615086568"
description = "description-1724546052"
expected_response = {
"name": name,
"display_name": display_name,
"description": description,
}
expected_response = datacatalog_pb2.EntryGroup(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
entry_group = {}
response = client.update_entry_group(entry_group)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.UpdateEntryGroupRequest(
entry_group=entry_group)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_tags(self):
# Setup Expected Response
next_page_token = ""
tags_element = {}
tags = [tags_element]
expected_response = {"next_page_token": next_page_token, "tags": tags}
expected_response = datacatalog_pb2.ListTagsResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
parent = client.entry_path("[PROJECT]", "[LOCATION]", "[ENTRY_GROUP]",
"[ENTRY]")
paged_list_response = client.list_tags(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.tags[0] == resources[0]
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.ListTagsRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_update_tag_template(self):
# Setup Expected Response
name = "name3373707"
display_name = "displayName1615086568"
expected_response = {"name": name, "display_name": display_name}
expected_response = tags_pb2.TagTemplate(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
tag_template = {}
response = client.update_tag_template(tag_template)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.UpdateTagTemplateRequest(
tag_template=tag_template)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def sample_get_entry(project_id: str, location_id: str, entry_group_id: str,
entry_id: str):
# [START data_catalog_get_entry_v1beta1]
from google.cloud import datacatalog_v1beta1
"""
Get Entry
Args:
project_id (str): Your Google Cloud project ID
location_id (str): Google Cloud region, e.g. us-central1
entry_group_id (str): ID of the Entry Group, e.g. @bigquery, @pubsub, my_entry_group
entry_id (str): ID of the Entry
"""
client = datacatalog_v1beta1.DataCatalogClient()
# project_id = '[Google Cloud Project ID]'
# location_id = '[Google Cloud Location ID]'
# entry_group_id = '[Entry Group ID]'
# entry_id = '[Entry ID]'
name = client.entry_path(project_id, location_id, entry_group_id, entry_id)
entry = client.get_entry(request={"name": name})
print(f"Entry name: {entry.name}")
print(f"Entry type: {datacatalog_v1beta1.EntryType(entry.type_).name}")
print(f"Linked resource: {entry.linked_resource}")
# [END data_catalog_get_entry_v1beta1]
return entry
def test_rename_tag_template_field(self):
# Setup Expected Response
name_2 = "name2-1052831874"
display_name = "displayName1615086568"
is_required = True
expected_response = {
"name": name_2,
"display_name": display_name,
"is_required": is_required,
}
expected_response = tags_pb2.TagTemplateField(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
name = client.field_path("[PROJECT]", "[LOCATION]", "[TAG_TEMPLATE]",
"[FIELD]")
new_tag_template_field_id = "newTagTemplateFieldId-1668354591"
response = client.rename_tag_template_field(name,
new_tag_template_field_id)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.RenameTagTemplateFieldRequest(
name=name, new_tag_template_field_id=new_tag_template_field_id)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_create_entry_group(self):
# Setup Expected Response
name = "name3373707"
display_name = "displayName1615086568"
description = "description-1724546052"
expected_response = {
"name": name,
"display_name": display_name,
"description": description,
}
expected_response = datacatalog_pb2.EntryGroup(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
entry_group_id = "entryGroupId-43122680"
response = client.create_entry_group(parent, entry_group_id)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.CreateEntryGroupRequest(
parent=parent, entry_group_id=entry_group_id
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_entry(self):
# Setup Expected Response
name_2 = "name2-1052831874"
linked_resource = "linkedResource1544625012"
user_specified_type = "userSpecifiedType-940364963"
user_specified_system = "userSpecifiedSystem-1776119406"
display_name = "displayName1615086568"
description = "description-1724546052"
expected_response = {
"name": name_2,
"linked_resource": linked_resource,
"user_specified_type": user_specified_type,
"user_specified_system": user_specified_system,
"display_name": display_name,
"description": description,
}
expected_response = datacatalog_pb2.Entry(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
name = client.entry_path("[PROJECT]", "[LOCATION]", "[ENTRY_GROUP]", "[ENTRY]")
response = client.get_entry(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.GetEntryRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_set_iam_policy(self):
# Setup Expected Response
version = 351608024
etag = b"21"
expected_response = {"version": version, "etag": etag}
expected_response = policy_pb2.Policy(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
resource = "resource-341064690"
policy = {}
response = client.set_iam_policy(resource, policy)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = iam_policy_pb2.SetIamPolicyRequest(
resource=resource, policy=policy
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_create_entry(self):
# Setup Expected Response
name = "name3373707"
linked_resource = "linkedResource1544625012"
display_name = "displayName1615086568"
description = "description-1724546052"
expected_response = {
"name": name,
"linked_resource": linked_resource,
"display_name": display_name,
"description": description,
}
expected_response = datacatalog_pb2.Entry(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
parent = client.entry_group_path("[PROJECT]", "[LOCATION]",
"[ENTRY_GROUP]")
entry_id = "entryId-2093663224"
entry = {}
response = client.create_entry(parent, entry_id, entry)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.CreateEntryRequest(
parent=parent, entry_id=entry_id, entry=entry)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_lookup_entry(self):
# Setup Expected Response
name = "name3373707"
linked_resource = "linkedResource1544625012"
user_specified_type = "userSpecifiedType-940364963"
user_specified_system = "userSpecifiedSystem-1776119406"
display_name = "displayName1615086568"
description = "description-1724546052"
expected_response = {
"name": name,
"linked_resource": linked_resource,
"user_specified_type": user_specified_type,
"user_specified_system": user_specified_system,
"display_name": display_name,
"description": description,
}
expected_response = datacatalog_pb2.Entry(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
response = client.lookup_entry()
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.LookupEntryRequest()
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def sample_search_catalog(include_project_id, include_gcp_public_datasets,
query):
"""
Search Catalog
Args:
include_project_id Your Google Cloud project ID.
include_gcp_public_datasets If true, include Google Cloud Platform (GCP) public
datasets in the search results.
query Your query string.
See: https://cloud.google.com/data-catalog/docs/how-to/search-reference
Example: system=bigquery type=dataset
"""
client = datacatalog_v1beta1.DataCatalogClient()
# include_project_id = '[Google Cloud Project ID]'
# include_gcp_public_datasets = False
# query = '[String in search query syntax]'
include_project_ids = [include_project_id]
scope = {
"include_project_ids": include_project_ids,
"include_gcp_public_datasets": include_gcp_public_datasets,
}
# Iterate over all results
for response_item in client.search_catalog(scope, query):
print(u"Result type: {}".format(
enums.SearchResultType(response_item.search_result_type).name))
print(u"Result subtype: {}".format(
response_item.search_result_subtype))
print(u"Relative resource name: {}".format(
response_item.relative_resource_name))
print(u"Linked resource: {}\n".format(response_item.linked_resource))
def test_list_entries(self):
# Setup Expected Response
next_page_token = "nextPageToken-1530815211"
expected_response = {"next_page_token": next_page_token}
expected_response = datacatalog_pb2.ListEntriesResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
parent = client.entry_group_path("[PROJECT]", "[LOCATION]",
"[ENTRY_GROUP]")
response = client.list_entries(parent)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.ListEntriesRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_create_tag_template(self):
# Setup Expected Response
name = "name3373707"
display_name = "displayName1615086568"
expected_response = {"name": name, "display_name": display_name}
expected_response = tags_pb2.TagTemplate(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
tag_template_id = "tagTemplateId-2020335141"
tag_template = {}
response = client.create_tag_template(parent, tag_template_id,
tag_template)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.CreateTagTemplateRequest(
parent=parent,
tag_template_id=tag_template_id,
tag_template=tag_template)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_entry_group(self):
# Setup Expected Response
name_2 = "name2-1052831874"
display_name = "displayName1615086568"
description = "description-1724546052"
expected_response = {
"name": name_2,
"display_name": display_name,
"description": description,
}
expected_response = datacatalog_pb2.EntryGroup(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup Request
name = client.entry_group_path("[PROJECT]", "[LOCATION]",
"[ENTRY_GROUP]")
response = client.get_entry_group(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = datacatalog_pb2.GetEntryGroupRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_lookup_entry_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
with pytest.raises(CustomException):
client.lookup_entry()
def lookup_bigquery_dataset(project_id, dataset_id):
"""Retrieves Data Catalog entry for the given BigQuery Dataset."""
from google.cloud import datacatalog_v1beta1
datacatalog = datacatalog_v1beta1.DataCatalogClient()
resource_name = '//bigquery.googleapis.com/projects/{}/datasets/{}'\
.format(project_id, dataset_id)
return datacatalog.lookup_entry(linked_resource=resource_name)
def lookup_pubsub_topic(project_id, topic_id):
"""Retrieves Data Catalog entry for the given Pub/Sub Topic."""
from google.cloud import datacatalog_v1beta1
datacatalog = datacatalog_v1beta1.DataCatalogClient()
resource_name = '//pubsub.googleapis.com/projects/{}/topics/{}'\
.format(project_id, topic_id)
return datacatalog.lookup_entry(linked_resource=resource_name)
def lookup_bigquery_dataset_sql_resource(project_id, dataset_id):
"""Retrieves Data Catalog entry for the given BigQuery Dataset by
sql_resource.
"""
from google.cloud import datacatalog_v1beta1
datacatalog = datacatalog_v1beta1.DataCatalogClient()
sql_resource = 'bigquery.dataset.`{}`.`{}`'.format(project_id, dataset_id)
return datacatalog.lookup_entry(sql_resource=sql_resource)
def lookup_pubsub_topic_sql_resource(project_id, topic_id):
"""Retrieves Data Catalog entry for the given Pub/Sub Topic by
sql_resource.
"""
from google.cloud import datacatalog_v1beta1
datacatalog = datacatalog_v1beta1.DataCatalogClient()
sql_resource = 'pubsub.topic.`{}`.`{}`'.format(project_id, topic_id)
return datacatalog.lookup_entry(sql_resource=sql_resource)
def test_get_iam_policy_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup request
resource = "resource-341064690"
with pytest.raises(CustomException):
client.get_iam_policy(resource)
def test_update_tag_template_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup request
tag_template = {}
with pytest.raises(CustomException):
client.update_tag_template(tag_template)
def test_get_tag_template_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup request
name = client.tag_template_path("[PROJECT]", "[LOCATION]", "[TAG_TEMPLATE]")
with pytest.raises(CustomException):
client.get_tag_template(name)
def test_list_entry_groups_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = datacatalog_v1beta1.DataCatalogClient()
# Setup request
parent = client.location_path("[PROJECT]", "[LOCATION]")
with pytest.raises(CustomException):
client.list_entry_groups(parent)
