def CreateAnnotation(taxonomy_resource,
display_name,
description=None,
parent_annotation_id=None):
"""Creates an annotation in the specified taxonomy.
Dy default the annotation is created as a root annotation, but an annotation
can also be created as a child of another annotation by specifying a
parent_annotation.
Args:
taxonomy_resource: The taxonomy container for the new annotation.
display_name: The display name given to the annotation.
description: The annotation description.
parent_annotation_id: The parent annotation, if any.
Returns:
The created Annotation message.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerProjectsTaxonomiesAnnotationsCreateRequest(
parent=taxonomy_resource.RelativeName(),
annotation=messages.Annotation(
displayName=display_name,
description=description,
parentAnnotationId=parent_annotation_id))
return utils.GetClientInstance().projects_taxonomies_annotations.Create(
request=req)
def Apply(data, taxonomy_id, annotation_id):
"""Applies an annotation to a data asset.
Args:
data: Name of the data to be annotated.
taxonomy_id: Id of the policy taxonomy.
annotation_id: Id of the annotation. This annotation must belong to
the policy taxonomy specified in taxonomy_name.
Returns:
An AnnotationTag message.
"""
client = utils.GetClientInstance().data_taxonomyStores_dataTaxonomies
messages = utils.GetMessagesModule()
name = resources.REGISTRY.Create(
'datapol.data.taxonomy',
data=data,
taxonomyStoresId=utils.GetTaxonomyStoresId(),
dataTaxonomiesId=taxonomy_id).RelativeName()
try:
return client.ApplyAnnotationTag(
messages.
DatapolDataTaxonomyStoresDataTaxonomiesApplyAnnotationTagRequest(
name=name,
applyAnnotationTagRequest=messages.ApplyAnnotationTagRequest(
annotationId=annotation_id)))
except apitools_exceptions.HttpError as e:
raise utils.ErrorWrapper(e, name)
def ListDataAssets(annotations=None,
include_annotated_by_group=False,
annotatable_only=True,
filter_exp='',
limit=None):
"""Lists resource names of all data assets with the given annotations.
Args:
annotations: A list of annotations. Each returned data asset will be tagged
with at one of those annotations.
include_annotated_by_group: If true, and a given annotation has child
annotations, also returns data assets that are annotated with those child
annotations.
annotatable_only: If true, only returns data assets that are annotatable by
the caller.
filter_exp: A expression to further filter data assets
limit: The number of resource names to limit the resutls to.
Returns:
Generator that yields resource names of data assets.
"""
request = utils.GetMessagesModule().DatapolDataAssetsListResourceNamesRequest(
annotations=annotations or [],
includeAnnotatedByGroup=include_annotated_by_group,
annotatableOnly=annotatable_only,
filter=filter_exp)
return list_pager.YieldFromList(
utils.GetClientInstance().dataAssets,
request,
limit=limit,
method='ListResourceNames',
field='dataAssets',
batch_size_attribute='pageSize')
def GetTaxonomyStoreFromOrgResource(org_resource):
"""Gets a taxonomy store from an organization resource.
Args:
org_resource: A cloudresourcemanager.organizations core.Resource object.
Returns:
A TaxonomyStore message.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerOrganizationsGetTaxonomyStoreRequest(
parent=org_resource.RelativeName())
return utils.GetClientInstance().organizations.GetTaxonomyStore(request=req)
def DeleteTaxonomy(taxonomy_resource):
"""Deletes a taxonomy resource.
Args:
taxonomy_resource: The resource path of the taxonomy to delete.
Returns:
An empty message.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerProjectsTaxonomiesDeleteRequest(
name=taxonomy_resource.RelativeName())
return utils.GetClientInstance().projects_taxonomies.Delete(request=req)
def ListTaxonomies(project_resource):
"""Lists all taxonomies in a project resource.
Args:
project_resource: The project resource container of the taxonomies to list.
Returns:
A list of taxonomy messages.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerProjectsTaxonomiesListRequest(
parent=project_resource.RelativeName())
return utils.GetClientInstance().projects_taxonomies.List(request=req)
def ListAssetAnnotationTags(asset_resource):
"""Gets all annotation tags associated with an asset.
Args:
asset_resource: A category_manager.assets core.Resource asset object.
Returns:
A ListAnnotationTagsResponse message.
"""
messages = utils.GetMessagesModule()
# Set url_escape=True because the resource name of the asset must be escaped.
req = messages.CategorymanagerAssetsAnnotationTagsListRequest(
name=asset_resource.RelativeName(url_escape=True))
return utils.GetClientInstance().assets_annotationTags.List(request=req)
def GetIamPolicy(store_ref):
"""Gets IAM policy for a given taxonomy store.
Args:
store_ref: a taxonomy store resource reference.
Returns:
An IamPolicy message.
"""
messages = utils.GetMessagesModule()
return utils.GetClientInstance().taxonomyStores.GetIamPolicy(
messages.CategorymanagerTaxonomyStoresGetIamPolicyRequest(
resource=store_ref.RelativeName(),
getIamPolicyRequest=messages.GetIamPolicyRequest()))
def SetIamPolicy(store_ref, policy):
"""Sets IAM policy on a taxonomy store.
Args:
store_ref: a taxonomy store resource reference.
policy: An IamPolicy message.
Returns:
An IamPolicy message.
"""
messages = utils.GetMessagesModule()
return utils.GetClientInstance().taxonomyStores.SetIamPolicy(
messages.CategorymanagerTaxonomyStoresSetIamPolicyRequest(
resource=store_ref.RelativeName(),
setIamPolicyRequest=messages.SetIamPolicyRequest(policy=policy)))
def GetIamPolicy(taxonomy_store_resource):
"""Gets IAM policy for a given taxonomy store.
Args:
taxonomy_store_resource: A categorymanager.taxonomyStores core.Resource
object.
Returns:
An IamPolicy message.
"""
messages = utils.GetMessagesModule()
return utils.GetClientInstance().taxonomyStores.GetIamPolicy(
messages.CategorymanagerTaxonomyStoresGetIamPolicyRequest(
resource=taxonomy_store_resource.RelativeName(),
getIamPolicyRequest=messages.GetIamPolicyRequest()))
def UpdateTaxonomy(taxonomy_resource, description):
"""Updates the description of a taxonomy.
Args:
taxonomy_resource: A category_manager.taxonomies core.Resource object.
description: A string representing the new taxonomy description.
Returns:
A Taxonomy message.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerProjectsTaxonomiesPatchRequest(
name=taxonomy_resource.RelativeName(),
taxonomy=messages.Taxonomy(description=description))
return utils.GetClientInstance().projects_taxonomies.Patch(request=req)
def SetIamPolicy(taxonomy_store_resource, policy):
"""Sets IAM policy on a taxonomy store.
Args:
taxonomy_store_resource: A categorymanager.taxonomyStores core.Resource
object.
policy: An IamPolicy message.
Returns:
An IamPolicy message.
"""
messages = utils.GetMessagesModule()
return utils.GetClientInstance().taxonomyStores.SetIamPolicy(
messages.CategorymanagerTaxonomyStoresSetIamPolicyRequest(
resource=taxonomy_store_resource.RelativeName(),
setIamPolicyRequest=messages.SetIamPolicyRequest(policy=policy)))
def UpdateAnnotation(annotation_resource, description):
"""Updates the description of an annotation.
Args:
annotation_resource: A category_manager.taxonomies.annotations
core.Resource object.
description: A string representing the new annotation description.
Returns:
An Annotation message.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerProjectsTaxonomiesAnnotationsPatchRequest(
name=annotation_resource.RelativeName(),
annotation=messages.Annotation(description=description))
return utils.GetClientInstance().projects_taxonomies_annotations.Patch(
request=req)
def ApplyAnnotationTag(asset_ref, annotation_ref, sub_asset=None):
"""Applies an annotation tag to an asset.
Args:
asset_ref: An asset reference object.
annotation_ref: An annotation reference object.
sub_asset: A string representing the asset's sub-asset, if any.
Returns:
AnnotationTag response message.
"""
messages = utils.GetMessagesModule()
# Set url_escape=True because the resource name of the asset must be escaped.
req = messages.CategorymanagerAssetsApplyAnnotationTagRequest(
name=asset_ref.RelativeName(url_escape=True),
applyAnnotationTagRequest=messages.ApplyAnnotationTagRequest(
annotation=annotation_ref.RelativeName(), subAsset=sub_asset))
return utils.GetClientInstance().assets.ApplyAnnotationTag(request=req)
def CreateTaxonomy(project_resource, display_name, description=None):
"""Creates a taxonomy in the project resource container.
Args:
project_resource: The project resource representing the project to create
the taxonomy in.
display_name: The display name given to the taxonomy.
description: The taxonomy description.
Returns:
The created Taxonomy message.
"""
messages = utils.GetMessagesModule()
req = messages.CategorymanagerProjectsTaxonomiesCreateRequest(
parent=project_resource.RelativeName(),
taxonomy=messages.Taxonomy(displayName=display_name,
description=description))
return utils.GetClientInstance().projects_taxonomies.Create(request=req)
def DeleteAnnotationTag(asset_ref, annotation_ref, sub_asset=None):
"""Delete an annotation tag on an asset.
Args:
asset_ref: An asset reference object.
annotation_ref: An annotation reference object.
sub_asset: A string representing the asset's sub-asset, if any.
Returns:
DeleteAnnotationTag response message.
"""
messages = utils.GetMessagesModule()
# Set url_escape=True because the resource name of the asset must be escaped.
req = messages.CategorymanagerAssetsDeleteAnnotationTagRequest(
name=DELETE_TAG_NAME_PATTERN.format(
asset_ref.RelativeName(url_escape=True)),
annotation=annotation_ref.RelativeName(),
subAsset=sub_asset)
return utils.GetClientInstance().assets.DeleteAnnotationTag(request=req)
def ListTags(data, limit=None):
"""Lists all annotation tags on a data asset.
Args:
data: Name of the data.
limit: The number of annotation tags to limit the resutls to.
Returns:
Generator that yields annnotation tags.
"""
request = utils.GetMessagesModule(
).DatapolDataTaxonomyStoresAnnotationTagsListRequest(
parent='data/{data}'.format(data=data))
return list_pager.YieldFromList(
utils.GetClientInstance().data_taxonomyStores_annotationTags,
request,
limit=limit,
field='tags',
batch_size_attribute='pageSize')
def Delete(data, taxonomy_id):
"""Deletes an annotation on a data asset.
Args:
data: Name of the data to be annotated.
taxonomy_id: Id of the policy taxonomy.
Returns:
An Empty message.
"""
client = utils.GetClientInstance().data_taxonomyStores_dataTaxonomies
name = resources.REGISTRY.Create(
'datapol.data.tag',
data=data,
taxonomyStoresId=utils.GetTaxonomyStoresId(),
dataTaxonomiesId=taxonomy_id).RelativeName()
try:
return client.DeleteAnnotationTag(
utils.GetMessagesModule()
.DatapolDataTaxonomyStoresDataTaxonomiesDeleteAnnotationTagRequest(
name=name))
except apitools_exceptions.HttpError as e:
raise utils.ErrorWrapper(e, name)
def _GetService():
"""Gets the data policy taxonomiy service."""
return utils.GetClientInstance().taxonomyStores_dataTaxonomies
def GetCommonStore():
"""Gets the common taxonomy store."""
messages = utils.GetMessagesModule()
return utils.GetClientInstance().taxonomyStores.GetCommon(
messages.CategorymanagerTaxonomyStoresGetCommonRequest())
def SearchAssets(annotations, show_only_annotatable, match_child_annotations,
query_filter, page_size, limit):
"""Performs backend call to search for assets given a set of constraints.
Args:
annotations: Array of annotation strings of the annotations to be looked up.
show_only_annotatable: A boolean indicating whether or not to exclude
assets that are not annotatable.
match_child_annotations: A boolean value which if set to true
indicates that for any annotation with child annotations, also list assets
that are annotated by those child annotations.
query_filter: A filter string that includes additional predicates for assets
page_size: The request page size.
limit: The maximum number of assets returned.
Yields:
A generator of Asset objects matching the given set of constraints.
Raises:
HttpRequestFailError: An HTTP request error if backend call fails.
MessageDecodeError: An error indicating that the received server payload
could not be decoded into a valid response.
Notes:
This method is doing the HTTP request to search assets manually because the
generated python apitools API does not support '.' characters in the query
params, see b/31244944.
Furthermore, this method does not support multiple retries on failure. The
issue with implementing retries appears to be that using a generator saves
the function's state and prevents resetting the generator to enable the
function to be called again.
"""
asset_limit = limit or _MAX_ASSET_LIMIT
# A page size of None will use the default page size set by the server.
if page_size is not None:
page_size = min(page_size, asset_limit)
query_params = [
('query.filter', query_filter), # pylint: disable=ugly-g4-fix-formatting
('query.annotatable_only', show_only_annotatable),
('query.include_annotated_by_group', match_child_annotations),
('pageSize', page_size),
]
for annotation in annotations:
query_params.append(('query.annotations', annotation))
# Filter away query params which have not been specified.
query_params = [(k, v) for k, v in query_params if v is not None]
base_url = utils.GetClientInstance().BASE_URL + utils.API_VERSION + '/'
endpoint_base_url = resources.GetApiBaseUrl(utils.API_NAME, utils.API_VERSION)
# Override the base url if another endpoint is explicitly set.
if endpoint_base_url is not None:
base_url = endpoint_base_url
search_response_class = utils.GetMessagesModule().SearchAssetsResponse
while asset_limit > 0:
url = _SEARCH_NAME_FORMAT.format(
base_url, six.moves.urllib.parse.urlencode(query_params))
response, raw_content = http.Http().request(uri=url, headers=_HEADERS)
content = core_encoding.Decode(raw_content)
status_code = response['status']
if status_code != '200':
msg = _HTTP_ERROR_FORMAT.format(status_code, content)
raise exceptions.HttpException(msg)
try:
search_response = encoding.JsonToMessage(search_response_class, content)
except ValueError as e:
err_msg = ('Failed receiving proper response from server, cannot'
'parse received assets. Error details: ' + str(e))
raise MessageDecodeError(err_msg)
for asset in search_response.assets:
yield asset
next_token = getattr(search_response, 'nextPageToken', None)
if next_token is None:
return
_AddPageTokenQueryParam(query_params, next_token)
asset_limit -= len(search_response.assets)
def _GetService():
"""Gets the data policy annotation service."""
return utils.GetClientInstance().taxonomyStores
