def inspect_bigquery(
project,
bigquery_project,
dataset_id,
table_id,
topic_id,
subscription_id,
info_types,
custom_dictionaries=None,
custom_regexes=None,
min_likelihood=None,
max_findings=None,
timeout=300,
):
"""Uses the Data Loss Prevention API to analyze BigQuery data.
Args:
project: The Google Cloud project id to use as a parent resource.
bigquery_project: The Google Cloud project id of the target table.
dataset_id: The id of the target BigQuery dataset.
table_id: The id of the target BigQuery table.
topic_id: The id of the Cloud Pub/Sub topic to which the API will
broadcast job completion. The topic must already exist.
subscription_id: The id of the Cloud Pub/Sub subscription to listen on
while waiting for job completion. The subscription must already
exist and be subscribed to the topic.
info_types: A list of strings representing info types to look for.
A full list of info type categories can be fetched from the API.
namespace_id: The namespace of the Datastore document, if applicable.
min_likelihood: A string representing the minimum likelihood threshold
that constitutes a match. One of: 'LIKELIHOOD_UNSPECIFIED',
'VERY_UNLIKELY', 'UNLIKELY', 'POSSIBLE', 'LIKELY', 'VERY_LIKELY'.
max_findings: The maximum number of findings to report; 0 = no maximum.
timeout: The number of seconds to wait for a response from the API.
Returns:
None; the response from the API is printed to the terminal.
"""
# Import the client library.
import google.cloud.dlp
# This sample additionally uses Cloud Pub/Sub to receive results from
# potentially long-running operations.
import google.cloud.pubsub
# This sample also uses threading.Event() to wait for the job to finish.
import threading
# Instantiate a client.
dlp = google.cloud.dlp_v2.DlpServiceClient()
# Prepare info_types by converting the list of strings into a list of
# dictionaries (protos are also accepted).
if not info_types:
info_types = ["FIRST_NAME", "LAST_NAME", "EMAIL_ADDRESS"]
info_types = [{"name": info_type} for info_type in info_types]
# Prepare custom_info_types by parsing the dictionary word lists and
# regex patterns.
if custom_dictionaries is None:
custom_dictionaries = []
dictionaries = [
{
"info_type": {"name": "CUSTOM_DICTIONARY_{}".format(i)},
"dictionary": {"word_list": {"words": custom_dict.split(",")}},
}
for i, custom_dict in enumerate(custom_dictionaries)
]
if custom_regexes is None:
custom_regexes = []
regexes = [
{
"info_type": {"name": "CUSTOM_REGEX_{}".format(i)},
"regex": {"pattern": custom_regex},
}
for i, custom_regex in enumerate(custom_regexes)
]
custom_info_types = dictionaries + regexes
# Construct the configuration dictionary. Keys which are None may
# optionally be omitted entirely.
inspect_config = {
"info_types": info_types,
"custom_info_types": custom_info_types,
"min_likelihood": min_likelihood,
"limits": {"max_findings_per_request": max_findings},
}
# Construct a storage_config containing the target Bigquery info.
storage_config = {
"big_query_options": {
"table_reference": {
"project_id": bigquery_project,
"dataset_id": dataset_id,
"table_id": table_id,
}
}
}
# Convert the project id into full resource ids.
topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
parent = dlp.location_path(project, 'global')
# Tell the API where to send a notification when the job is complete.
actions = [{"pub_sub": {"topic": topic}}]
# Construct the inspect_job, which defines the entire inspect content task.
inspect_job = {
"inspect_config": inspect_config,
"storage_config": storage_config,
"actions": actions,
}
operation = dlp.create_dlp_job(parent, inspect_job=inspect_job)
print("Inspection operation started: {}".format(operation.name))
# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
subscriber = google.cloud.pubsub.SubscriberClient()
subscription_path = subscriber.subscription_path(project, subscription_id)
# Set up a callback to acknowledge a message. This closes around an event
# so that it can signal that it is done and the main thread can continue.
job_done = threading.Event()
def callback(message):
try:
if message.attributes["DlpJobName"] == operation.name:
# This is the message we're looking for, so acknowledge it.
message.ack()
# Now that the job is done, fetch the results and print them.
job = dlp.get_dlp_job(operation.name)
if job.inspect_details.result.info_type_stats:
for finding in job.inspect_details.result.info_type_stats:
print(
"Info type: {}; Count: {}".format(
finding.info_type.name, finding.count
)
)
else:
print("No findings.")
# Signal to the main thread that we can exit.
job_done.set()
else:
# This is not the message we're looking for.
message.drop()
except Exception as e:
# Because this is executing in a thread, an exception won't be
# noted unless we print it manually.
print(e)
raise
# Register the callback and wait on the event.
subscriber.subscribe(subscription_path, callback=callback)
finished = job_done.wait(timeout=timeout)
if not finished:
print(
"No event received before the timeout. Please verify that the "
"subscription provided is subscribed to the topic provided."
)
def l_diversity_analysis(
project,
table_project_id,
dataset_id,
table_id,
topic_id,
subscription_id,
sensitive_attribute,
quasi_ids,
timeout=300,
):
"""Uses the Data Loss Prevention API to compute the l-diversity of a
column set in a Google BigQuery table.
Args:
project: The Google Cloud project id to use as a parent resource.
table_project_id: The Google Cloud project id where the BigQuery table
is stored.
dataset_id: The id of the dataset to inspect.
table_id: The id of the table to inspect.
topic_id: The name of the Pub/Sub topic to notify once the job
completes.
subscription_id: The name of the Pub/Sub subscription to use when
listening for job completion notifications.
sensitive_attribute: The column to measure l-diversity relative to.
quasi_ids: A set of columns that form a composite key.
timeout: The number of seconds to wait for a response from the API.
Returns:
None; the response from the API is printed to the terminal.
"""
# Import the client library.
import google.cloud.dlp
# This sample additionally uses Cloud Pub/Sub to receive results from
# potentially long-running operations.
import google.cloud.pubsub
# Create helper function for unpacking values
def get_values(obj):
return int(obj.integer_value)
# Instantiate a client.
dlp = google.cloud.dlp_v2.DlpServiceClient()
# Convert the project id into a full resource id.
topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
parent = dlp.location_path(project, 'global')
# Location info of the BigQuery table.
source_table = {
"project_id": table_project_id,
"dataset_id": dataset_id,
"table_id": table_id,
}
# Convert quasi id list to Protobuf type
def map_fields(field):
return {"name": field}
quasi_ids = map(map_fields, quasi_ids)
# Tell the API where to send a notification when the job is complete.
actions = [{"pub_sub": {"topic": topic}}]
# Configure risk analysis job
# Give the name of the numeric column to compute risk metrics for
risk_job = {
"privacy_metric": {
"l_diversity_config": {
"quasi_ids": quasi_ids,
"sensitive_attribute": {
"name": sensitive_attribute
},
}
},
"source_table": source_table,
"actions": actions,
}
# Call API to start risk analysis job
operation = dlp.create_dlp_job(parent, risk_job=risk_job)
def callback(message):
if message.attributes["DlpJobName"] == operation.name:
# This is the message we're looking for, so acknowledge it.
message.ack()
# Now that the job is done, fetch the results and print them.
job = dlp.get_dlp_job(operation.name)
histogram_buckets = (
job.risk_details.l_diversity_result.
sensitive_value_frequency_histogram_buckets # noqa: E501
)
# Print bucket stats
for i, bucket in enumerate(histogram_buckets):
print("Bucket {}:".format(i))
print(" Bucket size range: [{}, {}]".format(
bucket.sensitive_value_frequency_lower_bound,
bucket.sensitive_value_frequency_upper_bound,
))
for value_bucket in bucket.bucket_values:
print(" Quasi-ID values: {}".format(
map(get_values, value_bucket.quasi_ids_values)))
print(" Class size: {}".format(
value_bucket.equivalence_class_size))
for value in value_bucket.top_sensitive_values:
print(
(" Sensitive value {} occurs {} time(s)".format(
value.value, value.count)))
subscription.set_result(None)
else:
# This is not the message we're looking for.
message.drop()
# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
subscriber = google.cloud.pubsub.SubscriberClient()
subscription_path = subscriber.subscription_path(project, subscription_id)
subscription = subscriber.subscribe(subscription_path, callback)
try:
subscription.result(timeout=timeout)
except TimeoutError:
print("No event received before the timeout. Please verify that the "
"subscription provided is subscribed to the topic provided.")
subscription.close()
def k_map_estimate_analysis(
project,
table_project_id,
dataset_id,
table_id,
topic_id,
subscription_id,
quasi_ids,
info_types,
region_code="US",
timeout=300,
):
"""Uses the Data Loss Prevention API to compute the k-map risk estimation
of a column set in a Google BigQuery table.
Args:
project: The Google Cloud project id to use as a parent resource.
table_project_id: The Google Cloud project id where the BigQuery table
is stored.
dataset_id: The id of the dataset to inspect.
table_id: The id of the table to inspect.
column_name: The name of the column to compute risk metrics for.
topic_id: The name of the Pub/Sub topic to notify once the job
completes.
subscription_id: The name of the Pub/Sub subscription to use when
listening for job completion notifications.
quasi_ids: A set of columns that form a composite key and optionally
their reidentification distributions.
info_types: Type of information of the quasi_id in order to provide a
statistical model of population.
region_code: The ISO 3166-1 region code that the data is representative
of. Can be omitted if using a region-specific infoType (such as
US_ZIP_5)
timeout: The number of seconds to wait for a response from the API.
Returns:
None; the response from the API is printed to the terminal.
"""
# Import the client library.
import google.cloud.dlp
# This sample additionally uses Cloud Pub/Sub to receive results from
# potentially long-running operations.
import google.cloud.pubsub
# Create helper function for unpacking values
def get_values(obj):
return int(obj.integer_value)
# Instantiate a client.
dlp = google.cloud.dlp_v2.DlpServiceClient()
# Convert the project id into full resource ids.
topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
parent = dlp.location_path(project, 'global')
# Location info of the BigQuery table.
source_table = {
"project_id": table_project_id,
"dataset_id": dataset_id,
"table_id": table_id,
}
# Check that numbers of quasi-ids and info types are equal
if len(quasi_ids) != len(info_types):
raise ValueError("""Number of infoTypes and number of quasi-identifiers
must be equal!""")
# Convert quasi id list to Protobuf type
def map_fields(quasi_id, info_type):
return {"field": {"name": quasi_id}, "info_type": {"name": info_type}}
quasi_ids = map(map_fields, quasi_ids, info_types)
# Tell the API where to send a notification when the job is complete.
actions = [{"pub_sub": {"topic": topic}}]
# Configure risk analysis job
# Give the name of the numeric column to compute risk metrics for
risk_job = {
"privacy_metric": {
"k_map_estimation_config": {
"quasi_ids": quasi_ids,
"region_code": region_code,
}
},
"source_table": source_table,
"actions": actions,
}
# Call API to start risk analysis job
operation = dlp.create_dlp_job(parent, risk_job=risk_job)
def callback(message):
if message.attributes["DlpJobName"] == operation.name:
# This is the message we're looking for, so acknowledge it.
message.ack()
# Now that the job is done, fetch the results and print them.
job = dlp.get_dlp_job(operation.name)
histogram_buckets = (job.risk_details.k_map_estimation_result.
k_map_estimation_histogram)
# Print bucket stats
for i, bucket in enumerate(histogram_buckets):
print("Bucket {}:".format(i))
print(" Anonymity range: [{}, {}]".format(
bucket.min_anonymity, bucket.max_anonymity))
print(" Size: {}".format(bucket.bucket_size))
for value_bucket in bucket.bucket_values:
print(" Values: {}".format(
map(get_values, value_bucket.quasi_ids_values)))
print(" Estimated k-map anonymity: {}".format(
value_bucket.estimated_anonymity))
subscription.set_result(None)
else:
# This is not the message we're looking for.
message.drop()
# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
subscriber = google.cloud.pubsub.SubscriberClient()
subscription_path = subscriber.subscription_path(project, subscription_id)
subscription = subscriber.subscribe(subscription_path, callback)
try:
subscription.result(timeout=timeout)
except TimeoutError:
print("No event received before the timeout. Please verify that the "
"subscription provided is subscribed to the topic provided.")
subscription.close()
def numerical_risk_analysis(
project,
table_project_id,
dataset_id,
table_id,
column_name,
topic_id,
subscription_id,
timeout=300,
):
"""Uses the Data Loss Prevention API to compute risk metrics of a column
of numerical data in a Google BigQuery table.
Args:
project: The Google Cloud project id to use as a parent resource.
table_project_id: The Google Cloud project id where the BigQuery table
is stored.
dataset_id: The id of the dataset to inspect.
table_id: The id of the table to inspect.
column_name: The name of the column to compute risk metrics for.
topic_id: The name of the Pub/Sub topic to notify once the job
completes.
subscription_id: The name of the Pub/Sub subscription to use when
listening for job completion notifications.
timeout: The number of seconds to wait for a response from the API.
Returns:
None; the response from the API is printed to the terminal.
"""
# Import the client library.
import google.cloud.dlp
# This sample additionally uses Cloud Pub/Sub to receive results from
# potentially long-running operations.
import google.cloud.pubsub
# Instantiate a client.
dlp = google.cloud.dlp_v2.DlpServiceClient()
# Convert the project id into full resource ids.
topic = google.cloud.pubsub.PublisherClient.topic_path(project, topic_id)
parent = dlp.location_path(project, 'global')
# Location info of the BigQuery table.
source_table = {
"project_id": table_project_id,
"dataset_id": dataset_id,
"table_id": table_id,
}
# Tell the API where to send a notification when the job is complete.
actions = [{"pub_sub": {"topic": topic}}]
# Configure risk analysis job
# Give the name of the numeric column to compute risk metrics for
risk_job = {
"privacy_metric": {
"numerical_stats_config": {
"field": {
"name": column_name
}
}
},
"source_table": source_table,
"actions": actions,
}
# Call API to start risk analysis job
operation = dlp.create_dlp_job(parent, risk_job=risk_job)
def callback(message):
if message.attributes["DlpJobName"] == operation.name:
# This is the message we're looking for, so acknowledge it.
message.ack()
# Now that the job is done, fetch the results and print them.
job = dlp.get_dlp_job(operation.name)
results = job.risk_details.numerical_stats_result
print("Value Range: [{}, {}]".format(
results.min_value.integer_value,
results.max_value.integer_value,
))
prev_value = None
for percent, result in enumerate(results.quantile_values):
value = result.integer_value
if prev_value != value:
print("Value at {}% quantile: {}".format(percent, value))
prev_value = value
subscription.set_result(None)
else:
# This is not the message we're looking for.
message.drop()
# Create a Pub/Sub client and find the subscription. The subscription is
# expected to already be listening to the topic.
subscriber = google.cloud.pubsub.SubscriberClient()
subscription_path = subscriber.subscription_path(project, subscription_id)
subscription = subscriber.subscribe(subscription_path, callback)
try:
subscription.result(timeout=timeout)
except TimeoutError:
print("No event received before the timeout. Please verify that the "
"subscription provided is subscribed to the topic provided.")
subscription.close()