def __init__(self,
project_id: str,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None) -> None:
"""Initialize new instance of BigQueryUtils.
Args:
project_id: GCP project id.
service_account_name: The service account name.
service_account_key_file: File containing service account key. If both
service_account_name and service_account_key_file are not passed the
default credential will be used.There are following ways to create
service accounts - 1) Use `build_service_client` method from
`cloud_auth` module. 2) Use `gcloud` command line utility as documented
here -
https://cloud.google.com/iam/docs/creating-managing-service-account-keys
"""
if service_account_name:
credentials = cloud_auth.impersonate_service_account(
service_account_name)
elif service_account_key_file:
credentials = cloud_auth.get_credentials(service_account_key_file)
else:
logging.info(
'Neither Service account key file nor service account '
'name was provided, so using default credentials.')
credentials = cloud_auth.get_default_credentials()
self.project_id = project_id
self.client = bigquery.Client(project=project_id,
credentials=credentials)
def __init__(self,
project_id: str,
location: str = _LOCATION,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION) -> None:
"""Initialise new instance of CloudComposerUtils.
Args:
project_id: GCP project id.
location: Optional. Region under which the Composer environment needs to
be managed. It defaults to 'us-central1'. Allowed values -
https://cloud.google.com/compute/docs/regions-zones/.
service_account_name: The service account name.
service_account_key_file: Optional. File containing service account key.
version: The version of the service. It defaults to 'v1beta1'.
"""
if service_account_name:
self.client = cloud_auth.build_impersonated_client(
_CLIENT_NAME, service_account_name, version)
else:
if service_account_key_file:
credentials = cloud_auth.get_credentials(
service_account_key_file)
else:
logging.info(
'Neither service account key file nor service account '
'name was provided, so using default credentials.')
credentials = cloud_auth.get_default_credentials()
self.client = cloud_auth.build_service_client(
_CLIENT_NAME, credentials)
self.project_id = project_id
self.location = location
def __init__(self,
project_id: str,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION) -> None:
"""Initialise new instance of CloudApiUtils.
Args:
project_id: GCP project id.
service_account_name: The service account name.
service_account_key_file: Optional. File containing service account key.
version: The version of the service usage service. It defaults to
'v1beta1'.
"""
if service_account_name:
self.client = cloud_auth.build_impersonated_client(
'serviceusage', service_account_name, version)
else:
if service_account_key_file:
credentials = cloud_auth.get_credentials(
service_account_key_file)
else:
logging.info(
'Neither service account key file nor service account '
'name was provided, so using default credentials.')
credentials = cloud_auth.get_default_credentials()
self.client = cloud_auth.build_service_client(
'serviceusage', credentials)
self.project_id = project_id
def __init__(self,
project_id: str,
service_account_info: Mapping[str, str] = None,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None) -> None:
"""Initialize new instance of CloudStorageUtils.
Args:
project_id: GCP project id.
service_account_info: Mapping containing the service account info, such
as the example below:
{
'type': 'service_account',
'project_id': '[PROJECT_ID]',
'private_key_id': '[PRIVATE_KEY_ID]',
'private_key': '[PRIVATE_KEY]',
'client_email': '[CLIENT_EMAIL]',
'client_id': '[CLIENT_ID]',
'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
'token_uri': 'https://accounts.google.com/o/oauth2/token',
'auth_provider_x509_cert_url':
'https://www.googleapis.com/oauth2/v1/certs',
'client_x509_cert_url': '[CERTIFICATE_URL]'
}
service_account_name: The service account name.
service_account_key_file: File containing service account key.
"""
if service_account_name:
credentials = cloud_auth.impersonate_service_account(
service_account_name)
elif service_account_key_file:
credentials = cloud_auth.get_credentials(service_account_key_file)
elif service_account_info:
credentials = cloud_auth.get_credentials_from_info(
service_account_info)
else:
logging.info(
'Neither Service account key file nor service account '
'name was provided, so using default credentials.')
credentials = cloud_auth.get_default_credentials()
self.client = storage.Client(project=project_id,
credentials=credentials)
def visualize_instances(config_file: str) -> None:
"""Visualizes the statistics from the Instance table in BigQuery.
This involves calculating statistics from the Instance table in BigQuery,
generates and outputs plots into a pdf file and uploads the pdf file to
a given location in Cloud Storage.
Args:
config_file: Path to the configuration file.
"""
viz_config = viz_utils.parse_config_file(config_file)
project_id = viz_config['project_id']
dataset = viz_config['dataset']
instance_table = viz_config['instance_table']
instance_table_path = f'{project_id}.{dataset}.{instance_table}'
bq_client = typing.cast(
client.Client,
cloud_auth.build_service_client(
service_name='bigquery',
service_account_credentials=cloud_auth.get_default_credentials()))
storage_client = cloud_storage.CloudStorageUtils(
project_id=viz_config['project_id'],
service_account_key_file=viz_config['service_account_key_file'])
pdf_output = matplotlib.backends.backend_pdf.PdfPages(
viz_config['output_local_path'])
ins_viz_obj = instance_visualizer.InstanceVisualizer(
bq_client=bq_client,
instance_table_path=instance_table_path,
num_instances=viz_config['num_instances'],
label_column=viz_config['label'],
positive_class_label=viz_config['True'],
negative_class_label=viz_config['False'])
ins_viz_obj.plot_instances(**viz_config['plot_style_params'])
pdf_output.savefig()
storage_client.upload_file_to_url(viz_config['output_local_path'],
viz_config['output_gcs_path'])
def visualize_facts(config_file: str) -> None:
"""Visualizes the statistics from the Facts table in BigQuery.
This involves calculating statistics from the Facts table in BigQuery,
generates and outputs plots into a pdf file and uploads the pdf file to
a given location in Cloud Storage.
Args:
config_file: Path to the configuration file.
"""
viz_config = viz_utils.parse_config_file(config_file)
project_id = viz_config['project_id']
dataset = viz_config['dataset']
facts_table = viz_config['facts_table']
facts_table_path = f'{project_id}.{dataset}.{facts_table}'
bq_client = typing.cast(
client.Client,
cloud_auth.build_service_client(
service_name='bigquery',
service_account_credentials=cloud_auth.get_default_credentials()))
storage_client = cloud_storage.CloudStorageUtils(
project_id=viz_config['project_id'],
service_account_key_file=viz_config['service_account_key_file'])
pdf_output = matplotlib.backends.backend_pdf.PdfPages(
viz_config['output_local_path'])
fact_viz_obj = fact_visualizer.FactVisualizer(
bq_client=bq_client,
facts_table_path=facts_table_path,
numerical_facts=viz_config['numerical_fact_list'],
categorical_facts=viz_config['categorical_fact_list'],
number_top_levels=viz_config['number_top_levels'])
fact_viz_obj.plot_facts(**viz_config['plot_style_params'])
pdf_output.savefig()
storage_client.upload_file_to_url(viz_config['output_local_path'],
viz_config['output_gcs_path'])