def test_build_service_client(self):
cloud_auth.build_service_client('service_name', self.mock_credentials)
self.mock_client.assert_called_once_with(
'service_name',
'v1',
credentials=self.mock_credentials,
cache_discovery=False)
def __init__(self,
project_id: str,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION) -> None:
"""Initialise new instance of CloudApiUtils.
Args:
project_id: GCP project id.
service_account_name: The service account name. If provided, the service
account will be impersonated to acquire credentials.
service_account_key_file: Optional. File containing service account key.
If both service_account_name and not service_account_key_file are not
passed the default credential will be used. There are following ways to
create service accounts:
1. Use `create_service_account_key` method from `cloud_auth` module.
2. Use `gcloud` command line utility as documented here -
https://cloud.google.com/iam/docs/creating-managing-service-account-keys
version: The version of the service usage service. It defaults to
'v1beta1'.
"""
if service_account_name:
self.client = cloud_auth.build_impersonated_client(
'serviceusage', service_account_name, version)
else:
if not service_account_key_file:
logging.info(
'Neither Service account key file nor servie account name '
'was provided. So using default credentials.')
self.client = cloud_auth.build_service_client(
'serviceusage', service_account_key_file)
self.project_id = project_id
def __init__(self,
project_id: str,
location: str = _LOCATION,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION) -> None:
"""Initialise new instance of CloudComposerUtils.
Args:
project_id: GCP project id.
location: Optional. Region under which the Composer environment needs to
be managed. It defaults to 'us-central1'. Allowed values -
https://cloud.google.com/compute/docs/regions-zones/.
service_account_name: The service account name.
service_account_key_file: Optional. File containing service account key.
version: The version of the service. It defaults to 'v1beta1'.
"""
if service_account_name:
self.client = cloud_auth.build_impersonated_client(
_CLIENT_NAME, service_account_name, version)
else:
if service_account_key_file:
credentials = cloud_auth.get_credentials(
service_account_key_file)
else:
logging.info(
'Neither service account key file nor service account '
'name was provided, so using default credentials.')
credentials = cloud_auth.get_default_credentials()
self.client = cloud_auth.build_service_client(
_CLIENT_NAME, credentials)
self.project_id = project_id
self.location = location
def __init__(self,
project_id: str,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION) -> None:
"""Initialise new instance of CloudApiUtils.
Args:
project_id: GCP project id.
service_account_name: The service account name.
service_account_key_file: Optional. File containing service account key.
version: The version of the service usage service. It defaults to
'v1beta1'.
"""
if service_account_name:
self.client = cloud_auth.build_impersonated_client(
'serviceusage', service_account_name, version)
else:
if service_account_key_file:
credentials = cloud_auth.get_credentials(
service_account_key_file)
else:
logging.info(
'Neither service account key file nor service account '
'name was provided, so using default credentials.')
credentials = cloud_auth.get_default_credentials()
self.client = cloud_auth.build_service_client(
'serviceusage', credentials)
self.project_id = project_id
def __init__(self,
project_id: str,
service_account_key_file: str = None) -> None:
"""Initialise new instance of CloudApiUtils.
Args:
project_id: GCP project id.
service_account_key_file: Optional. File containing service account key.
If not passed the default credential will be used. There are following
ways to create service accounts:
1. Use `create_service_account_key` method from `cloud_auth` module.
2. Use `gcloud` command line utility as documented here -
https://cloud.google.com/iam/docs/creating-managing-service-account-keys
"""
self.client = cloud_auth.build_service_client(
'serviceusage', service_account_key_file)
self.project_id = project_id
def __init__(self,
project_id: str,
location: str = _LOCATION,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION):
"""Initializes new instance of CloudSchedulerUtils.
Args:
project_id: GCP project id.
location: Optional. Region under which the Cloud Scheduler needs to be
managed. It defaults to 'us-central1'. Allowed values -
https://cloud.google.com/compute/docs/regions-zones/.
service_account_name: The service account name.
service_account_key_file: Optional. File containing service account key.
If not passed the default credential will be used. There are following
ways to create service accounts: 1. Use `build_service_client` method
from `cloud_auth` module. 2. Use `gcloud` command line utility as
documented here -
https://cloud.google.com/iam/docs/creating-managing-service-account-keys
version: The version of the service. It defaults to 'v1beta1'.
Raises:
ValueError: If neither service_account_key_file or service_account_name
were provided.
"""
if not service_account_key_file and not service_account_name:
raise ValueError(
'Service account key file or service account name is not provided. '
'Provide either path to service account key file or name of the '
'service account.')
if service_account_key_file:
credentials = cloud_auth.get_credentials(service_account_key_file)
self._client = cloud_auth.build_service_client(
_CLIENT_NAME, credentials)
else:
self._client = cloud_auth.build_impersonated_client(
_CLIENT_NAME, service_account_name, version)
self._parent = f'projects/{project_id}/locations/{location}'
def __init__(self,
project_id: str,
location: str = _LOCATION,
service_account_key_file: str = None) -> None:
"""Initialise new instance of CloudComposerUtils.
Args:
project_id: GCP project id.
location: Optional. Region under which the Composer environment needs to
be managed. It defaults to 'us-central1'. Allowed values -
https://cloud.google.com/compute/docs/regions-zones/.
service_account_key_file: Optional. File containing service account key.
If not passed the default credential will be used. There are following
ways to create service accounts:
1. Use `build_service_client` method from `cloud_auth` module.
2. Use `gcloud` command line utility as documented here -
https://cloud.google.com/iam/docs/creating-managing-service-account-keys
"""
self.client = cloud_auth.build_service_client(
'composer', service_account_key_file)
self.project_id = project_id
self.location = location
def visualize_instances(config_file: str) -> None:
"""Visualizes the statistics from the Instance table in BigQuery.
This involves calculating statistics from the Instance table in BigQuery,
generates and outputs plots into a pdf file and uploads the pdf file to
a given location in Cloud Storage.
Args:
config_file: Path to the configuration file.
"""
viz_config = viz_utils.parse_config_file(config_file)
project_id = viz_config['project_id']
dataset = viz_config['dataset']
instance_table = viz_config['instance_table']
instance_table_path = f'{project_id}.{dataset}.{instance_table}'
bq_client = typing.cast(
client.Client,
cloud_auth.build_service_client(
service_name='bigquery',
service_account_credentials=cloud_auth.get_default_credentials()))
storage_client = cloud_storage.CloudStorageUtils(
project_id=viz_config['project_id'],
service_account_key_file=viz_config['service_account_key_file'])
pdf_output = matplotlib.backends.backend_pdf.PdfPages(
viz_config['output_local_path'])
ins_viz_obj = instance_visualizer.InstanceVisualizer(
bq_client=bq_client,
instance_table_path=instance_table_path,
num_instances=viz_config['num_instances'],
label_column=viz_config['label'],
positive_class_label=viz_config['True'],
negative_class_label=viz_config['False'])
ins_viz_obj.plot_instances(**viz_config['plot_style_params'])
pdf_output.savefig()
storage_client.upload_file_to_url(viz_config['output_local_path'],
viz_config['output_gcs_path'])
def visualize_facts(config_file: str) -> None:
"""Visualizes the statistics from the Facts table in BigQuery.
This involves calculating statistics from the Facts table in BigQuery,
generates and outputs plots into a pdf file and uploads the pdf file to
a given location in Cloud Storage.
Args:
config_file: Path to the configuration file.
"""
viz_config = viz_utils.parse_config_file(config_file)
project_id = viz_config['project_id']
dataset = viz_config['dataset']
facts_table = viz_config['facts_table']
facts_table_path = f'{project_id}.{dataset}.{facts_table}'
bq_client = typing.cast(
client.Client,
cloud_auth.build_service_client(
service_name='bigquery',
service_account_credentials=cloud_auth.get_default_credentials()))
storage_client = cloud_storage.CloudStorageUtils(
project_id=viz_config['project_id'],
service_account_key_file=viz_config['service_account_key_file'])
pdf_output = matplotlib.backends.backend_pdf.PdfPages(
viz_config['output_local_path'])
fact_viz_obj = fact_visualizer.FactVisualizer(
bq_client=bq_client,
facts_table_path=facts_table_path,
numerical_facts=viz_config['numerical_fact_list'],
categorical_facts=viz_config['categorical_fact_list'],
number_top_levels=viz_config['number_top_levels'])
fact_viz_obj.plot_facts(**viz_config['plot_style_params'])
pdf_output.savefig()
storage_client.upload_file_to_url(viz_config['output_local_path'],
viz_config['output_gcs_path'])
def __init__(self,
project_id: str,
location: str = _LOCATION,
service_account_name: Optional[str] = None,
service_account_key_file: Optional[str] = None,
version: str = _VERSION) -> None:
"""Initialise new instance of CloudComposerUtils.
Args:
project_id: GCP project id.
location: Optional. Region under which the Composer environment needs to
be managed. It defaults to 'us-central1'. Allowed values -
https://cloud.google.com/compute/docs/regions-zones/.
service_account_name: The service account name.
service_account_key_file: Optional. File containing service account key.
If both service_account_name and not service_account_key_file are not
passed the default credential will be used. There are following ways to
create service accounts:
1. Use `build_service_client` method from `cloud_auth` module.
2. Use `gcloud` command line utility as
documented here -
https://cloud.google.com/iam/docs/creating-managing-service-account-keys
version: The version of the service. It defaults to 'v1beta1'.
"""
if service_account_name:
self.client = cloud_auth.build_impersonated_client(
_CLIENT_NAME, service_account_name, version)
else:
if not service_account_key_file:
logging.info(
'Neither Service account key file nor service account '
'name was provided. So using default credentials.')
self.client = cloud_auth.build_service_client(
_CLIENT_NAME, service_account_key_file)
self.project_id = project_id
self.location = location
def test_build_service_client(self):
cloud_auth.build_service_client('service_name')
self.mock_auth_default.assert_called_once()