def __init__(self, project_id: str, service_account_name: Optional[str] = None, service_account_key_file: Optional[str] = None) -> None: """Initialize new instance of BigQueryUtils. Args: project_id: GCP project id. service_account_name: The service account name. service_account_key_file: File containing service account key. If both service_account_name and service_account_key_file are not passed the default credential will be used.There are following ways to create service accounts - 1) Use `build_service_client` method from `cloud_auth` module. 2) Use `gcloud` command line utility as documented here - https://cloud.google.com/iam/docs/creating-managing-service-account-keys """ if service_account_name: credentials = cloud_auth.impersonate_service_account( service_account_name) elif service_account_key_file: credentials = cloud_auth.get_credentials(service_account_key_file) else: logging.info( 'Neither Service account key file nor service account ' 'name was provided, so using default credentials.') credentials = cloud_auth.get_default_credentials() self.project_id = project_id self.client = bigquery.Client(project=project_id, credentials=credentials)
def __init__(self, project_id: str, location: str = _LOCATION, service_account_name: Optional[str] = None, service_account_key_file: Optional[str] = None, version: str = _VERSION) -> None: """Initialise new instance of CloudComposerUtils. Args: project_id: GCP project id. location: Optional. Region under which the Composer environment needs to be managed. It defaults to 'us-central1'. Allowed values - https://cloud.google.com/compute/docs/regions-zones/. service_account_name: The service account name. service_account_key_file: Optional. File containing service account key. version: The version of the service. It defaults to 'v1beta1'. """ if service_account_name: self.client = cloud_auth.build_impersonated_client( _CLIENT_NAME, service_account_name, version) else: if service_account_key_file: credentials = cloud_auth.get_credentials( service_account_key_file) else: logging.info( 'Neither service account key file nor service account ' 'name was provided, so using default credentials.') credentials = cloud_auth.get_default_credentials() self.client = cloud_auth.build_service_client( _CLIENT_NAME, credentials) self.project_id = project_id self.location = location
def __init__(self, project_id: str, service_account_name: Optional[str] = None, service_account_key_file: Optional[str] = None, version: str = _VERSION) -> None: """Initialise new instance of CloudApiUtils. Args: project_id: GCP project id. service_account_name: The service account name. service_account_key_file: Optional. File containing service account key. version: The version of the service usage service. It defaults to 'v1beta1'. """ if service_account_name: self.client = cloud_auth.build_impersonated_client( 'serviceusage', service_account_name, version) else: if service_account_key_file: credentials = cloud_auth.get_credentials( service_account_key_file) else: logging.info( 'Neither service account key file nor service account ' 'name was provided, so using default credentials.') credentials = cloud_auth.get_default_credentials() self.client = cloud_auth.build_service_client( 'serviceusage', credentials) self.project_id = project_id
def __init__(self, project_id: str, service_account_info: Mapping[str, str] = None, service_account_name: Optional[str] = None, service_account_key_file: Optional[str] = None) -> None: """Initialize new instance of CloudStorageUtils. Args: project_id: GCP project id. service_account_info: Mapping containing the service account info, such as the example below: { 'type': 'service_account', 'project_id': '[PROJECT_ID]', 'private_key_id': '[PRIVATE_KEY_ID]', 'private_key': '[PRIVATE_KEY]', 'client_email': '[CLIENT_EMAIL]', 'client_id': '[CLIENT_ID]', 'auth_uri': 'https://accounts.google.com/o/oauth2/auth', 'token_uri': 'https://accounts.google.com/o/oauth2/token', 'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs', 'client_x509_cert_url': '[CERTIFICATE_URL]' } service_account_name: The service account name. service_account_key_file: File containing service account key. """ if service_account_name: credentials = cloud_auth.impersonate_service_account( service_account_name) elif service_account_key_file: credentials = cloud_auth.get_credentials(service_account_key_file) elif service_account_info: credentials = cloud_auth.get_credentials_from_info( service_account_info) else: logging.info( 'Neither Service account key file nor service account ' 'name was provided, so using default credentials.') credentials = cloud_auth.get_default_credentials() self.client = storage.Client(project=project_id, credentials=credentials)
def visualize_instances(config_file: str) -> None: """Visualizes the statistics from the Instance table in BigQuery. This involves calculating statistics from the Instance table in BigQuery, generates and outputs plots into a pdf file and uploads the pdf file to a given location in Cloud Storage. Args: config_file: Path to the configuration file. """ viz_config = viz_utils.parse_config_file(config_file) project_id = viz_config['project_id'] dataset = viz_config['dataset'] instance_table = viz_config['instance_table'] instance_table_path = f'{project_id}.{dataset}.{instance_table}' bq_client = typing.cast( client.Client, cloud_auth.build_service_client( service_name='bigquery', service_account_credentials=cloud_auth.get_default_credentials())) storage_client = cloud_storage.CloudStorageUtils( project_id=viz_config['project_id'], service_account_key_file=viz_config['service_account_key_file']) pdf_output = matplotlib.backends.backend_pdf.PdfPages( viz_config['output_local_path']) ins_viz_obj = instance_visualizer.InstanceVisualizer( bq_client=bq_client, instance_table_path=instance_table_path, num_instances=viz_config['num_instances'], label_column=viz_config['label'], positive_class_label=viz_config['True'], negative_class_label=viz_config['False']) ins_viz_obj.plot_instances(**viz_config['plot_style_params']) pdf_output.savefig() storage_client.upload_file_to_url(viz_config['output_local_path'], viz_config['output_gcs_path'])
def visualize_facts(config_file: str) -> None: """Visualizes the statistics from the Facts table in BigQuery. This involves calculating statistics from the Facts table in BigQuery, generates and outputs plots into a pdf file and uploads the pdf file to a given location in Cloud Storage. Args: config_file: Path to the configuration file. """ viz_config = viz_utils.parse_config_file(config_file) project_id = viz_config['project_id'] dataset = viz_config['dataset'] facts_table = viz_config['facts_table'] facts_table_path = f'{project_id}.{dataset}.{facts_table}' bq_client = typing.cast( client.Client, cloud_auth.build_service_client( service_name='bigquery', service_account_credentials=cloud_auth.get_default_credentials())) storage_client = cloud_storage.CloudStorageUtils( project_id=viz_config['project_id'], service_account_key_file=viz_config['service_account_key_file']) pdf_output = matplotlib.backends.backend_pdf.PdfPages( viz_config['output_local_path']) fact_viz_obj = fact_visualizer.FactVisualizer( bq_client=bq_client, facts_table_path=facts_table_path, numerical_facts=viz_config['numerical_fact_list'], categorical_facts=viz_config['categorical_fact_list'], number_top_levels=viz_config['number_top_levels']) fact_viz_obj.plot_facts(**viz_config['plot_style_params']) pdf_output.savefig() storage_client.upload_file_to_url(viz_config['output_local_path'], viz_config['output_gcs_path'])