'kind': 'compute#machineType',
'maximumPersistentDisks': 16,
'maximumPersistentDisksSizeGb': '3072',
'memoryMb': 614,
'name': 'f1-micro',
'selfLink': 'https://www.googleapis.com/compute/v1/projects/lyrical-diagram-192415/zones/us-east1-b/machineTypes/f1-micro',
'zone': 'us-east1-b'}
Enter a machine type to get more detailed info or c to continue: c
"""
import googleapiclient.discovery
from google.cloud import resource_manager
from prettytable import PrettyTable
from pprint import pprint
t = PrettyTable(["Instance Name", "Status", "Zone", "Machine Type"])
client = resource_manager.Client()
compute = googleapiclient.discovery.build('compute', 'v1')
for project in client.list_projects():
projId = project.project_id
projName = project.name
print("For project " + projName + ", the instances are: ")
request = compute.zones().list(project=projId)
while request is not None:
response = request.execute()
for zone in response['items']:
zoneName = zone['name']
instanceRequest = compute.instances().list(project=projId,
zone=zoneName)
while instanceRequest is not None:
instanceResponse = instanceRequest.execute()
if 'items' in instanceResponse:
from google.cloud import resource_manager
from google.cloud import container_v1
projectList = resource_manager.Client()
for project in projectList.list_projects():
client = container_v1.ClusterManagerClient()
clusterDetails = client.list_clusters(parent="projects/" +
project.project_id + "/locations/-")
for clusterName in clusterDetails.clusters:
print("Cluster Name: " + clusterName.name + " ---- Master IP Range: " +
clusterName.private_cluster_config.master_ipv4_cidr_block)
def client(self):
return resource_manager.Client()
def buildClient():
return rm.Client()
def main(gcs_bucket, gcs_file_path, billing_account_id, default_pubsub_topic,
mysql_host, mysql_port, mysql_user, mysql_pass, mysql_db, statsd_host,
local_mode, dry_run):
logformat = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format=logformat)
# TODO - troubleshoot metric uploading with local container, and test it works on GKE as well.
# Remove this line to see WARNINGs
logging.getLogger('datadog.dogstatsd').setLevel(logging.ERROR)
# Load config.yaml file
if local_mode:
with open("config.yaml", "r") as f:
budget_dict = yaml.load(f, Loader=yaml.SafeLoader)
setup_metrics("localhost")
else:
# Load yaml configuration file from GCS
gcs_client = storage.Client()
bucket = gcs_client.bucket(gcs_bucket)
blob = bucket.blob(gcs_file_path)
yaml_string = blob.download_as_string()
budget_dict = yaml.load(yaml_string, Loader=yaml.SafeLoader)
# Setup metrics with configured statsd host
setup_metrics(statsd_host)
# Setup gcp helper
billing_client = billing_budgets_v1beta1.BudgetServiceClient()
resource_manager_client = resource_manager.Client()
gcp = GcpHelper(billing_client, resource_manager_client)
# Setup mysql connection pool
pool = PooledDB(creator=pymysql,
host=mysql_host,
user=mysql_user,
password=mysql_pass,
database=mysql_db,
autocommit=True,
blocking=True,
maxconnections=5)
mysql_conn = pool.connection()
# Iterate over all configured project budgets
for project_dict in budget_dict['projects']:
config_type = PLUTUS_CONFIG_TYPE_PROJECT
if verify_project_yaml(project_dict):
project = ProjectBudget(project_dict, config_type,
billing_account_id, default_pubsub_topic)
budget = gcp.get_and_update_or_create_budget(project)
if budget is not None:
with mysql_conn.cursor() as mysql_cursor:
upsert_budget(mysql_cursor, budget, project.project_id,
config_type, project.alert_emails)
else:
log.error("Project config verification failed.")
metrics.incr("error_count",
tags=[
"type:misconfig",
f"project_id:{project.project_id}",
f"config_type:{config_type}"
])
sys.exit(1)
# Iterate over all configured parent folder id budgets
for parent_dict in budget_dict['parent_folders']:
parent_id = parent_dict['parent_folder_id']
config_type = PLUTUS_CONFIG_TYPE_PARENT
if verify_parent_yaml(parent_dict):
parent_filter = {'parent.id': parent_id}
for p in resource_manager_client.list_projects(parent_filter):
# Overwrite 'project_id' key for each project under parent folder
parent_dict['project_id'] = p.project_id
project = ProjectBudget(parent_dict, config_type,
billing_account_id,
default_pubsub_topic)
existing_project_budget_id = gcp.has_existing_project_budget(
project)
if existing_project_budget_id is None:
# No project one off budget configured for this projectid
budget = gcp.get_and_update_or_create_budget(project)
if budget is not None:
with mysql_conn.cursor() as mysql_cursor:
upsert_budget(mysql_cursor, budget,
project.project_id, config_type,
project.alert_emails)
else:
log.info(f"Skipping creating parent project budget for \
{p.project_id} since exiting plutus project budget found."
)
existing_parent_budget_id = gcp.has_existing_parent_budget(
parent_id, project)
if existing_parent_budget_id is not None:
# We have a configured plutus project budget. Delete parent budget
gcp.delete_budget(existing_parent_budget_id)
else:
log.error("Parent folder config verification failed.")
metrics.incr("error_count",
tags=[
"type:misconfig", f"parent_id:{parent_id}",
f"config_type:{config_type}"
])
sys.exit(1)
# Iterate over all configured label budgets
for label_dict in budget_dict['labels']:
config_type = PLUTUS_CONFIG_TYPE_LABEL
if verify_labels_yaml(label_dict):
labels_filter = {}
for row in label_dict['label_list']:
for key in row:
labels_filter[f"labels.{key}"] = row[key]
# Find projects that match the labels
for p in resource_manager_client.list_projects(
filter_params=labels_filter):
# Overwrite the 'project_id' key for each project that matches labels
label_dict['project_id'] = p.project_id
project = ProjectBudget(label_dict, config_type,
billing_account_id,
default_pubsub_topic)
existing_project_budget_id = gcp.has_existing_project_budget(
project)
if existing_project_budget_id is None:
# No project one off budget configured for this projectid
budget = gcp.get_and_update_or_create_budget(project)
if budget is not None:
with mysql_conn.cursor() as mysql_cursor:
upsert_budget(mysql_cursor, budget,
project.project_id, config_type,
project.alert_emails)
else:
log.info(f"Skipping creating label project budget for \
{p.project_id} since exiting plutus project budget found."
)
existing_labels_budget_id = gcp.has_existing_labels_budget(
project)
if existing_labels_budget_id is not None:
# We have a configured plutus project budget. Delete labels budget
gcp.delete_budget(existing_labels_budget_id)
else:
log.error("Labels config verification failed.")
metrics.incr("error_count",
tags=["type:misconfig", f"config_type:{config_type}"])
sys.exit(1)
# Default logic removed for now because it will create hundreds of budgets
# And we need to decide good thresholds for this
'''
default_dict = budget_dict['default']
config_type = PLUTUS_CONFIG_TYPE_DEFAULT
if verify_default_yaml(default_dict):
# Query for all projects. If no budget exists for project, add a default budget
for p in resource_manager_client.list_projects():
project_id = p.project_id
project_number = gcp.get_project_number(project_id)
if project_number is not None:
budgets = gcp.get_budgets_by_project(project, project_number)
if len(budgets) == 0:
# No budget exists for this project, so we create one
default_dict['project_id'] = project_id
# TODO - add and test
# project = ProjectBudget(default_dict, config_type,
# billing_account_id, default_pubsub_topic)
log.info(f"Creating default budget for plutus-default-{project_id}")
# budget = gcp.create_budget(project)
# metrics.incr("default_budget_created_count", tags=[])
#if budget is not None:
# with mysql_conn.cursor() as mysql_cursor:
# upsert_budget(mysql_cursor, budget, project.project_id,
# config_type, project.alert_emails)
# TODO - delete plutus-default budget if > 1 budget
elif len(budgets) > 1:
# TODO - if one of the budgets is a plutus-default, delete via API and in Mysql
pass
else:
log.error("Default config verification failed.")
metrics.incr("error_count", tags=["type:misconfig", f"config_type:{config_type}"])
sys.exit(1)
'''
log.info("Plutus run complete.")