def insert_entity(projectId, product, categories, table_name, version="v1", prefix="", items="items"): db = TinyDB("project_dbs/" + object_id_to_directory_name(projectId) + ".json") service = discovery.build(product, version, credentials=get_gcloud_creds()) while categories: api_entity = getattr(service, categories.pop(0))() service = api_entity try: request = api_entity.list(project=prefix + projectId) except TypeError: request = api_entity.list(name=prefix + projectId) try: while request is not None: response = request.execute() for item in response[items]: db.table(table_name).insert(item) try: request = api_entity.list_next(previous_request=request, previous_response=response) except AttributeError: request = None except KeyError: pass
def list_projects(project_or_org, specifier): service = discovery.build('cloudresourcemanager', 'v1', credentials=get_gcloud_creds()) service2 = discovery.build('cloudresourcemanager', 'v2', credentials=get_gcloud_creds()) # the filter criteria need double-quotes around them in case they contain special characters, like colons # this was not documented ANYWHERE that I could find when I made this change # but if the double-quotes are not there, you'll get errors like the following: # googleapiclient.errors.HttpError: <HttpError 400 when requesting https://cloudresourcemanager.googleapis.com/v1/projects?filter=id%3Adatadog%3Aproject&alt=json # returned "Request contains an invalid argument."> # HttpError 400 when requesting https://cloudresourcemanager.googleapis.com/v1/projects?filter=id%3Adatadog%3Aproject&alt=json # returned "field [query] has issue [Invalid filter query: resourceType="cloudresourcemanager.projects" AND (projectId = datadog:project)]" if project_or_org == "organization": child = service2.folders().list(parent='organizations/%s' % specifier) child_response = child.execute() request = service.projects().list(filter='parent.id:"%s"' % specifier) if 'folders' in child_response.keys(): for folder in child_response['folders']: list_projects("folder-id", folder['name'].strip(u'folders/')) elif project_or_org == "project-name": request = service.projects().list(filter='name:"%s"' % specifier) elif project_or_org == "project-id": request = service.projects().list(filter='id:"%s"' % specifier) elif project_or_org == "folder-id": child = service2.folders().list(parent='folders/%s' % specifier) child_response = child.execute() request = service.projects().list(filter='parent.id:%s' % specifier) if 'folders' in child_response.keys(): for folder in child_response['folders']: list_projects("folder-id", folder['name'].strip(u'folders/')) else: raise Exception('Organization or Project not specified.') add_projects(request, service)
def insert_sa_policies(projectId, db): service_accounts = db.table("Service Account").all() for account in service_accounts: resp, content = get_gcloud_creds().authorize(Http()).request( "https://iam.googleapis.com/v1/projects/" + projectId + "/serviceAccounts/" + account[ 'uniqueId'] + ":getIamPolicy", "POST", headers=headers) try: for policy in json.loads(content)['bindings']: db.table('Service Account').update( add_policy({ "permission": policy['role'], "scope": policy['members'] }), eids=[account.eid]) except KeyError: pass
def insert_subnet_entities(projectId, version="v1", prefix="", items="items"): product = "compute" categories = ["subnetworks"] table_name = "Subnet" db = TinyDB("project_dbs/" + object_id_to_directory_name(projectId) + ".json") service = discovery.build("compute", version, credentials=get_gcloud_creds()) region_list = [] request = service.regions().list(project=projectId) while request is not None: response = request.execute() if 'items' in response.keys(): for region in response['items']: #print("Debug: %s" % (region)) if 'description' in region.keys(): region_list.append(region['description']) else: print("Warning: no regions found for project '%s'" % (projectId)) request = service.regions().list_next(previous_request=request, previous_response=response) subnet_count = 0 for region in region_list: request = service.subnetworks().list(project=projectId, region=region) while request is not None: response = request.execute() if 'items' in response.keys(): for subnetwork in response['items']: #print("Debug: %s" % (subnetwork)) db.table(table_name).insert(subnetwork) subnet_count = subnet_count + 1 request = service.subnetworks().list_next( previous_request=request, previous_response=response) if subnet_count == 0: print("Warning: no subnets found for project '%s'" % (projectId))
from googleapiclient import discovery from oauth2client.client import GoogleCredentials from core.utility import get_gcloud_creds service = discovery.build('compute', 'v1', credentials=get_gcloud_creds()) def insert_addresses(projectId, db): project = projectId regions = ['us-central1', 'us-east1', 'us-east4', 'us-west1', 'us-west2'] #add more regions as desired for region in regions: try: request = service.addresses().list(project=project, region=region) while request is not None: response = request.execute() for address in response['items']: db.table("Address").insert(address) request = service.addresses().list_next( previous_request=request, previous_response=response) except KeyError: pass
def list_service_account_keys(sa, projectId): service = discovery.build("iam", "v1", credentials=get_gcloud_creds()) request = service.projects().serviceAccounts().keys().list( name="projects/" + projectId + "/serviceAccounts/" + sa['email']) response = request.execute() return response
from googleapiclient import discovery from oauth2client.file import Storage from tinydb import TinyDB db = TinyDB('projects.json') storage = Storage('creds.data') from core.utility import get_gcloud_creds service = discovery.build("storage", "v1", credentials=get_gcloud_creds()) def get_buckets(project): request = service.buckets().list(project=project['projectId']) response = request.execute() buckets = [] if (response.get('items')): for bucket in response['items']: if "logging" not in bucket: buckets.append(bucket) return buckets
import json from googleapiclient import discovery from core.utility import get_gcloud_creds service = discovery.build('cloudresourcemanager', 'v1', credentials=get_gcloud_creds()) def insert_roles(projectId, db): try: try: request = service.projects().getIamPolicy(resource=projectId, body={}) response = request.execute()['bindings'] role_list = None if 'roles' in response: role_list = response['roles'] else: print("Warning: no roles returned for project '%s'" % (projectId)) except Exception as e: print("Error obtaining role list: %s" % (e)) if role_list: for role in role_list: try: db.table('Role').insert(role) except Exception as e: print("Error inserting role into database: %s" % (e)) except Exception as e: print("Error enumerating roles: %s" % (e))
def list_log_services(): projectId = TinyDB('projects.json').table("Project").all() resp, content = get_gcloud_creds().authorize(Http()).request( "https://logging.googleapis.com/v1beta3/projects/" + projectId + "/logServices", "GET") return [service['name'] for service in json.loads(content)['logServices']]
from googleapiclient import discovery from oauth2client.file import Storage from core.utility import get_gcloud_creds storage = Storage('creds.data') service = discovery.build('storage', 'v1', credentials=get_gcloud_creds()) def insert_acls(db): for bucket in db.table('Bucket').all(): request = service.bucketAccessControls().list(bucket=bucket['name']) try: response = request.execute() if 'items' in response.keys(): for acl in response['items']: acl_role = "" acl_entity = "" if 'role' in acl.keys(): acl_role = acl['role'] if 'entity' in acl.keys(): acl_entity = acl['entity'] db.table('Bucket').update(add_acl({ "permission": acl_role, "scope": acl_entity }), eids=[bucket.eid]) except Exception as e: print("Error getting bucket ACLs for bucket '%s': %s" % (bucket, e))
from googleapiclient import discovery from tinydb import TinyDB from core.utility import get_gcloud_creds from core.insert_entity import insert_entity db = TinyDB('entities.json') from oauth2client.file import Storage storage = Storage('creds.data') service = discovery.build('pubsub', 'v1', credentials=get_gcloud_creds()) request = service.projects().topics().list(project="projects/goat-sounds") request = service.projects().subscriptions().list( project="projects/goat-sounds") request = service.projects().subscriptions().getIamPolicy( resource="projects/goat-sounds/subscriptions/baaaa") insert_entity("pubsub", ["projects", "topics"], "Topics", "v1", {"project": "projects/" + projectId}, "topics") insert_entity("pubsub", "subscriptions", "Pub/Sub", "v1", {"project": "projects/" + projectId}, "subscriptions")