def insert_entity(projectId,
product,
categories,
table_name,
version="v1",
prefix="",
items="items"):
db = TinyDB("project_dbs/" + object_id_to_directory_name(projectId) +
".json")
service = discovery.build(product, version, credentials=get_gcloud_creds())
while categories:
api_entity = getattr(service, categories.pop(0))()
service = api_entity
try:
request = api_entity.list(project=prefix + projectId)
except TypeError:
request = api_entity.list(name=prefix + projectId)
try:
while request is not None:
response = request.execute()
for item in response[items]:
db.table(table_name).insert(item)
try:
request = api_entity.list_next(previous_request=request,
previous_response=response)
except AttributeError:
request = None
except KeyError:
pass
def list_projects(project_or_org, specifier):
service = discovery.build('cloudresourcemanager',
'v1',
credentials=get_gcloud_creds())
service2 = discovery.build('cloudresourcemanager',
'v2',
credentials=get_gcloud_creds())
# the filter criteria need double-quotes around them in case they contain special characters, like colons
# this was not documented ANYWHERE that I could find when I made this change
# but if the double-quotes are not there, you'll get errors like the following:
# googleapiclient.errors.HttpError: <HttpError 400 when requesting https://cloudresourcemanager.googleapis.com/v1/projects?filter=id%3Adatadog%3Aproject&alt=json
# returned "Request contains an invalid argument.">
# HttpError 400 when requesting https://cloudresourcemanager.googleapis.com/v1/projects?filter=id%3Adatadog%3Aproject&alt=json
# returned "field [query] has issue [Invalid filter query: resourceType="cloudresourcemanager.projects" AND (projectId = datadog:project)]"
if project_or_org == "organization":
child = service2.folders().list(parent='organizations/%s' % specifier)
child_response = child.execute()
request = service.projects().list(filter='parent.id:"%s"' % specifier)
if 'folders' in child_response.keys():
for folder in child_response['folders']:
list_projects("folder-id", folder['name'].strip(u'folders/'))
elif project_or_org == "project-name":
request = service.projects().list(filter='name:"%s"' % specifier)
elif project_or_org == "project-id":
request = service.projects().list(filter='id:"%s"' % specifier)
elif project_or_org == "folder-id":
child = service2.folders().list(parent='folders/%s' % specifier)
child_response = child.execute()
request = service.projects().list(filter='parent.id:%s' % specifier)
if 'folders' in child_response.keys():
for folder in child_response['folders']:
list_projects("folder-id", folder['name'].strip(u'folders/'))
else:
raise Exception('Organization or Project not specified.')
add_projects(request, service)
def insert_sa_policies(projectId, db):
service_accounts = db.table("Service Account").all()
for account in service_accounts:
resp, content = get_gcloud_creds().authorize(Http()).request(
"https://iam.googleapis.com/v1/projects/" + projectId + "/serviceAccounts/" + account[
'uniqueId'] + ":getIamPolicy", "POST", headers=headers)
try:
for policy in json.loads(content)['bindings']:
db.table('Service Account').update(
add_policy({
"permission": policy['role'],
"scope": policy['members']
}),
eids=[account.eid])
except KeyError:
pass
def insert_subnet_entities(projectId, version="v1", prefix="", items="items"):
product = "compute"
categories = ["subnetworks"]
table_name = "Subnet"
db = TinyDB("project_dbs/" + object_id_to_directory_name(projectId) +
".json")
service = discovery.build("compute",
version,
credentials=get_gcloud_creds())
region_list = []
request = service.regions().list(project=projectId)
while request is not None:
response = request.execute()
if 'items' in response.keys():
for region in response['items']:
#print("Debug: %s" % (region))
if 'description' in region.keys():
region_list.append(region['description'])
else:
print("Warning: no regions found for project '%s'" % (projectId))
request = service.regions().list_next(previous_request=request,
previous_response=response)
subnet_count = 0
for region in region_list:
request = service.subnetworks().list(project=projectId, region=region)
while request is not None:
response = request.execute()
if 'items' in response.keys():
for subnetwork in response['items']:
#print("Debug: %s" % (subnetwork))
db.table(table_name).insert(subnetwork)
subnet_count = subnet_count + 1
request = service.subnetworks().list_next(
previous_request=request, previous_response=response)
if subnet_count == 0:
print("Warning: no subnets found for project '%s'" % (projectId))
from googleapiclient import discovery
from oauth2client.client import GoogleCredentials
from core.utility import get_gcloud_creds
service = discovery.build('compute', 'v1', credentials=get_gcloud_creds())
def insert_addresses(projectId, db):
project = projectId
regions = ['us-central1', 'us-east1', 'us-east4', 'us-west1', 'us-west2']
#add more regions as desired
for region in regions:
try:
request = service.addresses().list(project=project, region=region)
while request is not None:
response = request.execute()
for address in response['items']:
db.table("Address").insert(address)
request = service.addresses().list_next(
previous_request=request, previous_response=response)
except KeyError:
pass
def list_service_account_keys(sa, projectId):
service = discovery.build("iam", "v1", credentials=get_gcloud_creds())
request = service.projects().serviceAccounts().keys().list(
name="projects/" + projectId + "/serviceAccounts/" + sa['email'])
response = request.execute()
return response
from googleapiclient import discovery
from oauth2client.file import Storage
from tinydb import TinyDB
db = TinyDB('projects.json')
storage = Storage('creds.data')
from core.utility import get_gcloud_creds
service = discovery.build("storage", "v1", credentials=get_gcloud_creds())
def get_buckets(project):
request = service.buckets().list(project=project['projectId'])
response = request.execute()
buckets = []
if (response.get('items')):
for bucket in response['items']:
if "logging" not in bucket:
buckets.append(bucket)
return buckets
import json
from googleapiclient import discovery
from core.utility import get_gcloud_creds
service = discovery.build('cloudresourcemanager',
'v1',
credentials=get_gcloud_creds())
def insert_roles(projectId, db):
try:
try:
request = service.projects().getIamPolicy(resource=projectId,
body={})
response = request.execute()['bindings']
role_list = None
if 'roles' in response:
role_list = response['roles']
else:
print("Warning: no roles returned for project '%s'" %
(projectId))
except Exception as e:
print("Error obtaining role list: %s" % (e))
if role_list:
for role in role_list:
try:
db.table('Role').insert(role)
except Exception as e:
print("Error inserting role into database: %s" % (e))
except Exception as e:
print("Error enumerating roles: %s" % (e))
def list_log_services():
projectId = TinyDB('projects.json').table("Project").all()
resp, content = get_gcloud_creds().authorize(Http()).request(
"https://logging.googleapis.com/v1beta3/projects/" + projectId + "/logServices", "GET")
return [service['name'] for service in json.loads(content)['logServices']]
from googleapiclient import discovery
from oauth2client.file import Storage
from core.utility import get_gcloud_creds
storage = Storage('creds.data')
service = discovery.build('storage', 'v1', credentials=get_gcloud_creds())
def insert_acls(db):
for bucket in db.table('Bucket').all():
request = service.bucketAccessControls().list(bucket=bucket['name'])
try:
response = request.execute()
if 'items' in response.keys():
for acl in response['items']:
acl_role = ""
acl_entity = ""
if 'role' in acl.keys():
acl_role = acl['role']
if 'entity' in acl.keys():
acl_entity = acl['entity']
db.table('Bucket').update(add_acl({
"permission": acl_role,
"scope": acl_entity
}),
eids=[bucket.eid])
except Exception as e:
print("Error getting bucket ACLs for bucket '%s': %s" %
(bucket, e))
from googleapiclient import discovery
from tinydb import TinyDB
from core.utility import get_gcloud_creds
from core.insert_entity import insert_entity
db = TinyDB('entities.json')
from oauth2client.file import Storage
storage = Storage('creds.data')
service = discovery.build('pubsub', 'v1', credentials=get_gcloud_creds())
request = service.projects().topics().list(project="projects/goat-sounds")
request = service.projects().subscriptions().list(
project="projects/goat-sounds")
request = service.projects().subscriptions().getIamPolicy(
resource="projects/goat-sounds/subscriptions/baaaa")
insert_entity("pubsub", ["projects", "topics"], "Topics", "v1",
{"project": "projects/" + projectId}, "topics")
insert_entity("pubsub", "subscriptions", "Pub/Sub", "v1",
{"project": "projects/" + projectId}, "subscriptions")
