def get_halo_groups(session):
old_agent_count = 0
if not os.path.exists("reports"):
os.makedirs("reports")
get_halo_groups_list = cloudpassage.HttpHelper(session)
root_group_id = get_root_group_id(session)
# Set agent_version to the highest version you wish to report on.
# Default is 3.9.7
agent_version = '4.0.0'
# write report for top level group
get_halo_servers_list = cloudpassage.HttpHelper(session)
root_group_reply=get_halo_servers_list.get_paginated("/v2/servers?group_id=" + root_group_id + "&state=active&agent_version_lt=" + agent_version + "&descendants=false","servers",30)
out_file = "reports/Agents_Report_Ungrouped_" + time.strftime("%Y%m%d") + ".csv"
start_csv_file(out_file, [])
gen_servers_report(root_group_reply,out_file)
# write reports for subgroups
subgroup_reply=get_halo_groups_list.get_paginated("/v2/groups?parent_id=" + root_group_id + "&descendants=true","groups",15)
for group in subgroup_reply:
group_id = group['id']
group_name = group['name']
# print "\nGroup %s:\n" % group['name']
out_file = "reports/Agents_Report_" + group_name + "_" + time.strftime("%Y%m%d") + ".csv"
start_csv_file(out_file, [])
get_halo_servers_list = cloudpassage.HttpHelper(session)
servers_reply=get_halo_servers_list.get_paginated("/v2/servers?group_id=" + group_id + "&state=active&agent_version_lt=" + agent_version + "&descendants=true","servers",30)
gen_servers_report(servers_reply,out_file)
def vulnerable_image_check(self):
"""We use this class to scan images for vulnerabilities."""
NONE = 0
FAIL = "1"
# get an HTTP helper object to call REST endpoints
http_helper = cloudpassage.HttpHelper(self.session)
# get all the critical, active issues
image_issues_endpoint = \
"/v1/image_issues?critical=true&status=active"
# set any query parameters set in environment
image_issues_endpoint = \
self.paramaterize_images_issues_endpoint(image_issues_endpoint)
# check for vulnerabilities
image_issues = http_helper.get(image_issues_endpoint)
if image_issues["count"] != NONE:
os.environ["FAIL_ON_CRITICAL"] = FAIL
else:
image_issues = {}
# issue vulnerability data if exists
return image_issues
def get_halo_servers_ami_id(session):
get_halo_servers_list = cloudpassage.HttpHelper(session)
reply = get_halo_servers_list.get_paginated("/v1/servers?state=active",
"servers", 20)
halo_ami_list = []
unique_ami = []
if not os.path.exists("reports"):
os.makedirs("reports")
out_file = "reports/halo_ami_list_" + time.strftime(
"%Y%m%d-%H%M%S") + ".csv"
oamifile = open(out_file, "w")
oamifile.write('AMI Image ID,OS PLatform')
for server in reply:
if 'aws_ec2' in server:
ec2_data = server['aws_ec2']
if ec2_data['ec2_image_id'] not in unique_ami:
halo_ami_list.append({
'ami_image_id': ec2_data['ec2_image_id'],
'os_platform': server['platform']
})
unique_ami.append(ec2_data['ec2_image_id'])
oamifile.write('\n')
rowami = "{0},{1}".format(ec2_data['ec2_image_id'],
server['platform'])
oamifile.write(rowami)
oamifile.close()
halo_ami_list = byteify(halo_ami_list)
#print halo_ami_list
print "Halo AMI List Complete " + time.strftime("%Y%m%d-%H%M%S")
return halo_ami_list
def __init__(self, config):
session = cloudpassage.HaloSession(config.halo_key,
config.halo_secret,
api_host=config.halo_url)
self.server_obj = cloudpassage.Server(session)
self.request_obj = cloudpassage.HttpHelper(session)
self.sva_key = config.key
def get_halo_servers_id(session):
old_agent_count = 0
get_halo_servers_list = cloudpassage.HttpHelper(session)
reply = get_halo_servers_list.get_paginated("/v1/servers?state=active",
"servers", 15)
halo_server_id_list = []
for server in reply:
if 'aws_ec2' in server:
ec2_data = server['aws_ec2']
halo_server_id_list.append({
'halo_server_id':
server['id'],
'aws_instance_id':
ec2_data['ec2_instance_id'],
'aws_account_id':
ec2_data['ec2_account_id']
})
elif server['server_label'] and "_" in server['server_label']:
server_label = server['server_label']
server_label_parts = server_label.split("_")
#print server_label_parts[1]
#old_agent_count += 1
server_label_account = server_label_parts[0]
server_label_isntance = server_label_parts[1]
halo_server_id_list.append({
'halo_server_id': server['id'],
'aws_instance_id': server_label_isntance,
'aws_account_id': server_label_account
})
halo_instance_id_list = byteify(halo_server_id_list)
print "Halo Server ID and AWS Account ID Lookup Complete " + time.strftime(
"%Y%m%d-%H%M%S")
return halo_instance_id_list
def add_csp_accounts(session, csvFile):
cspAccounts = read_accounts_csv(csvFile)
with open('cloudpassage.yml') as config_settings:
script_options_info = yaml.load(config_settings)
home_group = script_options_info['defaults']['home_group_id']
roleArn = script_options_info['defaults']['role_arn']
externalId = str(script_options_info['defaults']['externalId'])
AddCspAccount = cloudpassage.HttpHelper(session)
for account in cspAccounts:
if account['account_name']:
payload = {
"external_id": str(externalId),
"role_arn": str(roleArn),
"sns_arn": account['sns_arn'],
"group_id": home_group,
"csp_account_type": account['account_type'],
"account_display_name": account['account_name']
}
else:
payload = {
"external_id": str(externalId),
"role_arn": str(roleArn),
"sns_arn": account['sns_arn'],
"group_id": home_group,
"csp_account_type": account['account_type']
}
#print payload
AddCspAccount.post("/v1/csp_accounts", payload)
print "Added CSP Account {0} to Halo Cloud Secure\n".format(
account['account_id'])
def get_pages(self, url_list):
halo_session = self.build_halo_session()
page_helper = cloudpassage.HttpHelper(halo_session)
pool = ThreadPool(self.max_threads)
results = pool.map(page_helper.get, url_list)
pool.close()
pool.join()
return results
def get(self, per_page, date, page):
"""HTTP GET events from Halo"""
session = self.create_halo_session_object()
api = cloudpassage.HttpHelper(session)
url = "/v1/events?per_page=%s&page=%s&since=%s" % (per_page, page,
date)
return api.get(url)
def latest_event(self, per_page, date, page):
"""get the latest event from Halo"""
session = self.create_halo_session_object()
api = cloudpassage.HttpHelper(session)
url = "/v1/events?sort_by=created_at.desc&per_page=%s&page=%s&since=%s" % (
per_page, page, date) # NOQA
return api.get(url)
def build_http_session(self):
key_id = os.environ['KEY_ID'] or CONFIG['key_id']
secret_key = os.environ['SECRET_KEY'] or CONFIG['secret_key']
integration_string = self.get_integration_string()
session = cloudpassage.HaloSession(key_id,
secret_key,
integration_string=integration_string)
return cloudpassage.HttpHelper(session)
def retrieve_coreos_csm(session):
with open('cloudpassage.yml', 'r') as f:
config_file = yaml.load(f)
coreos_policy_id = config_file['defaults']['csm_policy_id']
csm_policy = cloudpassage.HttpHelper(session)
coreos_policy = csm_policy.get("/v1/policies/" + coreos_policy_id)
cores_policy = fixup(coreos_policy, u'desired_value')
return coreos_policy
def build_http_session():
key_id = CONFIG['key_id']
secret_key = CONFIG['secret_key']
session = cloudpassage.HaloSession(key_id,
secret_key,
api_port=CONFIG["api_port"],
api_host=CONFIG["api_hostname"])
return cloudpassage.HttpHelper(session)
def get_pages(self, url_list):
"""Magic happens here... we map pages to threads in a pool, return
results when it's all done."""
halo_session = self.build_halo_session()
page_helper = cloudpassage.HttpHelper(halo_session)
pool = ThreadPool(self.max_threads)
results = pool.map(page_helper.get, url_list)
pool.close()
pool.join()
return results
def test_get_rekey(self):
endpoint = "/v1/servers"
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.auth_token = "abc123"
req = cloudpassage.HttpHelper(session)
json_response = req.get(endpoint)
assert "servers" in json_response
def get_pages(cls, halo_key, halo_secret, api_host, api_port, ua,
max_threads, url_list):
"""We map pages to threads, return results when it's all done."""
halo_session = cls.build_halo_session(halo_key, halo_secret, api_host,
api_port, ua)
page_helper = cloudpassage.HttpHelper(halo_session)
pool = ThreadPool(max_threads)
results = pool.map(page_helper.get, url_list)
pool.close()
pool.join()
return results
def test_post_rekey(self):
endpoint = nonexistent_url
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.auth_token = "abc123"
req = cloudpassage.HttpHelper(session)
with pytest.raises(cloudpassage.CloudPassageResourceExistence) as e:
req.post(endpoint, {"nonexist": "nothing"})
assert '404' in str(e)
def test_get_paginated_events_99(self):
endpoint = "/v1/events"
key = "events"
pages = 5
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
req = cloudpassage.HttpHelper(session)
json_response = req.get_paginated(endpoint, key, pages)
assert "id" in json_response[0]
def get_root_group_id(session):
get_root_id = cloudpassage.HttpHelper(session)
root_id_reply=get_root_id.get_paginated("/v2/groups","groups",30)
return_value = ''
for group in root_id_reply:
group_name = group['name']
group_id = group['id']
parent_id = group['parent_id']
if parent_id == None:
return_value = group_id
break
return return_value
def test_get_404(self):
endpoint = nonexistent_url
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.authenticate_client()
req = cloudpassage.HttpHelper(session)
with pytest.raises(cloudpassage.CloudPassageResourceExistence) as e:
req.get(endpoint)
assert '404' in str(e)
def test_delete_rekey(self):
endpoint = "/v1/servers/123455432"
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.auth_token = "abc123"
req = cloudpassage.HttpHelper(session)
with pytest.raises(cloudpassage.CloudPassageResourceExistence) as e:
req.delete(endpoint)
assert '404' in str(e)
def create_grp(self, grp_name, grp_id):
"""create new server group"""
api = cloudpassage.HttpHelper(self.create_halo_session_object())
data = {
"group": {
"name": grp_name,
"parent_id": grp_id
}
}
resp = api.post('/v1/groups', data)
return resp["group"]["id"]
def test_put_rekey(self):
body = {"server": {"retire": True}}
endpoint = "/v1/servers/1234567890"
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.auth_token = "abc123"
req = cloudpassage.HttpHelper(session)
with pytest.raises(cloudpassage.CloudPassageResourceExistence) as e:
req.put(endpoint, body)
assert '404' in str(e)
def test_post_bad_payload(self):
endpoint = "/v1/groups"
put_data = {"whatevs": "becausenobodycares"}
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.authenticate_client()
req = cloudpassage.HttpHelper(session)
with pytest.raises(cloudpassage.CloudPassageResourceExistence) as e:
req.put(endpoint, put_data)
assert '404' in str(e)
def test_get_paginated_toomany(self):
endpoint = "/v1/events"
key = "events"
pages = 301
session = cloudpassage.HaloSession(key_id,
secret_key,
api_host=api_hostname,
api_port=api_port,
integration_string="SDK-Smoke")
session.auth_token = "abc123"
req = cloudpassage.HttpHelper(session)
with pytest.raises(cloudpassage.CloudPassageValidation) as e:
req.get_paginated(endpoint, key, pages)
assert '300 max.' in str(e)
def __init__(self, key, secret, api_host):
"""Instantiate with key, secret, and API host.
Args:
config (ConfigHelper): Config Object
"""
self.logger = Logger()
integration = self.get_integration_string()
self.session = cloudpassage.HaloSession(key,
secret,
api_host=api_host,
integration_string=integration)
self.issue = cloudpassage.Issue(self.session, endpoint_version=3)
self.http_helper = cloudpassage.HttpHelper(self.session)
self.cve_detail = cloudpassage.CveDetails(self.session)
def find_api_key_owner(session):
with open('cloudpassage.yml') as config_settings:
script_options_info = yaml.load(config_settings)
look_for_me = script_options_info['defaults']['search_api_key']
api_results_list = cloudpassage.HttpHelper(session)
list_of_groups = api_results_list.get_paginated("/v1/groups?per_page=1000", "groups", 20)
for group in list_of_groups:
query_url = "/v2/groups/" + group['id'] + "/api_keys"
list_of_keys = api_results_list.get_paginated(query_url, "api_keys", 10)
for key in list_of_keys:
if key['key'] == look_for_me:
print "Owner group is name = {0}, groupID = {1} \n group_path = {2}".format(group['name'], group['id'], group['group_path'])
break
#if you got here, no match found
print "No Owner group found, please check your other accounts"
def get_last_timestamp(self):
"""Get latest object's timestamp from Halo API."""
if self.halo_module == "scans":
url = "/v1/scans?sort_by=created_at.desc&per_page=1"
elif self.halo_module == "events":
url = "/v1/events?sort_by=created_at.desc&per_page=1"
else:
print("Unrecognized module: %s" % self.halo_module)
session = cloudpassage.HaloSession(
self.halo_key,
self.halo_secret,
api_host=self.halo_api_hostname,
integration_string=self.integration_name)
http_helper = cloudpassage.HttpHelper(session)
timestamp = http_helper.get(url)[self.halo_module][0]["created_at"]
return timestamp
def provision_csp_account(self,
account_id,
display_name,
role_name,
external_id,
group_id,
csp_account_type="AWS"):
"""Provision a CSP account in Halo.
Args:
account_id (str): Account ID to be used in provisioning.
display_name (str): Account display name.
role_name (str): Used with account_id to construct role ARN.
external_id (str): External ID required for assuming role in AWS
account.
csp_account_type (str): Cloud provider. Only `aws` is currently
supported.
Returns:
str: Halo ID for CSP account
"""
http_helper = cloudpassage.HttpHelper(self.session)
payload = {
"external_id": external_id,
"role_arn": self.construct_role_arn(account_id, role_name),
"group_id": group_id,
"csp_account_type": csp_account_type
}
try:
halo_csp_account_id = http_helper.post("/v1/csp_accounts", payload)
except cloudpassage.CloudPassageAuthorization as e:
print("Failed to provision account. Not authorized!\n%s" % e)
sys.exit(1)
except cloudpassage.CloudPassageValidation as e:
print("Validation failure when creating account: %s" % e)
print("ARN and external ID will fail if they don't exist.")
sys.exit(1)
return halo_csp_account_id
def update_csp_display_name(self, halo_csp_account_id, display_name):
"""Update account_display_name for CSP account in Halo.
Args:
halo_csp_account_id (str): ID for Halo CSP account object. This is
NOT the CSP account ID.
display_name (str): String to be substituted for existing
`account_display_name` for CSP account in Halo.
Returns:
None
"""
# Guard against URL traversal, etc...
try:
self.validate_object_id(halo_csp_account_id)
except cloudpassage.CloudPassageValidation as e:
print("Halo CSP ID failed validation!: %s" % e)
sys.exit(1)
http_helper = cloudpassage.HttpHelper(self.session)
url = "/v1/csp_accounts/%s" % halo_csp_account_id
payload = {"account_display_name": display_name}
http_helper.put(url, payload)
return
def remove_groups(session):
groups_deleted = []
super_count = 1
while super_count > 0:
count = 0
api_results_list = cloudpassage.HttpHelper(session)
list_of_groups = api_results_list.get_paginated("/v1/groups?per_page=1000", "groups", 20)
#print list_of_groups
for group in list_of_groups:
if group['server_counts']['active'] == 0 and group['has_children'] == False and group['parent_id']:
query_url = "/v1/groups/" + group['id']
#print "found one"
api_results_list.delete(query_url)
groups_deleted.append({'group name': group['name'],'group path': group['group_path'],'group ID': group['id']})
count += 1
time.sleep(30) #wait for API to update
if count == 0:
super_count = 0
if len(groups_deleted) > 0:
print groups_deleted
print "\nScript results: {0} groups deleted\n".format(str(len(groups_deleted)))
else:
print "No groups deleted... Please move active servers from groups to be deleted"