def create_new_role(conn, project_name, scan_id, team_id, scanid_valid, teamid_valid):
"""
This method is to validate that tenant are not allowed to create the new role in
P3 platform.
:param conn: connection handle to OpenStack project
:param project_name: project name
:param seq_nums_list: empty list
:param params_list: empty list
:param scan_id: ScanID received from AWS SQS
:param team_id: TeamID
:param session: session handle to Kinesis
:return: Compliant | Non-Compliant | None
"""
try:
role = []
for new_role in conn.identity.create_role():
role = new_role
print("ERROR: Tenant are being allowed to create a role")
compliance_status = "Non-compliant"
return compliance_status
except openstack.exceptions.HttpException as exp_err:
print("INFO: Error Received while attempting to create role - %s" % str(exp_err))
if str(exp_err).find("You are not authorized"):
print("INFO: Tenant are not allowed to create a role")
compliance_status = "Compliant"
else:
compliance_status = "Non-compliant"
resource = project_name + "-" + "create_role"
if scanid_valid and teamid_valid:
if general_util.params_list_update(scan_id, tc, team_id, resource, compliance_status, params_list):
print("INFO: Updating params_list")
else:
print("ERROR: Issue observed while updating params_list")
return None
else:
print("INFO: ScanId or TeamId passed to main() method is not valid,"
" hence ignoring Kinesis part")
return compliance_status
except Exception as e:
print("ERROR: Issue observed while calling create_role() API - %s" % str(e))
return None
def change_domain(conn, domain_name, project_name, scan_id, team_id, scanid_valid, teamid_valid):
"""
This method is to validate that tenant are not allowed to change the domain
in P3 platform.
:param conn: Connection handle to OpenStack project
:param domain_name: current domain name
:param project_name: Project Name
:param seq_nums_list: empty list
:param params_list: empty list
:param scan_id: ScanID received from AWS SQS
:param team_id: TeamID
:return: Compliant | Non-Compliant | None
"""
try:
domain_change = []
audit_time = int(time.time()) * 1000
for change_domain in conn.identity.update_domain(domain_name, name="admin", description=None, enabled=None):
new_domain = change_domain
print("ERROR: Tenant are being allowed to change a domain")
compliance_status = "Non-compliant"
return compliance_status
except openstack.exceptions.ResourceNotFound as exp_err:
print("INFO: Error Received while attempting to change domain - %s" % str(exp_err))
if str(exp_err).find("Could not find domain"):
print("INFO: Tenant are not allowed to change a domain")
compliance_status = "Compliant"
else:
compliance_status = "Non-compliant"
resource = project_name + "-" + "change_domain"
if scanid_valid and teamid_valid:
if general_util.params_list_update(scan_id, tc, team_id, resource, compliance_status, params_list):
print("INFO: Updating params_list")
else:
print("ERROR: Issue observed while updating params_list")
return None
else:
print("INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part")
return compliance_status
except Exception as e:
print("ERROR: Issue observed while calling create domain() API - %s" % str(e))
return None
def list_domain(conn, project_name, scan_id, team_id, scanid_valid, teamid_valid):
"""
This method is to validate that tenant are not allowed to list/read the domains
in P3 platform.
:param conn: connection handle to OpenStack project
:param project_name: project name
:param seq_nums_list: empty list
:param params_list: empty list
:param scan_id: ScanID received from AWS SQS
:param team_id: TeamID
:return: Compliant | Non-Compliant | None
"""
try:
domain_list = []
audit_time = int(time.time()) * 1000
for list_domain in conn.identity.domains():
domain = json.dumps(list_domain)
print("ERROR: Tenant are being allowed to list domain")
compliance_status = "Non-compliant"
return compliance_status
except openstack.exceptions.HttpException as exp_err:
print("INFO: Error Received while attempting to list domains - %s" % str(exp_err))
if str(exp_err).find("You are not authorized"):
print("INFO: Tenant are not allowed to list domains")
compliance_status = "Compliant"
else:
compliance_status = "Non-compliant"
resource = project_name + "-" + "list_domain"
if scanid_valid and teamid_valid:
if general_util.params_list_update(scan_id, tc, team_id, resource, compliance_status, params_list):
print("INFO: Updating params_list")
else:
print("ERROR: Issue observed while updating params_list")
return None
else:
print("INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part")
return compliance_status
except Exception as e:
print("ERROR: Issue observed while calling listing domains() API - %s" % str(e))
return None
def complete_data(rolebinding_all, project_app, project_name_id_mapping,
trusted_roles, session, team_id, scan_id, path_url,
scanid_valid, teamid_valid, rolebinding_groupName_all):
"""
Method to filter and append complete rolebinding data of the given project from metadata dictionay to csv file
:param rolebinding_all:
:param project_app:
:param project_name_id_mapping:
:param trusted_roles:
:param session:
:param team_id:
:param scan_id:
:param path_url:
:param scanid_valid:
:param teamid_valid:
:param rolebinding_groupName_all:
:return:
"""
try:
data_all = []
for project in rolebinding_all:
if rolebinding_all[project] is not None:
for role in rolebinding_all[project]:
print(
"INFO: Calculating the age of role assigned to the user"
)
dt = dateutil.parser.parse(
rolebinding_all[project][role]['timecreated'])
dt = dt.replace(tzinfo=None)
diff = datetime.datetime.now() - dt
diff = ("%s Days %s Hours %s Mins" %
(diff.days, diff.seconds // 3600,
(diff.seconds // 60) % 60))
groupname = rolebinding_groupName_all[project][role]
if type(groupname) == list:
groupname = ''.join(groupname)
if project in project_app:
if rolebinding_all[project][role][
'usernames'] is not None:
for user in set(rolebinding_all[project][role]
['usernames']):
if role in trusted_roles:
compliance_status = "Compliant"
else:
compliance_status = "Non-compliant"
if "admin" == role and user != "citeis-orchadm.gen":
compliance_status = "Non-compliant"
data_all.append([
path_url, project,
project_name_id_mapping[project],
project_app[project]['app_id'],
project_app[project]['app_name'], role,
user, groupname, diff, compliance_status
])
role_user = role + "-" + user
if scanid_valid and teamid_valid:
if general_util.params_list_update(
scan_id, tc, team_id, role_user,
compliance_status, params_list):
print("INFO: Updating params_list")
else:
print(
"ERROR: Issue observed while updating params_list in complete_data method"
)
return None
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid,"
" hence ignoring Kinesis part")
else:
if role in trusted_roles:
compliance_status = "Compliant"
else:
compliance_status = "Non-compliant"
if "admin" == role:
compliance_status = "Non-compliant"
data_all.append([
path_url, project,
project_name_id_mapping[project],
project_app[project]['app_id'],
project_app[project]['app_name'], role, "None",
groupname, diff, compliance_status
])
else:
if role in trusted_roles:
compliance_status = "Compliant"
else:
compliance_status = "Non-compliant"
if "admin" == role and user != "citeis-orchadm.gen":
compliance_status = "Non-compliant"
if rolebinding_all[project][role][
'usernames'] is not None:
for user in rolebinding_all[project][role][
'usernames']:
data_all.append([
path_url, project, "None", "None", role,
user, groupname, diff, compliance_status
])
else:
data_all.append([
path_url, project, "None", "None", role,
"None", groupname, diff, compliance_status
])
# Creating the Compliant cases file index
df = pd.DataFrame(data_all,
columns=[
"URL", "Tenant Name", "Tenant ID",
"Application ID", "Application Name",
"Role Name", "Users with Associate Roles",
"Group Names", "Role Provisioned Age",
"Compliant Status"
])
#creating Compliant cases csv file with the date stamp in the name
date = datetime.datetime.now().strftime('%m%d%y')
metadata_file = os.path.expanduser(
"~") + "/logs/cae_identity_mgmt_tc_1_" + date + ".csv"
if os.path.isfile(metadata_file):
with open(metadata_file, 'a') as f:
df.to_csv(f, header=False, index=False)
else:
df.to_csv(metadata_file, index=False)
if scanid_valid and teamid_valid:
print("INFO: Adding result to Stream")
stream_info = general_util.add_result_to_stream(
session, "CAE", str(team_id), tc, params_list)
if stream_info is None:
raise Exception(
"ERROR: Issue observed while calling add_result_to_stream() API in complete_data"
)
return None
seq_nums_list.append(stream_info)
print("INFO: Sending result complete")
send_result = general_util.send_result_complete(
session, "CAE", scan_id, team_id, tc, seq_nums_list)
if send_result:
print("INFO: Successfully submitted the result to Kinesis")
else:
print(
"ERROR: Failed to submit the result to Kinesis in complete_data"
)
return None
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part"
)
except Exception as e:
print(
"ERROR: Fail to retrieve the complete user data in complete_data with error : %s"
% str(e))
def unsecured_images_list(all_images_list, scan_id, team_id, scanid_valid,
teamid_valid):
"""
Method to fetch out the all unsecured images details in the OpenStack project
:param all_images_list: list_all_images
:param scan_id: ScanID received from AWS SQS
:param team_id: TeamID
:param scanid_valid:
:param teamid_valid:
:return: Compliant | Non-Compliant | None
"""
try:
all_unsecured_images = list()
for image in all_images_list:
if image[6] != 'public':
all_unsecured_images.append(image)
compliance_status = "Non-compliant"
else:
compliance_status = "Compliant"
resource = "Image:" + image[5]
if scanid_valid and teamid_valid:
if general_util.params_list_update(scan_id, tc, team_id,
resource, compliance_status,
params_list):
print("INFO: Updating params_list")
else:
print("ERROR: Issue observed while updating params_list")
return None, None
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part"
)
date_stamp = datetime.datetime.now().strftime('%m%d%y')
csv_filename = os.path.expanduser(
"~") + "/logs/p3_all_unsecured_images_list_" + date_stamp + ".csv"
headers = [
"Owner Id", "Tenant Id", "Tenant Name", "Tenant External Url",
"Image Id", "Image Name", "Visibility", "Status"
]
with open(csv_filename, 'a') as f:
file_is_empty = os.stat(csv_filename).st_size == 0
writer = csv.writer(f, lineterminator='\n')
if file_is_empty:
writer.writerow(headers)
writer.writerows(all_unsecured_images)
if len(all_unsecured_images) == 0:
print(
"INFO: All images test: Compliant(Tenant has no unsecured images)"
)
flag3 = "Compliant"
else:
print(
"INFO: All images test: Non-compliant(Tenant has unsecured images)"
)
flag3 = "Non-compliant"
return all_unsecured_images, flag3
except Exception as err:
print("ERROR: Failed to retrieve list of unsecured images due to %s" %
str(err))
return None, None
def list_unused_unsecured_images(unused_images, scan_id, team_id, scanid_valid,
teamid_valid):
"""
This method is to list the unused unsecured images.
:param unused_images: list_unused_images
:param scan_id: ScanID received from AWS SQS
:param team_id: TeamID
:param scanid_valid: Scan id
:param: teamid_valid: Team id
:return: Compliant | Non-Compliant | None
"""
try:
unused_unsecured_images = list()
for image in unused_images:
if image[5] != 'public':
unused_unsecured_images.append(image)
compliance_status = "Non-compliant"
else:
compliance_status = "Compliant"
resource = "Unused_Image:" + image[4]
if scanid_valid and teamid_valid:
if general_util.params_list_update(scan_id, tc, team_id,
resource, compliance_status,
params_list):
print("INFO: Updating params_list")
else:
print("ERROR: Issue observed while updating params_list")
return None, None
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part"
)
date_stamp = datetime.datetime.now().strftime('%m%d%y')
csv_filename = os.path.expanduser(
"~"
) + "/logs/p3_unused_unsecured_image_list_" + date_stamp + ".csv"
headers = [
"Image Owner Id", "Tenant Name", "Tenant External URL", "Image Id",
"Image Name", "Visibility", "Unused", "Direct URL",
"Image_Updated_on"
]
with open(csv_filename, 'a') as f:
file_is_empty = os.stat(csv_filename).st_size == 0
writer = csv.writer(f, lineterminator='\n')
if file_is_empty:
writer.writerow(headers)
writer.writerows(unused_unsecured_images)
if len(unused_unsecured_images) == 0:
print("INFO: There is no unused unsecured images")
flag2 = "Compliant"
else:
print("INFO: There are unused unsecured images")
flag2 = "Non-compliant"
return unused_unsecured_images, flag2
except Exception as err:
print(
"ERROR: Failed to retrieve list of unused unsecured images due to %s"
% str(err))
return None, None
def list_unsecured_servers(servers, scan_id, team_id, scanid_valid,
teamid_valid):
"""
This method is to list the servers with the unsecured images and their details.
:param servers: list_servers(conn, images, project_name, os_auth_url)
:param scan_id: ScanID received from AWS SQS
:param team_id: TeamID
:return: Compliant | Non-Compliant | None
"""
try:
unsecured_servers = list()
for server in servers:
if server[8] == False:
unsecured_servers.append(server)
compliance_status = "Non-compliant"
else:
compliance_status = "Compliant"
resource = "Instance: " + server[1]
if scanid_valid and teamid_valid:
if general_util.params_list_update(scan_id, tc, team_id,
resource, compliance_status,
params_list):
print("INFO: Updating params_list")
else:
print("ERROR: Issue observed while updating params_list")
return None
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part"
)
date_stamp = datetime.datetime.now().strftime('%m%d%y')
csv_filename = os.path.expanduser(
"~") + "/logs/p3_unsecured_servers_list_" + date_stamp + ".csv"
headers = [
"VM Id", "VM Name", "Tenant Id", "Tenant Name",
"Tenant External URL", "Image Id", "Image Owner Id", "Image Name",
"Secured", "Image Direct URL", "Image Updated Days",
"VM Availability Zone", "VM Updated Ago", "VM IP Address",
"VM Host Id", "VM User Id"
]
with open(csv_filename, 'a') as f:
file_is_empty = os.stat(csv_filename).st_size == 0
writer = csv.writer(f, lineterminator='\n')
if file_is_empty:
writer.writerow(headers)
writer.writerows(unsecured_servers)
if len(unsecured_servers) == 0:
print(
"INFO: VM test: Compliant(Unsecured image is not used in VM)")
flag1 = "Compliant"
else:
print(
"INFO: VM test: Non-compliant(Unsecured image is used in VM)")
flag1 = "Non-compliant"
return unsecured_servers, flag1
except Exception as err:
print(
"ERROR: Failed to retrieve list of unsecured servers list due to %s"
% str(err))
return None, None
def main(url, namespace, scan_id, team_id):
"""
Main method to start the audit over images used in the given NameSpace
:param url:hold the urls
:param namespace:holds the name of the project
:param scan_id:
:param team_id:
:return: Compliant | Non-compliant | None
"""
try:
summary_dict = {
'No_of_POD(s)_evaluated': 0,
'No_of_Image(s)_evaluated': 0,
'No_of_Unsecured_Image(s)_found': 0,
'No_of_POD(s)_using_unsecured_image(s)': 0,
'No_of_Tenants_with_unsecure_image(s)': 0
}
flag = "Compliant"
unsecured_tenant_count = 0
scanid_valid = False
teamid_valid = False
if scan_id and team_id is not None:
scanid_valid = common_lib.scanid_validation(scan_id)
teamid_valid = cae_lib.cae_teamid_validation(team_id)
else:
print("INFO: Valid ScanId or TeamId not found")
print("INFO: Execution will proceed without Kinesis update")
requests.packages.urllib3.disable_warnings()
print(
"INFO: Based on URL accepted, fetching respective Kube config file"
)
if url is not None:
session = general_util.session_handle()
if session:
if scanid_valid and teamid_valid:
print(
"INFO: Update the scan record with \"InProgress\" Status"
)
update = general_util.updateScanRecord(
session, "CAE", scan_id, team_id, tc, "InProgress")
if update is None:
raise Exception(
"INFO: Issue observed with UpdateScanRecord API call for \"InProgress\" status"
)
return None, summary_dict
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part"
)
create_csv_file_headers(csv_filename)
print(
"INFO: Check whether source of image used is a trusted one"
)
proj_txt = get_projects(namespace, url)
compliance_status = "Compliant"
non_compliant_str = 'Non-compliant'
if proj_txt is not None:
pods_list, pod_count = get_pods(namespace, url)
if pods_list is not None:
temp_pod_count = 0
pod_status_sub_str2 = ""
image_count = 0
unsecured_pod_count = 0
unsecured_image_count_total = 0
for pod in pods_list:
if temp_pod_count < 5:
metadata = pod.obj['metadata']
pod_name = none_check(
metadata.get('name', None))
pod_stat = pod.obj['status']
pod_status = none_check(
pod_stat.get('phase', None))
pod_status_sub_str = pod_name[:-5]
if pod_status_sub_str.lower() == pod_status_sub_str2.lower() \
and pod_status.lower() == "failed":
temp_pod_count += 1
else:
temp_pod_count -= 1
pod_status_sub_str2 = pod_name[:-5]
unsecured_image_count = 0
image_data, image_count, unsecured_image_count = get_image(
namespace, pod, url, compliance_status,
scan_id, team_id, scanid_valid,
teamid_valid, image_count,
unsecured_image_count)
unsecured_image_count_total = unsecured_image_count_total + unsecured_image_count
if unsecured_image_count > 0:
unsecured_pod_count += 1
if image_data is not None:
flag_txt = image_data[2]
if flag_txt.lower(
) == non_compliant_str.lower():
flag = non_compliant_str
unsecured_tenant_count = 1
else:
print(
"ERROR: Exiting after 5 unsuccessful retries"
)
break
summary_dict = build_summary_report(
pod_count, image_count,
unsecured_image_count_total, unsecured_pod_count,
unsecured_tenant_count)
else:
print("INFO: No Pods running in the project")
empty_metadata(namespace, url, compliance_status)
pod = 'NULL'
if scanid_valid and teamid_valid:
if general_util.params_list_update(
scan_id, tc, team_id, pod,
compliance_status, params_list):
print("INFO: Updating params_list")
else:
print(
"ERROR: Issue observed while updating params_list"
)
return None, summary_dict
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, "
"hence ignoring Kinesis part")
else:
update = general_util.updateScanRecord(
session, "CAE", scan_id, team_id, tc, "Failed")
if update is None:
raise Exception(
"ERROR: Issue observed with updateScanRecord API call in main method"
)
return None, summary_dict
proj_txt = [namespace] + ["Does not exist"] * 3
image_file_text = ["Does not exist"] * 9
print_metadata(image_file_text, proj_txt, csv_filename)
return None, summary_dict
else:
print("ERROR:URL not found")
return None, summary_dict
if scanid_valid and teamid_valid:
stream_info = general_util.add_result_to_stream(
session, "CAE", str(team_id), tc, params_list)
if stream_info is None:
raise Exception(
"ERROR: Issue observed while calling add_result_to_stream() API"
)
return None, summary_dict
seq_nums_list.append(stream_info)
print("INFO: Sending result complete")
send_result = general_util.send_result_complete(
session, "CAE", scan_id, team_id, tc, seq_nums_list)
if send_result:
print("INFO: Successfully submitted the result to Kinesis")
else:
print("INFO: Failed to submit the result to Kinesis")
return None, summary_dict
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, hence ignoring Kinesis part"
)
return flag, summary_dict
except Exception as e:
print("INFO: Failed to retrieve image list and pod list => %s %s" %
(str(e), summary_dict))
update = general_util.updateScanRecord(session, "CAE", scan_id,
team_id, tc, "Failed")
if update is None:
raise Exception(
"ERROR: Issue observed with updateScanRecord API call")
return None, summary_dict
raise Exception("ERROR: Failed to fetch either Projects" % str(e))
return None, summary_dict
def get_image(project_img, pod, url, compliance_status, scan_id, team_id,
scanid_valid, teamid_valid, image_count, unsecured_image_count):
"""
Method to get image details of application running on pod under specified project
:param project_img: holds Project Name
:param pod: holds POD Name
:param path: holds path to Kube config file
:return: image_data | None
"""
try:
api = cae_lib.load_config(url)
for i in range(0, 100):
try:
try:
print("INFO: Listing the images in the pods: %s " % pod)
pod_image = pykube.Pod.objects(api).filter(
namespace=project_img).get(name=pod)
except Exception as e:
pod_image = None
print(e)
print("ERROR: Unable to get image for the pod %s" % pod)
pass
if pod_image is not None:
metadata = pod_image.obj['metadata']
pod_name = none_check(metadata.get('name', None))
pod_namespace = none_check(metadata.get('namespace', None))
pod_stat = pod_image.obj['status']
pod_status = none_check(pod_stat.get('phase', None))
container_status = pod_image.obj['status']
container_info = container_status.get(
'containerStatuses', None)
if container_info is not None:
for container in container_info:
image_name = none_check(
container.get('image', None))
image_id = none_check(
container.get('imageID', None))
if image_id is not 'None' and image_id != 'None':
image_count += 1
try:
container_id = none_check(
container.get('containerID', None))
except KeyError:
container_id = 'None'
try:
container_state = container.get(
'state', {}).get('running', {})
container_start_date = none_check(
container_state.get('startedAt', None))
except KeyError:
container_start_date = 'None'
for container_list in pod_image.obj["spec"][
"containers"]:
print("INFO:checking the container_list")
image = none_check(
container_list.get('image', None))
compliance_status = compliance_status_validation(
image)
try:
ports = " "
for container_port in container_list[
'ports']:
container_exposed_port = str(
container_port.get(
'containerPort', None))
print("INFO:checking the ports")
ports = container_exposed_port + "/" + ports
except KeyError:
ports = 'None'
if scanid_valid and teamid_valid:
if container_id is not None and container_id != 'None':
""""updating params_list with pod name and last 5 digits of container ID"""
resource_name = str(pod) + "_" + str(
container_id.split("//")[1][:7])
else:
"""updating params_list with pod name """
resource_name = str(pod)
if general_util.params_list_update(
scan_id, tc, team_id, resource_name,
compliance_status, params_list):
print("INFO: Updating params_list")
else:
print(
"ERROR: Issue observed while updating params_list"
)
return None
else:
print(
"INFO: ScanId or TeamId passed to main() method is not valid, "
"hence ignoring Kinesis part")
image_file_text = [
pod_name, pod_namespace, pod_status,
container_id, image_name, image_id,
container_start_date, ports, compliance_status
]
proj_txt = get_projects(project_img, url)
image_data = [
image_file_text, proj_txt, compliance_status
]
build_metadata(image_data, csv_filename)
non_compliant_str = 'Non-compliant'
if compliance_status.lower(
) == non_compliant_str.lower():
unsecured_image_count += 1
""" Write a new CSV file when the image is Non-compliant """
create_csv_file_headers(csv_filename2)
build_metadata(image_data, csv_filename2)
else:
image_file_text = [
pod_name, pod_namespace, pod_status, 'None',
'None', 'None', 'None', 'None', compliance_status
]
proj_txt = get_projects(project_img, url)
image_data = [
image_file_text, proj_txt, compliance_status
]
build_metadata(image_data, csv_filename)
return image_data, image_count, unsecured_image_count
else:
print("INFO: No images found")
image_file_text = [
pod, 'Does not exist', 'Does not exist',
'Does not exist', 'Does not exist', 'Does not exist',
'Does not exist', 'Does not exist', 'Does not exist'
]
proj_txt = get_projects(project_img, url)
image_data = [image_file_text, proj_txt, 'Does not exist']
build_metadata(image_data, csv_filename)
return image_data, 0, 0
except Exception as e:
exception_str = str(e)
if '429' in exception_str:
time.sleep(5)
print(
"LOG:RETRYING When too many requests hit the cluster",
exception_str)
continue
break
except Exception as e:
print("ERROR: Failed to retrieve images with error => %s" % str(e))
return None, 0, 0