def cloudsploit_error(event, context):
"""
Handle errors in cloudsploit setup or populate. Invoked by step function.
Adds a DNC score for each cloudsploit based requirement for the account
that failed. Also records error in scan record.
Expected input event format
{
'accountId': account_id,
'scanId': scan_id,
}
"""
account_id = event['accountId']
scan_id = event['scanId']
error = event['error']
# remove traceback for cloudsploit errors
try:
error['Cause'] = json.dumps({
**json.loads(error['Cause']), 'trace': None
})
except: # pylint: disable=bare-except
pass
account = accounts_table.get_account(account_id)
cloudsploit_based_requirements = requirements_table.get_cloudsploit_based_requirements(
)
applying_requirements, not_applying_requirements = split_requirements(
cloudsploit_based_requirements, account)
scores_to_put = []
for requirement in applying_requirements:
scores_to_put.append(
scores_table.new_score(
scan_id,
account_id,
requirement,
scores_table.DATA_NOT_COLLECTED,
scores_table.DATA_NOT_COLLECTED,
))
for requirement in not_applying_requirements:
scores_to_put.append(
scores_table.new_score(
scan_id,
account_id,
requirement,
scores_table.NOT_APPLICABLE,
scores_table.NOT_APPLICABLE,
))
scores_table.batch_put_records(scores_to_put)
scans_table.add_error(scan_id, context.function_name, event['error'])
def account_detailed_scores_handler(event, context):
"""
:param event:
:param context:
:raises HttpInvalidException if missing 'scanId' or 'accountIds':
:return account scores of all account_ids in event:
"""
try:
account_ids = event['pathParameters']['accountIds']
scan_id = event['scanId']
except KeyError:
raise HttpInvalidException(
'account ids or scan id not found in request')
accounts = []
account_ids = urllib.parse.unquote(account_ids).split(',')
require_can_read_account(event['userRecord'], account_ids)
for account_id in account_ids:
try:
account_name = accounts_table.get_account(account_id).get(
'account_name', account_id)
except KeyError:
raise HttpNotFoundException(
f'account record not found for {account_id}')
if len(account_id) > 0:
detailed_score = {
'accountId': account_id,
'accountName': account_name,
}
requirements = []
to_parse = scores_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id)
& Key('accntId_rqrmntId').begins_with(account_id), )
for item in to_parse:
requirements.append({
'requirementId': item['requirementId'],
'score': item['score']
})
detailed_score['requirementsScores'] = requirements
accounts.append(detailed_score)
return {'accounts': accounts}
def cloudsploit_setup(event, context):
"""
Create parameters for external cloudsploit scanning lambda based on
account settings and requirement definition.
Expected input event format
{
'accountId': account_id,
'scanId': scan_id,
'cloudsploitSettingsMap': {
'default': {...},
'settings1': {...},
...
}
}
"""
account = accounts_table.get_account(event['accountId'])
if 'cross_account_role' not in account:
raise ValueError(
f'cross_account_role not specified in account {event["accountId"]}'
)
cross_account_role = account.get('cross_account_role', None)
if 'scorecard_profile' in account:
try:
settings = event['cloudsploitSettingsMap'][
account['scorecard_profile']]
except KeyError:
raise KeyError(
f'cloudsploit settings {account["scorecard_profile"]} specified in {event} does not exist'
)
else:
settings = event['cloudsploitSettingsMap']['default']
return {
'cloud': 'aws',
'cloudConfig': {
'roleArn': cross_account_role
},
'settings': settings,
's3Prefix': event['accountId']
}
def cloudsploit_populate(event, context):
"""
Import cloudsploit findings from s3 and convert to NCRs based on
requirement definitions
Expected input event format
{
scanId: string,
accountId: string,
}
"""
account_id = event['accountId']
scan_id = event['scanId']
if not isinstance(scan_id, str):
raise TypeError(f'scanId should be str, not {type(scan_id)}')
account = accounts_table.get_account(account_id)
# keys here will be the ncr's sort key which will uniquely identify it
# since the partion key will be the scanId for all ncrs created.
all_ncrs = {}
scores_to_put = []
cloudsploit_based_requirements = requirements_table.get_cloudsploit_based_requirements(
)
# split requirements based on whether they apply to the account
applying_requirements, not_applying_requirements = split_requirements(
cloudsploit_based_requirements, account)
s3_key = cloudsploit_prefix + '/' + account_id + '/latest.json'
# load cloudsploit results
logger.info('Getting Cloudsploit results from s3://%s/%s', bucket_name,
s3_key)
response = S3.get_object(Bucket=bucket_name, Key=s3_key)
object_expiration_check(response, s3_key)
result = json.loads(response['Body'].read())
# group cloudsploit results by finding
grouped_results_data = defaultdict(list)
for result_object in result['resultsData']:
grouped_results_data[result_object['title']].append(result_object)
# add NCRs based on cloudsploit requirements
for requirement in applying_requirements:
requirement_titles = requirement['cloudsploit']['finding']
if isinstance(requirement_titles, str):
requirement_titles = [requirement_titles]
relevant_results = []
for requirement_title in requirement_titles:
relevant_results.extend(
grouped_results_data.get(requirement_title, []))
failing_statuses = determine_failing_statuses(requirement)
# create NCR for each cloudsploit result
for finding_object in relevant_results:
if finding_object['status'] in failing_statuses:
resource_type = None
# set resource if cloudsploit didn't provide one
if finding_object[
'resource'] == ncr_table.CLOUDSPLOIT_FINDING_NA and requirement[
'cloudsploit'].get('regional', False) is True:
resource_id = finding_object['region']
elif finding_object[
'resource'] == ncr_table.CLOUDSPLOIT_FINDING_NA:
resource_id = str(account_id)
else:
(resource_type, resource_id) = extract_resource_id(
finding_object['resource'])
# Set the resource type based on the requirement spec (not cloudsploit), if it wasn't overridden by extract_resource_id()
if resource_type is None:
resource_type = f'{requirement["service"]}-{requirement["component"]}'
ncr_key = ncr_table.create_sort_key(
account_id, resource_id, requirement['requirementId'])
if ncr_key in all_ncrs:
all_ncrs[ncr_key]['reason'][
finding_object['message']] = None
else:
all_ncrs[ncr_key] = ncr_table.new_ncr_record(
{
'accountId': account_id,
'accountName': account['account_name'],
'requirementId': requirement['requirementId'],
'resourceId': resource_id,
'resourceType': resource_type,
'region': finding_object['region'],
'reason': {
finding_object['message']: None
}, # dict as set with 1 item (set to deduplicate reasons)
'cloudsploitStatus': finding_object[
'status'] # Send the cloudsploit status to the NCR Record
},
scan_id)
# determine number of resources for score
if requirement['cloudsploit'].get('source'):
service_name, api_call = requirement['cloudsploit'].get(
'source').split('.')
cloudsploit_data = result['collectionData']['aws'][service_name][
api_call]
try:
num_resources = sum(
len(region_object['data'])
for region_object in cloudsploit_data.values()
if 'err' not in region_object)
except KeyError:
raise RuntimeError(
f'{service_name}, {api_call} collectionData contained region object lacking both "data" and "err" keys"'
)
else:
num_resources = len(relevant_results)
scores_to_put.append(
scores_table.new_score(scan_id, account_id, requirement,
num_resources))
# add N/A score for requirements that don't apply
for requirement in not_applying_requirements:
scores_to_put.append(
scores_table.new_score(scan_id, account_id, requirement,
scores_table.NOT_APPLICABLE,
scores_table.NOT_APPLICABLE))
for ncr in all_ncrs.values():
ncr['reason'] = '\n'.join(ncr['reason'].keys())
logger.info('Adding ncrs: %s', json.dumps(all_ncrs))
ncr_table.batch_put_records(all_ncrs.values())
scores_table.batch_put_records(scores_to_put)
def score_calc_handler(event, context):
"""
:param event: {
scanId: string,
accountIds: list of accountIds,
}
:param context: dict
:return: None
"""
scan_id = event['openScan']['scanId']
account_ids = event['load']['accountIds']
date = scan_id[0:10]
all_scores_to_put = []
all_account_scores = []
for account_id in account_ids:
account_name = accounts_table.get_account(account_id).get('account_name')
scores_to_put = {
record['requirementId']: record
for record in scores_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id) & Key('accntId_rqrmntId').begins_with(account_id)
)
}
existing_ncr_records = ncr_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id) & Key('accntId_rsrceId_rqrmntId').begins_with(account_id),
)
grouped_ncr_data = defaultdict(list)
for ncr_object in existing_ncr_records:
grouped_ncr_data[ncr_object['requirementId']].append(ncr_object)
for requirement_object in all_requirements:
severity = requirement_object['severity']
record_to_edit = scores_to_put.get(requirement_object['requirementId'], False)
if record_to_edit is False:
continue # data not collected for this account for this scan for this requirement, moving on
score_object = record_to_edit['score'][severity]
# check if score is DNC if so we skip counting failing resources
if scores_table.DATA_NOT_COLLECTED in score_object.values():
continue
if score_object['numFailing'] is None:
score_object['numFailing'] = Decimal(0)
matching_ncrs = grouped_ncr_data.get(requirement_object['requirementId'], [])
for ncr_record in matching_ncrs:
is_excluded = ncr_record.get('exclusionApplied', False)
# TODO handle hidden ncr's also (decrement numResources)
if is_excluded:
continue
else:
score_object['numFailing'] += 1
all_scores_to_put.append(record_to_edit)
account_score = {
'accountId': account_id,
'accountName': account_name,
'date': date,
'scanId': scan_id,
'score': scores_table.weighted_score_aggregate_calc(scores_to_put.values())
}
all_account_scores.append(account_score)
scores_table.batch_put_records(all_scores_to_put)
account_scores_table.batch_put_records(all_account_scores)