def get_ncr(scan_id: str, accounts: list, sheet_type: SheetTypes) -> list:
"""
Wrapper function for ddb queries to get ncr data.
Parameters:
scan_id (str): The id for the scan to lookup NCR data for.
accounts (list): All accounts for this spreadsheet.
global_scan (bool): Boolean value representing whether this is a global scan or not.
Returns:
list: List of NCR records.
"""
logger.debug('Getting NCRs')
scan_results = []
if sheet_type == SheetTypes.GLOBAL:
scan_results = ncr_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id)
)
else:
for account in accounts:
results = ncr_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id) &
Key('accntId_rsrceId_rqrmntId').begins_with(account['accountId'])
)
scan_results.extend(results)
return scan_results
def exclude_handler(event, context):
"""
Find and apply matching exclusion for all NCRs
Expected input event format
{
"scanId": scan_id,
}
"""
exclusion_types = config_table.get_config(config_table.EXCLUSIONS)
ncrs = ncr_table.query_all(KeyConditionExpression=Key('scanId').eq(event['openScan']['scanId']))
all_exclusions = exclusions_table.scan_all()
grouped_exclusions = group_exclusions(all_exclusions)
logger.info('Found %s exclusions', len(all_exclusions))
ncr_updated_count = 0
partial_exclusion_prioritizer = partial(exclusion_prioritizer, exclusion_types)
records = []
for ncr in ncrs:
matched_exclusions = match_exclusions(ncr, grouped_exclusions)
ncr_exclusion = pick_exclusion(matched_exclusions, partial_exclusion_prioritizer)
if ncr_exclusion:
updated_ncr = update_ncr_exclusion(ncr, ncr_exclusion, exclusion_types)
ncr_updated_count += 1
records.append(updated_ncr)
ncr_table.batch_put_records(records)
logger.info('Updated %s NCRs out of %s', ncr_updated_count, len(ncrs))
def test_populate_na_requirement(self, get_cs_requirements, s3_stubber, special_account):
"""Test populate with requirement that doesn't apply to account"""
cs_data = create_cloudsploit_data({}, [
{
'plugin': 'pluginNameOne',
'category': 'S3',
'title': 'Plugin Name One',
'resource': 'arn:aws:s3:::aaa-aaa-aaa-aaa',
'region': 'global',
'status': 'FAIL',
'statusNumber': 2,
'message': 'sample text'
},
])
get_cs_requirements.return_value = [{
'requirementId': '10',
'weight': Decimal(100),
'description': 'sample text',
'source': 'cloudsploit',
'severity': 'medium',
'onlyAppliesTo': ['other-than-special'],
'cloudsploit': {
'finding': 'Plugin Name One'
}
}]
s3_stubber.add_response('get_object', create_cloudsploit_s3_response(cs_data), {
'Bucket': os.getenv('CLOUDSPLOIT_RESULT_BUCKET'),
'Key': os.getenv('CLOUDSPLOIT_PREFIX') + '/' + special_account['accountId'] + '/latest.json',
})
scan_id = scans_table.create_new_scan_id()
cloudsploit_populate({'scanId': scan_id, 'accountId': special_account['accountId']}, {})
# check NCR record is created
ncr_records_in_db = ncr_table.query_all(KeyConditionExpression=Key('scanId').eq(scan_id))
assert ncr_records_in_db == []
# check score record is created
score_records_in_db = scores_table.query_all(KeyConditionExpression=Key('scanId').eq(scan_id))
for record in score_records_in_db:
assert record.pop('ttl')
assert score_records_in_db == [
scores_table.new_score(
scan_id,
special_account['accountId'],
{
'requirementId': '10',
'severity': 'medium',
'weight': Decimal(100)
},
scores_table.NOT_APPLICABLE,
scores_table.NOT_APPLICABLE,
)
]
def ncr_handler(event, context):
scan_id = event['scanId']
user = event['userRecord']
multivalue_querystring_parameters = event.get(
'multiValueQueryStringParameters') or {}
querystring_parameters = event.get('queryStringParameters') or {}
account_ids = multivalue_querystring_parameters.get('accountId', [])
requirement_id = querystring_parameters.get('requirementId', False)
logger.debug('Account Ids: %s', json.dumps(account_ids, default=str))
logger.debug('Requirement ID: %s', json.dumps(requirement_id, default=str))
authz.require_can_read_account(user, account_ids)
# get requirements
if requirement_id:
requirements = {requirement_id: requirements_table.get(requirement_id)}
else:
all_requirements = requirements_table.scan_all()
requirements = {}
for requirement in all_requirements:
requirements[requirement['requirementId']] = requirement
logger.debug('Requirements: %s', json.dumps(requirements, default=str))
ncr_records, to_parse = [], []
for account_id in account_ids:
if isinstance(requirement_id, str):
to_parse = ncr_table.query_all(
IndexName='by-scanId',
KeyConditionExpression=Key('scanId').eq(scan_id)
& Key('rqrmntId_accntId').eq('{}#{}'.format(
requirement_id, account_id)))
else:
to_parse = ncr_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id)
& Key('accntId_rsrceId_rqrmntId').begins_with(account_id))
logger.debug('To parse: %s', json.dumps(to_parse, default=str))
for item in to_parse:
ncr = prepare_allowed_actions_output(
initialize_output(scan_id, item), item, user, account_id,
requirements[item['requirementId']])
ncr_records.append(prepare_resource_output(ncr, item))
return {'ncrRecords': ncr_records}
def test_exclude_handler(self, exclusions_mock: Mock):
scan_id = scans_table.create_new_scan_id()
ncr = {
'scanId': scan_id,
'accntId_rsrceId_rqrmntId':
'12345678901_arn:aws:lambda:us-west-2:12345678901:function:test-function_requirementId01',
'accountId': '1234578901',
'accountName': 'TEST ACCOUNT NAME',
'requirementId': 'requirementId01',
'resourceId':
'arn:aws:lambda:us-west-2:12345678901:function:test-function',
'rqrmntId_accntId': 'requirementId01_12345678901',
}
ncr_table.put_item(Item=ncr)
exclusions_mock.return_value = [
{
'status': 'rejected',
'accountId': '*',
'requirementId': 'requirementId02',
'resourceId': '*',
'expirationDate': '2999/12/31'
},
{
'status': 'rejected',
'accountId': '1234578901',
'requirementId': 'requirementId01',
'resourceId': 'arn:aws:lambda:*',
'expirationDate': '2999/12/31'
},
{
'status': 'approved',
'accountId': '*',
'requirementId': 'requirementId01',
'resourceId': 'arn:aws:lambda:*',
'expirationDate': '2999/12/31',
'reason': 'inspected looks fine',
'type': 'justification',
},
]
event = {'openScan': {'scanId': scan_id}}
assert exclude.exclude_handler(event, {}) is None
updated_ncrs = sorted(
ncr_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id)),
key=lambda x: x['requirementId'],
)
expected_ncrs = [
exclude.update_ncr_exclusion(
ncr, exclusions_mock.return_value[2],
self.exclusion_types_effective_initial)
]
assert updated_ncrs == expected_ncrs
def score_calc_handler(event, context):
"""
:param event: {
scanId: string,
accountIds: list of accountIds,
}
:param context: dict
:return: None
"""
scan_id = event['openScan']['scanId']
account_ids = event['load']['accountIds']
date = scan_id[0:10]
all_scores_to_put = []
all_account_scores = []
for account_id in account_ids:
account_name = accounts_table.get_account(account_id).get('account_name')
scores_to_put = {
record['requirementId']: record
for record in scores_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id) & Key('accntId_rqrmntId').begins_with(account_id)
)
}
existing_ncr_records = ncr_table.query_all(
KeyConditionExpression=Key('scanId').eq(scan_id) & Key('accntId_rsrceId_rqrmntId').begins_with(account_id),
)
grouped_ncr_data = defaultdict(list)
for ncr_object in existing_ncr_records:
grouped_ncr_data[ncr_object['requirementId']].append(ncr_object)
for requirement_object in all_requirements:
severity = requirement_object['severity']
record_to_edit = scores_to_put.get(requirement_object['requirementId'], False)
if record_to_edit is False:
continue # data not collected for this account for this scan for this requirement, moving on
score_object = record_to_edit['score'][severity]
# check if score is DNC if so we skip counting failing resources
if scores_table.DATA_NOT_COLLECTED in score_object.values():
continue
if score_object['numFailing'] is None:
score_object['numFailing'] = Decimal(0)
matching_ncrs = grouped_ncr_data.get(requirement_object['requirementId'], [])
for ncr_record in matching_ncrs:
is_excluded = ncr_record.get('exclusionApplied', False)
# TODO handle hidden ncr's also (decrement numResources)
if is_excluded:
continue
else:
score_object['numFailing'] += 1
all_scores_to_put.append(record_to_edit)
account_score = {
'accountId': account_id,
'accountName': account_name,
'date': date,
'scanId': scan_id,
'score': scores_table.weighted_score_aggregate_calc(scores_to_put.values())
}
all_account_scores.append(account_score)
scores_table.batch_put_records(all_scores_to_put)
account_scores_table.batch_put_records(all_account_scores)
def test_populate(self, get_cs_requirements, s3_stubber, regular_account):
cs_data = create_cloudsploit_data({}, [
{
'plugin': 'pluginNameOne',
'category': 'S3',
'title': 'Plugin Name One',
'resource': 'arn:aws:s3:::aaa-aaa-aaa-aaa',
'region': 'global',
'status': 'FAIL',
'statusNumber': 2,
'message': 'sample text'
},
{
'plugin': 'pluginNameOne',
'category': 'S3',
'title': 'Plugin Name Two',
'resource': 'arn:aws:s3:::aaa-aaa-aaa-aaa',
'region': 'global',
'status': 'FAIL',
'statusNumber': 2,
'message': 'sample text'
},
{
'plugin': 'pluginNameOne',
'category': 'S3',
'title': 'Plugin Name Two',
'resource': 'arn:aws:s3:::aaa-aaa-aaa-aaa',
'region': 'global',
'status': 'FAIL',
'statusNumber': 2,
'message': 'sample text2'
},
])
get_cs_requirements.return_value = [{
'requirementId': '10',
'weight': Decimal(100),
'description': 'sample text',
'source': 'cloudsploit',
'severity': 'medium',
'service': 'organizations',
'component': 'account',
'cloudsploit': {
'finding': [
'Plugin Name One',
'Plugin Name Two',
]
}
}]
s3_stubber.add_response('get_object', create_cloudsploit_s3_response(cs_data), {
'Bucket': os.getenv('CLOUDSPLOIT_RESULT_BUCKET'),
'Key': os.getenv('CLOUDSPLOIT_PREFIX') + '/' + regular_account['accountId'] + '/latest.json',
})
scan_id = scans_table.create_new_scan_id()
cloudsploit_populate({'scanId': scan_id, 'accountId': regular_account['accountId']}, {})
# check NCR record is created
ncr_records_in_db = ncr_table.query_all(KeyConditionExpression=Key('scanId').eq(scan_id))
for record in ncr_records_in_db:
assert record.pop('ttl')
assert ncr_records_in_db == [
ncr_table.new_ncr_record({
'accountId': regular_account['accountId'],
'accountName': regular_account['account_name'],
'cloudsploitStatus': 'FAIL',
'requirementId': '10',
'resourceId': 'aaa-aaa-aaa-aaa',
'resourceType': 'organizations-account',
'region': 'global',
'reason': 'sample text\nsample text2',
}, scan_id)
]
# check score record is created
score_records_in_db = scores_table.query_all(KeyConditionExpression=Key('scanId').eq(scan_id))
for record in score_records_in_db:
assert record.pop('ttl')
assert score_records_in_db == [
scores_table.new_score(
scan_id,
regular_account['accountId'],
{
'requirementId': '10',
'severity': 'medium',
'weight': Decimal(100)
},
Decimal(3)
)
]