def create_delete_model(record):
"""Create a security group model from a record."""
data = cloudwatch.get_historical_base_info(record)
group_id = cloudwatch.filter_request_parameters('groupId', record)
vpc_id = cloudwatch.filter_request_parameters('vpcId', record)
group_name = cloudwatch.filter_request_parameters('groupName', record)
arn = get_arn(group_id, record['account'])
log.debug('[-] Deleting Dynamodb Records. Hash Key: {arn}'.format(arn=arn))
# tombstone these records so that the deletion event time can be accurately tracked.
data.update({
'configuration': {}
})
items = list(CurrentSecurityGroupModel.query(arn, limit=1))
if items:
model_dict = items[0].__dict__['attribute_values'].copy()
model_dict.update(data)
model = CurrentSecurityGroupModel(**model_dict)
model.save()
return model
def capture_update_records(records):
"""Writes all updated configuration info to DynamoDB"""
for record in records:
data = cloudwatch.get_historical_base_info(record)
group = describe_group(record)
if len(group) > 1:
raise Exception('[X] Multiple groups found. Record: {record}'.format(record=record))
if not group:
log.warning('[?] No group information found. Record: {record}'.format(record=record))
continue
group = group[0]
# determine event data for group
log.debug('Processing group. Group: {}'.format(group))
data.update({
'GroupId': group['GroupId'],
'GroupName': group['GroupName'],
'Description': group['Description'],
'VpcId': group.get('VpcId'),
'Tags': group.get('Tags', []),
'arn': get_arn(group['GroupId'], group['OwnerId']),
'OwnerId': group['OwnerId'],
'configuration': group,
'Region': cloudwatch.get_region(record)
})
log.debug('Writing Dynamodb Record. Records: {record}'.format(record=data))
current_revision = CurrentSecurityGroupModel(**data)
current_revision.save()
def test_collector(historical_role, mock_lambda_environment, historical_sqs,
security_groups, current_security_group_table):
from historical.security_group.models import CurrentSecurityGroupModel
from historical.security_group.collector import handler
event = CloudwatchEventFactory(detail=DetailFactory(
requestParameters={'groupId': security_groups['GroupId']},
eventName='CreateSecurityGroup'), )
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, None)
assert CurrentSecurityGroupModel.count() == 1
event = CloudwatchEventFactory(detail=DetailFactory(
requestParameters={'groupId': security_groups['GroupId']},
eventName='DeleteSecurityGroup'), )
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, None)
assert CurrentSecurityGroupModel.count() == 0
def test_current_table(current_security_group_table):
from historical.security_group.models import CurrentSecurityGroupModel
CurrentSecurityGroupModel(**SECURITY_GROUP).save()
items = list(CurrentSecurityGroupModel.query('arn:aws:ec2:us-east-1:123456789012:security-group/sg-1234568'))
assert len(items) == 1
assert isinstance(items[0].ttl, int)
assert items[0].ttl > 0
def current_security_group_table():
from historical.security_group.models import CurrentSecurityGroupModel
mock_dynamodb2().start()
yield CurrentSecurityGroupModel.create_table(read_capacity_units=1,
write_capacity_units=1,
wait=True)
mock_dynamodb2().stop()
def capture_update_records(records):
"""Writes all updated configuration info to DynamoDB"""
for rec in records:
data = cloudwatch.get_historical_base_info(rec)
group = describe_group(rec, cloudwatch.get_region(rec))
if len(group) > 1:
raise Exception(f'[X] Multiple groups found. Record: {rec}')
if not group:
LOG.warning(f'[?] No group information found. Record: {rec}')
continue
group = group[0]
# Determine event data for group - and pop off items that are going to the top-level:
LOG.debug(f'Processing group. Group: {group}')
data.update({
'GroupId':
group['GroupId'],
'GroupName':
group.pop('GroupName'),
'VpcId':
group.pop('VpcId', None),
'arn':
get_arn(group.pop('GroupId'), cloudwatch.get_region(rec),
group.pop('OwnerId')),
'Region':
cloudwatch.get_region(rec)
})
data['Tags'] = pull_tag_dict(group)
# Set the remaining items to the configuration:
data['configuration'] = group
# Set the version:
data['version'] = VERSION
LOG.debug(f'[+] Writing Dynamodb Record. Records: {data}')
current_revision = CurrentSecurityGroupModel(**data)
current_revision.save()
def test_collector(historical_role, mock_lambda_environment, historical_sqs, security_groups,
current_security_group_table):
"""Tests the Collector."""
# This should NOT be called at first:
def mock_describe_security_groups(**kwargs):
assert False
patch_sgs = patch('historical.security_group.collector.describe_security_groups', mock_describe_security_groups)
patch_sgs.start()
from historical.security_group.models import CurrentSecurityGroupModel
from historical.security_group.collector import handler
from cloudaux.aws.ec2 import describe_security_groups
sg_details = describe_security_groups(
account_number='012345678910',
assume_role='Historical',
region='us-east-1',
GroupIds=[security_groups['GroupId']])['SecurityGroups'][0]
event = CloudwatchEventFactory(
detail=DetailFactory(
requestParameters={'groupId': security_groups['GroupId']},
eventName='PollSecurityGroups',
collected=sg_details))
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, mock_lambda_environment)
patch_sgs.stop()
group = list(CurrentSecurityGroupModel.scan())
assert len(group) == 1
# Validate that Tags are correct:
assert len(group[0].Tags.attribute_values) == 2
assert group[0].Tags.attribute_values['Some'] == 'Value'
assert group[0].Tags.attribute_values['Empty'] == '<empty>'
group[0].delete()
# Standard SG events:
event = CloudwatchEventFactory(
detail=DetailFactory(
requestParameters={'groupId': security_groups['GroupId']},
eventName='CreateSecurityGroup'
),
)
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, mock_lambda_environment)
group = list(CurrentSecurityGroupModel.scan())
assert len(group) == 1
# Validate that Tags are correct:
assert len(group[0].Tags.attribute_values) == 2
assert group[0].Tags.attribute_values['Some'] == 'Value'
assert group[0].Tags.attribute_values['Empty'] == '<empty>'
event = CloudwatchEventFactory(
detail=DetailFactory(
requestParameters={'groupId': security_groups['GroupId']},
eventName='DeleteSecurityGroup'
),
)
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, mock_lambda_environment)
assert CurrentSecurityGroupModel.count() == 0
# Try to get it again -- this time, add the SG ID to the responseElements:
event = CloudwatchEventFactory(
detail=DetailFactory(
responseElements={'groupId': security_groups['GroupId']},
eventName='CreateSecurityGroup'
),
)
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, mock_lambda_environment)
assert CurrentSecurityGroupModel.count() == 1
# Create a security group in an off-region. Make sure that the ARN of the Security Group is correct and NOT
# set to the CURRENT_REGION:
client = boto3.client('ec2', region_name='eu-west-2')
sg_id = client.create_security_group(GroupName='London', Description='London', VpcId='vpc-test')['GroupId']
sg_details = describe_security_groups(
account_number='123456789012',
assume_role='Historical',
region='eu-west-2',
GroupIds=[sg_id])['SecurityGroups'][0]
event = CloudwatchEventFactory(
detail=DetailFactory(
requestParameters={'groupId': sg_id},
eventName='PollSecurityGroups',
awsRegion='eu-west-2',
collected=sg_details))
data = json.dumps(event, default=serialize)
data = RecordsFactory(records=[SQSDataFactory(body=data)])
data = json.dumps(data, default=serialize)
data = json.loads(data)
handler(data, mock_lambda_environment)
group = list(CurrentSecurityGroupModel.query(f'arn:aws:ec2:eu-west-2:123456789012:security-group/{sg_id}'))
assert len(group) == 1