def create_delete_model(record): """Create a security group model from a record.""" data = cloudwatch.get_historical_base_info(record) group_id = cloudwatch.filter_request_parameters('groupId', record) vpc_id = cloudwatch.filter_request_parameters('vpcId', record) group_name = cloudwatch.filter_request_parameters('groupName', record) arn = get_arn(group_id, record['account']) log.debug('[-] Deleting Dynamodb Records. Hash Key: {arn}'.format(arn=arn)) # tombstone these records so that the deletion event time can be accurately tracked. data.update({ 'configuration': {} }) items = list(CurrentSecurityGroupModel.query(arn, limit=1)) if items: model_dict = items[0].__dict__['attribute_values'].copy() model_dict.update(data) model = CurrentSecurityGroupModel(**model_dict) model.save() return model
def capture_update_records(records): """Writes all updated configuration info to DynamoDB""" for record in records: data = cloudwatch.get_historical_base_info(record) group = describe_group(record) if len(group) > 1: raise Exception('[X] Multiple groups found. Record: {record}'.format(record=record)) if not group: log.warning('[?] No group information found. Record: {record}'.format(record=record)) continue group = group[0] # determine event data for group log.debug('Processing group. Group: {}'.format(group)) data.update({ 'GroupId': group['GroupId'], 'GroupName': group['GroupName'], 'Description': group['Description'], 'VpcId': group.get('VpcId'), 'Tags': group.get('Tags', []), 'arn': get_arn(group['GroupId'], group['OwnerId']), 'OwnerId': group['OwnerId'], 'configuration': group, 'Region': cloudwatch.get_region(record) }) log.debug('Writing Dynamodb Record. Records: {record}'.format(record=data)) current_revision = CurrentSecurityGroupModel(**data) current_revision.save()
def test_collector(historical_role, mock_lambda_environment, historical_sqs, security_groups, current_security_group_table): from historical.security_group.models import CurrentSecurityGroupModel from historical.security_group.collector import handler event = CloudwatchEventFactory(detail=DetailFactory( requestParameters={'groupId': security_groups['GroupId']}, eventName='CreateSecurityGroup'), ) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, None) assert CurrentSecurityGroupModel.count() == 1 event = CloudwatchEventFactory(detail=DetailFactory( requestParameters={'groupId': security_groups['GroupId']}, eventName='DeleteSecurityGroup'), ) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, None) assert CurrentSecurityGroupModel.count() == 0
def test_current_table(current_security_group_table): from historical.security_group.models import CurrentSecurityGroupModel CurrentSecurityGroupModel(**SECURITY_GROUP).save() items = list(CurrentSecurityGroupModel.query('arn:aws:ec2:us-east-1:123456789012:security-group/sg-1234568')) assert len(items) == 1 assert isinstance(items[0].ttl, int) assert items[0].ttl > 0
def current_security_group_table(): from historical.security_group.models import CurrentSecurityGroupModel mock_dynamodb2().start() yield CurrentSecurityGroupModel.create_table(read_capacity_units=1, write_capacity_units=1, wait=True) mock_dynamodb2().stop()
def capture_update_records(records): """Writes all updated configuration info to DynamoDB""" for rec in records: data = cloudwatch.get_historical_base_info(rec) group = describe_group(rec, cloudwatch.get_region(rec)) if len(group) > 1: raise Exception(f'[X] Multiple groups found. Record: {rec}') if not group: LOG.warning(f'[?] No group information found. Record: {rec}') continue group = group[0] # Determine event data for group - and pop off items that are going to the top-level: LOG.debug(f'Processing group. Group: {group}') data.update({ 'GroupId': group['GroupId'], 'GroupName': group.pop('GroupName'), 'VpcId': group.pop('VpcId', None), 'arn': get_arn(group.pop('GroupId'), cloudwatch.get_region(rec), group.pop('OwnerId')), 'Region': cloudwatch.get_region(rec) }) data['Tags'] = pull_tag_dict(group) # Set the remaining items to the configuration: data['configuration'] = group # Set the version: data['version'] = VERSION LOG.debug(f'[+] Writing Dynamodb Record. Records: {data}') current_revision = CurrentSecurityGroupModel(**data) current_revision.save()
def test_collector(historical_role, mock_lambda_environment, historical_sqs, security_groups, current_security_group_table): """Tests the Collector.""" # This should NOT be called at first: def mock_describe_security_groups(**kwargs): assert False patch_sgs = patch('historical.security_group.collector.describe_security_groups', mock_describe_security_groups) patch_sgs.start() from historical.security_group.models import CurrentSecurityGroupModel from historical.security_group.collector import handler from cloudaux.aws.ec2 import describe_security_groups sg_details = describe_security_groups( account_number='012345678910', assume_role='Historical', region='us-east-1', GroupIds=[security_groups['GroupId']])['SecurityGroups'][0] event = CloudwatchEventFactory( detail=DetailFactory( requestParameters={'groupId': security_groups['GroupId']}, eventName='PollSecurityGroups', collected=sg_details)) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, mock_lambda_environment) patch_sgs.stop() group = list(CurrentSecurityGroupModel.scan()) assert len(group) == 1 # Validate that Tags are correct: assert len(group[0].Tags.attribute_values) == 2 assert group[0].Tags.attribute_values['Some'] == 'Value' assert group[0].Tags.attribute_values['Empty'] == '<empty>' group[0].delete() # Standard SG events: event = CloudwatchEventFactory( detail=DetailFactory( requestParameters={'groupId': security_groups['GroupId']}, eventName='CreateSecurityGroup' ), ) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, mock_lambda_environment) group = list(CurrentSecurityGroupModel.scan()) assert len(group) == 1 # Validate that Tags are correct: assert len(group[0].Tags.attribute_values) == 2 assert group[0].Tags.attribute_values['Some'] == 'Value' assert group[0].Tags.attribute_values['Empty'] == '<empty>' event = CloudwatchEventFactory( detail=DetailFactory( requestParameters={'groupId': security_groups['GroupId']}, eventName='DeleteSecurityGroup' ), ) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, mock_lambda_environment) assert CurrentSecurityGroupModel.count() == 0 # Try to get it again -- this time, add the SG ID to the responseElements: event = CloudwatchEventFactory( detail=DetailFactory( responseElements={'groupId': security_groups['GroupId']}, eventName='CreateSecurityGroup' ), ) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, mock_lambda_environment) assert CurrentSecurityGroupModel.count() == 1 # Create a security group in an off-region. Make sure that the ARN of the Security Group is correct and NOT # set to the CURRENT_REGION: client = boto3.client('ec2', region_name='eu-west-2') sg_id = client.create_security_group(GroupName='London', Description='London', VpcId='vpc-test')['GroupId'] sg_details = describe_security_groups( account_number='123456789012', assume_role='Historical', region='eu-west-2', GroupIds=[sg_id])['SecurityGroups'][0] event = CloudwatchEventFactory( detail=DetailFactory( requestParameters={'groupId': sg_id}, eventName='PollSecurityGroups', awsRegion='eu-west-2', collected=sg_details)) data = json.dumps(event, default=serialize) data = RecordsFactory(records=[SQSDataFactory(body=data)]) data = json.dumps(data, default=serialize) data = json.loads(data) handler(data, mock_lambda_environment) group = list(CurrentSecurityGroupModel.query(f'arn:aws:ec2:eu-west-2:123456789012:security-group/{sg_id}')) assert len(group) == 1