def test_get_regions_ignore_instances():
"""Test for method of the same name."""
found_instances = utils.get_regions(must_contain_instances=False)
expected_regions = ['eu-west-1', 'sa-east-1', 'us-east-1',
'ap-northeast-1', 'us-west-2', 'us-west-1']
for expected_region in expected_regions:
assert expected_region in found_instances
def test_get_regions_with_instances():
"""Test for method of the same name."""
client = boto3.client('ec2', region_name='us-west-2')
# toss an instance in us-west-2
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
# be sure we get us-west-2 *only*
assert ['us-west-2'] == utils.get_regions(must_contain_instances=True)
def test_get_regions_with_instances_or_snapshots():
"""Test for method of the same name."""
client_uswest2 = boto3.client('ec2', region_name='us-west-2')
# toss an instance in us-west-2
client_uswest2.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
# be sure we get us-west-2 *only*
assert ['us-west-2'] == utils.get_regions(must_contain_instances=True,
must_contain_snapshots=False)
# now say we don't filter by instances, be sure we get a lot of regions
found_regions = utils.get_regions(must_contain_instances=False,
must_contain_snapshots=False)
expected_regions = [
'eu-west-1', 'sa-east-1', 'us-east-1', 'ap-northeast-1', 'us-west-2',
'us-west-1'
]
for expected_region in expected_regions:
assert expected_region in found_regions
# now take a snapshot
client_uswest1 = boto3.client('ec2', region_name='us-west-1')
volume_results = client_uswest1.create_volume(
Size=100, AvailabilityZone='us-west-1a')
client_uswest1.create_snapshot(VolumeId=volume_results['VolumeId'])
# be sure that snapshot filter works and only returns the snapshot region
assert ['us-west-1'] == utils.get_regions(must_contain_instances=False,
must_contain_snapshots=True)
# now filter by both, should be nothing returned
found_regions = utils.get_regions(must_contain_instances=True,
must_contain_snapshots=True)
assert len(found_regions) == 0
# now snap in us-west-2 where we have an instance as well
volume_results2 = client_uswest2.create_volume(
Size=100, AvailabilityZone='us-west-2a')
client_uswest2.create_snapshot(VolumeId=volume_results2['VolumeId'])
found_regions = utils.get_regions(must_contain_instances=True,
must_contain_snapshots=True)
assert found_regions == ['us-west-2']
def perform_fanout_all_regions(context, cli=False):
"""For every region, run the supplied function"""
# get regions, regardless of instances
sns_topic = utils.get_topic_arn('CleanSnapshotTopic')
LOG.debug('perform_fanout_all_regions using SNS topic %s', sns_topic)
regions = utils.get_regions(must_contain_instances=True)
for region in regions:
sleep(5) # API limit relief
send_fanout_message(context,
region=region,
topic_arn=sns_topic,
cli=cli)
LOG.info('Function clean_perform_fanout_all_regions completed')
def perform_fanout_all_regions(context, cli=False):
"""For every region, send a message (lambda) or run replication (cli)"""
sns_topic = utils.get_topic_arn('ReplicationSnapshotTopic')
LOG.debug('perform_fanout_all_regions using SNS topic %s', sns_topic)
# get regions with instances running or stopped
regions = utils.get_regions(must_contain_snapshots=True)
for region in regions:
sleep(5) # API rate limiting help
send_fanout_message(context=context,
region=region,
sns_topic=sns_topic,
cli=cli)
def perform_fanout_all_regions(context, cli=False):
"""For every region, run the supplied function"""
sns_topic = utils.get_topic_arn('CreateSnapshotTopic')
LOG.debug('perform_fanout_all_regions using SNS topic %s', sns_topic)
# get regions with instances running or stopped
regions = utils.get_regions(must_contain_instances=True)
for region in regions:
sleep(5) # API rate limiting help
send_fanout_message(context=context,
region=region,
sns_topic=sns_topic,
cli=cli)
def perform_fanout_all_regions(context, cli=False, installed_region='us-east-1'):
"""For every region, send a message (lambda) or run snapshots (cli)"""
sns_topic = utils.get_topic_arn('CreateSnapshotTopic')
LOG.debug('perform_fanout_all_regions using SNS topic %s', sns_topic)
# configure replication based on extant configs for snapshots
if type(context) is not MockContext: # don't do in unit tests
ensure_cloudwatch_rule_for_replication(context, installed_region)
# get regions with instances running or stopped
regions = utils.get_regions(must_contain_instances=True)
for region in regions:
sleep(5) # API rate limiting help
send_fanout_message(
context=context,
region=region,
sns_topic=sns_topic,
cli=cli)
def test_perform_fanout_all_regions_clean(mocker):
"""Test for method of the same name."""
mocks.create_sns_topic('CleanSnapshotTopic')
mocks.create_dynamodb()
expected_regions = utils.get_regions()
for r in expected_regions: # must have an instance in the region to clean it
mocks.create_instances(region=r)
expected_sns_topic = utils.get_topic_arn('CleanSnapshotTopic', 'us-east-1')
ctx = utils.MockContext()
mocker.patch('ebs_snapper.clean.send_fanout_message')
# fan out, and be sure we touched every region
clean.perform_fanout_all_regions(ctx)
for r in expected_regions:
clean.send_fanout_message.assert_any_call( # pylint: disable=E1103
ctx,
cli=False,
region=r,
topic_arn=expected_sns_topic)
def sanity_check(context, installed_region='us-east-1', aws_account_id=None):
"""Retrieve configuration from DynamoDB and return array of dictionary objects"""
findings = []
# determine aws account id
if aws_account_id is None:
found_owners = utils.get_owner_id(context)
else:
found_owners = [aws_account_id]
if len(found_owners) <= 0:
findings.append(
'There are no instances I could find on this account. ' +
'Cannot figure out the account number without any instances. ' +
'Without account number, cannot figure out what to name the S3 bucket or stack.'
)
return findings
else:
aws_account = found_owners[0]
# The bucket does not exist or you have no access
bucket_exists = None
try:
s3_client = boto3.client('s3', region_name=installed_region)
ebs_bucket_name = 'ebs-snapper-{}'.format(aws_account)
s3_client.head_bucket(Bucket=ebs_bucket_name)
bucket_exists = True
except ClientError:
bucket_exists = False
# Configurations exist but tags do not
configurations = []
dynamodb_exists = None
try:
configurations = dynamo.list_configurations(context, installed_region)
dynamodb_exists = True
except ClientError:
configurations = []
dynamodb_exists = False
# we're going across all regions, but store these in one
regions = utils.get_regions(must_contain_instances=True)
ignored_tag_values = ['false', '0', 'no']
found_config_tag_values = []
found_backup_tag_values = []
# check out all the configs in dynamodb
for config in configurations:
# if it's missing the match section, ignore it
if not utils.validate_snapshot_settings(config):
findings.append(
"Found a snapshot configuration that isn't valid: {}".format(
str(config)))
continue
# build a boto3 filter to describe instances with
configuration_matches = config['match']
filters = utils.convert_configurations_to_boto_filter(
configuration_matches)
for k, v in configuration_matches.iteritems():
if str(v).lower() in ignored_tag_values:
continue
to_add = '{}, value:{}'.format(k, v)
found_config_tag_values.append(to_add)
# if we ended up with no boto3 filters, we bail so we don't snapshot everything
if len(filters) <= 0:
LOG.warn('Could not convert configuration match to a filter: %s',
configuration_matches)
findings.append(
"Found a snapshot configuration that couldn't be converted to a filter"
)
continue
filters.append({
'Name': 'instance-state-name',
'Values': ['running', 'stopped']
})
found_instances = None
for r in regions:
ec2 = boto3.client('ec2', region_name=r)
instances = ec2.describe_instances(Filters=filters)
res_list = instances.get('Reservations', [])
for reservation in res_list:
inst_list = reservation.get('Instances', [])
if len(inst_list) > 0:
found_instances = True
break
# Look at all the tags on instances
found_tag_data = ec2.describe_tags(Filters=[{
'Name': 'resource-type',
'Values': ['instance']
}])
for tag in found_tag_data.get('Tags', []):
k = tag['Key']
v = tag['Value']
if str(v).lower() in ignored_tag_values:
continue
to_add = 'tag:{}, value:{}'.format(k, v)
if k.lower() in ['backup'
] and to_add not in found_backup_tag_values:
found_backup_tag_values.append(to_add)
if not found_instances:
long_config = []
for k, v in configuration_matches.iteritems():
long_config.append('{}, value:{}'.format(k, v))
findings.append(
"{} was configured, but didn't match any instances".format(
", ".join(long_config)))
if len(found_backup_tag_values) > 0 or len(found_config_tag_values) > 0:
if not (bucket_exists and dynamodb_exists):
findings.append(
'Configuations or tags are present, but EBS snapper not fully deployed'
)
if bucket_exists and dynamodb_exists and len(configurations) == 0:
findings.append(
'No configurations existed for this account, but ebs-snapper was deployed'
)
# tagged instances without any config
for s in found_backup_tag_values:
if s not in found_config_tag_values:
findings.append(
'{} was tagged on an instance, but no configuration exists'.
format(s))
LOG.debug("configs: %s", str(found_config_tag_values))
LOG.debug("tags: %s", str(found_backup_tag_values))
return findings