def test_perform_fanout_all_regions_replication(mocker):
"""Test for method of the same name."""
# make a dummy SNS topic
mocks.create_sns_topic('ReplicationSnapshotTopic')
expected_sns_topic = utils.get_topic_arn('ReplicationSnapshotTopic', 'us-east-1')
dummy_regions = ['us-west-2', 'us-east-1']
# make some dummy snapshots in two regions
snapshot_map = {}
for dummy_region in dummy_regions:
client = boto3.client('ec2', region_name=dummy_region)
volume = client.create_volume(Size=100, AvailabilityZone=dummy_region + "a")
snapshot = client.create_snapshot(VolumeId=volume['VolumeId'])
snapshot_map[snapshot['SnapshotId']] = dummy_region
# patch the final message sender method
ctx = utils.MockContext()
mocker.patch('ebs_snapper.replication.send_fanout_message')
replication.perform_fanout_all_regions(ctx)
# fan out, and be sure we touched every instance we created before
for r in dummy_regions:
replication.send_fanout_message.assert_any_call(
context=ctx,
region=r,
sns_topic=expected_sns_topic,
cli=False) # pylint: disable=E1103
def test_clean_snapshots_tagged_timeout(mocker):
"""Test that we _DONT_ clean anything if runtime > 4 minutes"""
# default settings
region = 'us-east-1'
mocks.create_dynamodb(region)
ctx = utils.MockContext()
ctx.set_remaining_time_in_millis(5) # 5 millis remaining
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# setup the min # snaps for the instance
config_data = {
"match": {"instance-id": instance_id},
"snapshot": {
"retention": "6 days", "minimum": 0, "frequency": "13 hours"
}
}
# put it in the table, be sure it succeeded
dynamo.store_configuration(region, 'foo', '111122223333', config_data)
# figure out the EBS volume that came with our instance
volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
# make a snapshot that should be deleted today too
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on = now.strftime('%Y-%m-%d')
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on, region)
mocker.patch('ebs_snapper.utils.delete_snapshot')
clean.clean_snapshot(ctx, region)
# ensure we DO NOT take a snapshot if our runtime was 5 minutes
assert not utils.delete_snapshot.called
def test_perform_snapshot_ignore_instance(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {
'minimum': 5,
'frequency': '2 hours',
'retention': '5 days'
},
'match': {
'tag:backup': 'yes'
},
'ignore': []
}
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
snapshot_settings['ignore'].append(instance_id)
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333',
snapshot_settings)
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results
utils.snapshot_and_tag.assert_not_called() # pylint: disable=E1103
def test_configurations():
"""Test for method for get, fetch, delete."""
# region for our tests
region = 'us-east-1'
# create dummy ec2 instance so we can figure out account id
client = boto3.client('ec2', region_name=region)
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
# create a mock table
mocks.create_dynamodb(region)
ctx = utils.MockContext()
# make sure we successfully created the table
dynamodb = boto3.resource('dynamodb', region_name=region)
table = dynamodb.Table('ebs_snapshot_configuration')
assert table.table_status == "ACTIVE"
# put some data in
config_data = {
"match": {
"instance-id": "i-abc12345",
"tag:plant": "special_flower",
"tag:Name": "legacy_server"
},
"snapshot": {
"retention": "6 days",
"minimum": 6,
"frequency": "13 hours"
}
}
# put it in the table, be sure it succeeded
response = dynamo.store_configuration(region, 'foo', '111122223333', config_data)
assert response != {}
# now list everything, be sure it was present
fetched_configurations = dynamo.list_configurations(ctx, region)
assert fetched_configurations == [config_data]
# now get that specific one
specific_config = dynamo.get_configuration(ctx, region, 'foo', '111122223333')
assert specific_config == config_data
# be sure another get for invalid item returns none
missing_config = dynamo.get_configuration(ctx, region, 'abc', '111122223333')
assert missing_config is None
# be sure it returns in a list
fetched_configurations = dynamo.list_ids(ctx, region)
assert 'foo' in fetched_configurations
# now delete it and confirm both list and get return nothing
dynamo.delete_configuration(region, 'foo', '111122223333')
specific_config = dynamo.get_configuration(ctx, region, 'foo', '111122223333')
assert specific_config is None
fetched_configurations = dynamo.list_configurations(ctx, region)
assert fetched_configurations == []
def test_get_owner_id():
"""Test for method of the same name."""
# make some dummy instances
client = boto3.client('ec2', region_name='us-west-2')
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
# show that get_owner_id can get the dummy owner id
assert [AWS_MOCK_ACCOUNT] == utils.get_owner_id(utils.MockContext())
def test_send_fanout_message_clean(mocker):
"""Test for method of the same name."""
mocks.create_sns_topic('testing-topic')
expected_sns_topic = utils.get_topic_arn('testing-topic', 'us-east-1')
ctx = utils.MockContext()
mocker.patch('ebs_snapper.utils.sns_publish')
clean.send_fanout_message(ctx, region='us-west-2', topic_arn=expected_sns_topic)
utils.sns_publish.assert_any_call( # pylint: disable=E1103
TopicArn=expected_sns_topic,
Message=json.dumps({'region': 'us-west-2'}))
def test_store_bad_configuration():
"""Test for storing a bad config."""
# region for our tests
region = 'us-east-1'
# create dummy ec2 instance so we can figure out account id
client = boto3.client('ec2', region_name=region)
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
aws_account_id = AWS_MOCK_ACCOUNT
object_id = 'foo'
# create a mock table
mocks.create_dynamodb(region)
ctx = utils.MockContext()
# make sure we successfully created the table
dynamodb = boto3.resource('dynamodb', region_name=region)
table = dynamodb.Table('ebs_snapshot_configuration')
assert table.table_status == "ACTIVE"
# put some bad data in
config_data = {
"match_bad_name": {
"instance-id": "i-abc12345",
"tag:plant": "special_flower",
"tag:Name": "legacy_server"
}
}
# this should blow up
with pytest.raises(Exception):
dynamo.store_configuration(region, object_id, aws_account_id,
config_data)
# now force it
table.put_item(
Item={
'aws_account_id': aws_account_id,
'id': object_id,
'configuration': "{, 123 bare words :: }"
})
# now watch it blow up on listing them
with pytest.raises(EbsSnapperError):
dynamo.list_configurations(ctx, region, aws_account_id)
# now blow up on fetching a specific one by Key
with pytest.raises(EbsSnapperError):
dynamo.get_configuration(ctx, region, object_id, aws_account_id)
def test_clean_tagged_snapshots_ignore_volume(mocker):
"""Test for method of the same name."""
# default settings
region = 'us-east-1'
mocks.create_dynamodb(region)
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
ctx = utils.MockContext()
# setup the min # snaps for the instance
config_data = {
"match": {
"instance-id": instance_id
},
"snapshot": {
"retention": "6 days",
"minimum": 0,
"frequency": "13 hours"
},
"ignore": []
}
# put it in the table, be sure it succeeded
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
# figure out the EBS volume that came with our instance
volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
config_data["ignore"].append(volume_id)
# make a snapshot that should be deleted today too
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on = now.strftime('%Y-%m-%d')
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on,
region)
snapshot_id = utils.most_recent_snapshot(volume_id, region)['SnapshotId']
mocker.patch('ebs_snapper.utils.delete_snapshot')
clean.clean_snapshot(ctx, region)
# ensure we deleted this snapshot if it was ready to die today
utils.delete_snapshot.assert_any_call(snapshot_id, region) # pylint: disable=E1103
# now raise the minimum, and check to be sure we didn't delete
utils.delete_snapshot.reset_mock() # pylint: disable=E1103
config_data['snapshot']['minimum'] = 5
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
clean.clean_snapshot(ctx, region)
utils.delete_snapshot.assert_not_called() # pylint: disable=E1103
def test_clean_tagged_snapshots_ignore_retention(mocker):
"""Test for method of the same name."""
# default settings
region = 'us-east-1'
mocks.create_dynamodb(region)
# create two instances and record the id of one of them
instance_id_list = mocks.create_instances(region, count=2)
instance_id = instance_id_list[0]
ctx = utils.MockContext()
# setup the min # snaps for the instance
config_data = {
"match": {
"instance-id": instance_id
},
"snapshot": {
"retention": "6 days",
"minimum": 5,
"frequency": "13 hours"
},
"ignore_retention": True
}
# put it in the table, be sure it succeeded
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
# figure out the EBS volume that came with our instance
volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
# make a snapshot that should be deleted today (but 5 snap retention)
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on = now.strftime('%Y-%m-%d')
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on,
region)
snapshot_id = utils.most_recent_snapshot(volume_id, region)['SnapshotId']
# now delete the instance and volume, keep the snapshot
client = boto3.client('ec2', region_name=region)
client.terminate_instances(InstanceIds=[instance_id])
mocker.patch('ebs_snapper.utils.delete_snapshot')
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
clean.clean_snapshot(ctx, region)
utils.delete_snapshot.assert_any_call(snapshot_id, region) # pylint: disable=E1103
def test_perform_snapshot_skipped(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {
'minimum': 5,
'frequency': '2 hours',
'retention': '5 days'
},
'match': {
'tag:backup': 'yes'
}
}
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333',
snapshot_settings)
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# figure out the EBS volume that came with our instance
instance_details = utils.get_instance(instance_id, region)
block_devices = instance_details.get('BlockDeviceMappings', [])
volume_id = block_devices[0]['Ebs']['VolumeId']
# determine what we should be tagging the snapshot
ret, freq = utils.parse_snapshot_settings(snapshot_settings) # pylint: disable=unused-variable
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on_dt = now + ret
delete_on = delete_on_dt.strftime('%Y-%m-%d')
# now take a snapshot, so we expect this next one to be skipped
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on,
region)
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results (should not create a second snapshot)
utils.snapshot_and_tag.assert_not_called() # pylint: disable=E1103
def test_perform_fanout_all_regions_snapshot(mocker):
"""Test for method of the same name."""
# make a dummy SNS topic
mocks.create_event_rule('ScheduledRuleReplicationFunction')
mocks.create_sns_topic('CreateSnapshotTopic')
expected_sns_topic = utils.get_topic_arn('CreateSnapshotTopic',
'us-east-1')
dummy_regions = ['us-west-2', 'us-east-1']
# make some dummy instances in two regions
instance_maps = {}
for dummy_region in dummy_regions:
client = boto3.client('ec2', region_name=dummy_region)
create_results = client.run_instances(ImageId='ami-123abc',
MinCount=5,
MaxCount=5)
for instance_data in create_results['Instances']:
instance_maps[instance_data['InstanceId']] = dummy_region
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
config_data = {
"match": {
"instance-id": instance_maps.keys()
},
"snapshot": {
"retention": "3 days",
"minimum": 4,
"frequency": "11 hours"
}
}
dynamo.store_configuration('us-east-1', 'some_unique_id', AWS_MOCK_ACCOUNT,
config_data)
# patch the final message sender method
ctx = utils.MockContext()
mocker.patch('ebs_snapper.snapshot.send_fanout_message')
snapshot.perform_fanout_all_regions(ctx)
# fan out, and be sure we touched every instance we created before
for r in dummy_regions:
snapshot.send_fanout_message.assert_any_call(
context=ctx, region=r, sns_topic=expected_sns_topic, cli=False) # pylint: disable=E1103
def test_build_replication_cache():
"""Test that we build a list of snapshots with correct groupings"""
# setup variables
region = 'us-west-2'
installed_region = 'us-east-1'
context = utils.MockContext()
tags = ['replication_src_region', 'replication_dst_region']
configurations = []
client = boto3.client('ec2', region_name=region)
# toss a volume in us-west-2 and snapshot it twice
volume_results = client.create_volume(Size=100,
AvailabilityZone='us-west-1a')
src_snapshot = client.create_snapshot(VolumeId=volume_results['VolumeId'])
dst_snapshot = client.create_snapshot(VolumeId=volume_results['VolumeId'])
# build the cache and be sure they don't show up
cache1 = utils.build_replication_cache(context, tags, configurations,
region, installed_region)
for t in tags:
assert len(cache1.get(t, [])) == 0
# now tag...
client.create_tags(Resources=[src_snapshot['SnapshotId']],
Tags=[{
'Key': 'replication_src_region',
'Value': 'us-west-1'
}])
client.create_tags(Resources=[dst_snapshot['SnapshotId']],
Tags=[{
'Key': 'replication_dst_region',
'Value': 'us-west-1'
}])
# build cache again, and expect to see the tagged snapshots
cache2 = utils.build_replication_cache(context, tags, configurations,
region, installed_region)
assert cache2['replication_src_region'][0]['SnapshotId'] == src_snapshot[
'SnapshotId']
assert cache2['replication_dst_region'][0]['SnapshotId'] == dst_snapshot[
'SnapshotId']
def test_perform_fanout_all_regions_clean(mocker):
"""Test for method of the same name."""
mocks.create_sns_topic('CleanSnapshotTopic')
mocks.create_dynamodb()
expected_regions = utils.get_regions()
for r in expected_regions: # must have an instance in the region to clean it
mocks.create_instances(region=r)
expected_sns_topic = utils.get_topic_arn('CleanSnapshotTopic', 'us-east-1')
ctx = utils.MockContext()
mocker.patch('ebs_snapper.clean.send_fanout_message')
# fan out, and be sure we touched every region
clean.perform_fanout_all_regions(ctx)
for r in expected_regions:
clean.send_fanout_message.assert_any_call( # pylint: disable=E1103
ctx,
cli=False,
region=r,
topic_arn=expected_sns_topic)
def test_perform_snapshot_ignore_volume(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {
'minimum': 5,
'frequency': '2 hours',
'retention': '5 days'
},
'match': {
'tag:backup': 'yes'
},
'ignore': []
}
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', AWS_MOCK_ACCOUNT,
snapshot_settings)
# figure out the EBS volume that came with our instance
instance_details = utils.get_instance(instance_id, region)
block_devices = instance_details.get('BlockDeviceMappings', [])
volume_id = block_devices[0]['Ebs']['VolumeId']
snapshot_settings['ignore'].append(volume_id)
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results
utils.snapshot_and_tag.assert_not_called() # pylint: disable=E1103
def test_perform_snapshot(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {'minimum': 5, 'frequency': '2 hours', 'retention': '5 days'},
'match': {'tag:backup': 'yes'}
}
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333', snapshot_settings)
# figure out the EBS volume that came with our instance
instance_details = utils.get_instance(instance_id, region)
block_devices = instance_details.get('BlockDeviceMappings', [])
volume_id = block_devices[0]['Ebs']['VolumeId']
# determine what we should be tagging the snapshot
ret, freq = utils.parse_snapshot_settings(snapshot_settings) # pylint: disable=unused-variable
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on_dt = now + ret
delete_on = delete_on_dt.strftime('%Y-%m-%d')
# apply some tags
client = boto3.client('ec2', region_name=region)
instance_tags = [
{'Key': 'Name', 'Value': 'Foo'},
{'Key': 'Service', 'Value': 'Bar'},
{'Key': 'backup', 'Value': 'yes'},
]
client.create_tags(DryRun=False, Resources=[instance_id], Tags=instance_tags)
# override one of the tags
volume_tags = [{'Key': 'Service', 'Value': 'Baz'}]
client.create_tags(DryRun=False, Resources=[volume_id], Tags=volume_tags)
# when combined, we expect tags to be this.
tags = [
{'Key': 'Name', 'Value': 'Foo'},
{'Key': 'Service', 'Value': 'Baz'},
{'Key': 'backup', 'Value': 'yes'},
]
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results
utils.snapshot_and_tag.assert_any_call( # pylint: disable=E1103
instance_id,
'ami-123abc',
volume_id,
delete_on,
region,
additional_tags=tags)
def test_perform_replication(mocker):
"""Test for method of the same name."""
# some default settings for this test
region_a = 'us-west-1'
region_b = 'us-east-1'
ctx = utils.MockContext()
# setup some dummy configuration for snapshots
mocks.create_dynamodb('us-east-1')
snapshot_settings = {'snapshot': {'minimum': 5, 'frequency': '2 hours', 'retention': '5 days'},
'match': {'tag:backup': 'yes'}}
dynamo.store_configuration('us-east-1', 'some_unique_id', AWS_MOCK_ACCOUNT, snapshot_settings)
# clients
client_a = boto3.client('ec2', region_name=region_a)
client_b = boto3.client('ec2', region_name=region_b)
# create a volume in region_a
volume = client_a.create_volume(Size=100, AvailabilityZone=region_a + "a")
snapshot_description = "Something from EBS Snapper"
snapshot = client_a.create_snapshot(
VolumeId=volume['VolumeId'],
Description=snapshot_description
)
client_a.create_tags(
Resources=[snapshot['SnapshotId']],
Tags=[{'Key': 'replication_dst_region', 'Value': region_b}]
)
# trigger replication, assert that we copied a snapshot to region_b
mocker.patch('ebs_snapper.utils.copy_snapshot_and_tag')
replication.perform_replication(ctx, region_a)
utils.copy_snapshot_and_tag.assert_any_call( # pylint: disable=E1103
ctx,
region_a,
region_b,
snapshot['SnapshotId'],
snapshot_description)
# now create that snapshot manually
replica_volume = client_b.create_volume(Size=100, AvailabilityZone=region_b + "a")
replica_snapshot_description = "Something from EBS Snapper, copied to region b"
replica_snapshot = client_b.create_snapshot(
VolumeId=replica_volume['VolumeId'],
Description=replica_snapshot_description
)
client_b.create_tags(
Resources=[replica_snapshot['SnapshotId']],
Tags=[
{'Key': 'replication_src_region', 'Value': region_a},
{'Key': 'replication_snapshot_id', 'Value': snapshot['SnapshotId']}
]
)
# trigger replication a second time, and confirm still one copy (not 2)
utils.copy_snapshot_and_tag.reset_mock()
replication.perform_replication(ctx, region_a)
utils.copy_snapshot_and_tag.assert_not_called() # pylint: disable=E1103
# now delete the original and trigger replication, confirm replica deleted
utils.delete_snapshot(snapshot['SnapshotId'], region_a)
mocker.patch('ebs_snapper.utils.delete_snapshot')
replication.perform_replication(ctx, region_b)
utils.delete_snapshot.assert_any_call( # pylint: disable=E1103
replica_snapshot['SnapshotId'],
region_b
)
