def test_perform_snapshot_ignore_instance(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {
'minimum': 5,
'frequency': '2 hours',
'retention': '5 days'
},
'match': {
'tag:backup': 'yes'
},
'ignore': []
}
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
snapshot_settings['ignore'].append(instance_id)
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333',
snapshot_settings)
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results
utils.snapshot_and_tag.assert_not_called() # pylint: disable=E1103
def test_clean_snapshots_tagged_timeout(mocker):
"""Test that we _DONT_ clean anything if runtime > 4 minutes"""
# default settings
region = 'us-east-1'
mocks.create_dynamodb(region)
ctx = utils.MockContext()
ctx.set_remaining_time_in_millis(5) # 5 millis remaining
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# setup the min # snaps for the instance
config_data = {
"match": {"instance-id": instance_id},
"snapshot": {
"retention": "6 days", "minimum": 0, "frequency": "13 hours"
}
}
# put it in the table, be sure it succeeded
dynamo.store_configuration(region, 'foo', '111122223333', config_data)
# figure out the EBS volume that came with our instance
volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
# make a snapshot that should be deleted today too
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on = now.strftime('%Y-%m-%d')
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on, region)
mocker.patch('ebs_snapper.utils.delete_snapshot')
clean.clean_snapshot(ctx, region)
# ensure we DO NOT take a snapshot if our runtime was 5 minutes
assert not utils.delete_snapshot.called
def test_store_bad_configuration():
"""Test for storing a bad config."""
# region for our tests
region = 'us-east-1'
# create dummy ec2 instance so we can figure out account id
client = boto3.client('ec2', region_name=region)
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
aws_account_id = AWS_MOCK_ACCOUNT
object_id = 'foo'
# create a mock table
mocks.create_dynamodb(region)
ctx = utils.MockContext()
# make sure we successfully created the table
dynamodb = boto3.resource('dynamodb', region_name=region)
table = dynamodb.Table('ebs_snapshot_configuration')
assert table.table_status == "ACTIVE"
# put some bad data in
config_data = {
"match_bad_name": {
"instance-id": "i-abc12345",
"tag:plant": "special_flower",
"tag:Name": "legacy_server"
}
}
# this should blow up
with pytest.raises(Exception):
dynamo.store_configuration(region, object_id, aws_account_id,
config_data)
# now force it
table.put_item(
Item={
'aws_account_id': aws_account_id,
'id': object_id,
'configuration': "{, 123 bare words :: }"
})
# now watch it blow up on listing them
with pytest.raises(EbsSnapperError):
dynamo.list_configurations(ctx, region, aws_account_id)
# now blow up on fetching a specific one by Key
with pytest.raises(EbsSnapperError):
dynamo.get_configuration(ctx, region, object_id, aws_account_id)
def test_clean_tagged_snapshots_ignore_volume(mocker):
"""Test for method of the same name."""
# default settings
region = 'us-east-1'
mocks.create_dynamodb(region)
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
ctx = utils.MockContext()
# setup the min # snaps for the instance
config_data = {
"match": {
"instance-id": instance_id
},
"snapshot": {
"retention": "6 days",
"minimum": 0,
"frequency": "13 hours"
},
"ignore": []
}
# put it in the table, be sure it succeeded
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
# figure out the EBS volume that came with our instance
volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
config_data["ignore"].append(volume_id)
# make a snapshot that should be deleted today too
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on = now.strftime('%Y-%m-%d')
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on,
region)
snapshot_id = utils.most_recent_snapshot(volume_id, region)['SnapshotId']
mocker.patch('ebs_snapper.utils.delete_snapshot')
clean.clean_snapshot(ctx, region)
# ensure we deleted this snapshot if it was ready to die today
utils.delete_snapshot.assert_any_call(snapshot_id, region) # pylint: disable=E1103
# now raise the minimum, and check to be sure we didn't delete
utils.delete_snapshot.reset_mock() # pylint: disable=E1103
config_data['snapshot']['minimum'] = 5
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
clean.clean_snapshot(ctx, region)
utils.delete_snapshot.assert_not_called() # pylint: disable=E1103
def test_configurations():
"""Test for method for get, fetch, delete."""
# region for our tests
region = 'us-east-1'
# create dummy ec2 instance so we can figure out account id
client = boto3.client('ec2', region_name=region)
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
# create a mock table
mocks.create_dynamodb(region)
ctx = utils.MockContext()
# make sure we successfully created the table
dynamodb = boto3.resource('dynamodb', region_name=region)
table = dynamodb.Table('ebs_snapshot_configuration')
assert table.table_status == "ACTIVE"
# put some data in
config_data = {
"match": {
"instance-id": "i-abc12345",
"tag:plant": "special_flower",
"tag:Name": "legacy_server"
},
"snapshot": {
"retention": "6 days",
"minimum": 6,
"frequency": "13 hours"
}
}
# put it in the table, be sure it succeeded
response = dynamo.store_configuration(region, 'foo', '111122223333', config_data)
assert response != {}
# now list everything, be sure it was present
fetched_configurations = dynamo.list_configurations(ctx, region)
assert fetched_configurations == [config_data]
# now get that specific one
specific_config = dynamo.get_configuration(ctx, region, 'foo', '111122223333')
assert specific_config == config_data
# be sure another get for invalid item returns none
missing_config = dynamo.get_configuration(ctx, region, 'abc', '111122223333')
assert missing_config is None
# be sure it returns in a list
fetched_configurations = dynamo.list_ids(ctx, region)
assert 'foo' in fetched_configurations
# now delete it and confirm both list and get return nothing
dynamo.delete_configuration(region, 'foo', '111122223333')
specific_config = dynamo.get_configuration(ctx, region, 'foo', '111122223333')
assert specific_config is None
fetched_configurations = dynamo.list_configurations(ctx, region)
assert fetched_configurations == []
def test_clean_tagged_snapshots_ignore_retention(mocker):
"""Test for method of the same name."""
# default settings
region = 'us-east-1'
mocks.create_dynamodb(region)
# create two instances and record the id of one of them
instance_id_list = mocks.create_instances(region, count=2)
instance_id = instance_id_list[0]
ctx = utils.MockContext()
# setup the min # snaps for the instance
config_data = {
"match": {
"instance-id": instance_id
},
"snapshot": {
"retention": "6 days",
"minimum": 5,
"frequency": "13 hours"
},
"ignore_retention": True
}
# put it in the table, be sure it succeeded
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
# figure out the EBS volume that came with our instance
volume_id = utils.get_volumes([instance_id], region)[0]['VolumeId']
# make a snapshot that should be deleted today (but 5 snap retention)
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on = now.strftime('%Y-%m-%d')
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on,
region)
snapshot_id = utils.most_recent_snapshot(volume_id, region)['SnapshotId']
# now delete the instance and volume, keep the snapshot
client = boto3.client('ec2', region_name=region)
client.terminate_instances(InstanceIds=[instance_id])
mocker.patch('ebs_snapper.utils.delete_snapshot')
dynamo.store_configuration(region, 'foo', AWS_MOCK_ACCOUNT, config_data)
clean.clean_snapshot(ctx, region)
utils.delete_snapshot.assert_any_call(snapshot_id, region) # pylint: disable=E1103
def test_perform_snapshot_skipped(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {
'minimum': 5,
'frequency': '2 hours',
'retention': '5 days'
},
'match': {
'tag:backup': 'yes'
}
}
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333',
snapshot_settings)
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# figure out the EBS volume that came with our instance
instance_details = utils.get_instance(instance_id, region)
block_devices = instance_details.get('BlockDeviceMappings', [])
volume_id = block_devices[0]['Ebs']['VolumeId']
# determine what we should be tagging the snapshot
ret, freq = utils.parse_snapshot_settings(snapshot_settings) # pylint: disable=unused-variable
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on_dt = now + ret
delete_on = delete_on_dt.strftime('%Y-%m-%d')
# now take a snapshot, so we expect this next one to be skipped
utils.snapshot_and_tag(instance_id, 'ami-123abc', volume_id, delete_on,
region)
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results (should not create a second snapshot)
utils.snapshot_and_tag.assert_not_called() # pylint: disable=E1103
def test_perform_fanout_all_regions_snapshot(mocker):
"""Test for method of the same name."""
# make a dummy SNS topic
mocks.create_event_rule('ScheduledRuleReplicationFunction')
mocks.create_sns_topic('CreateSnapshotTopic')
expected_sns_topic = utils.get_topic_arn('CreateSnapshotTopic',
'us-east-1')
dummy_regions = ['us-west-2', 'us-east-1']
# make some dummy instances in two regions
instance_maps = {}
for dummy_region in dummy_regions:
client = boto3.client('ec2', region_name=dummy_region)
create_results = client.run_instances(ImageId='ami-123abc',
MinCount=5,
MaxCount=5)
for instance_data in create_results['Instances']:
instance_maps[instance_data['InstanceId']] = dummy_region
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
config_data = {
"match": {
"instance-id": instance_maps.keys()
},
"snapshot": {
"retention": "3 days",
"minimum": 4,
"frequency": "11 hours"
}
}
dynamo.store_configuration('us-east-1', 'some_unique_id', AWS_MOCK_ACCOUNT,
config_data)
# patch the final message sender method
ctx = utils.MockContext()
mocker.patch('ebs_snapper.snapshot.send_fanout_message')
snapshot.perform_fanout_all_regions(ctx)
# fan out, and be sure we touched every instance we created before
for r in dummy_regions:
snapshot.send_fanout_message.assert_any_call(
context=ctx, region=r, sns_topic=expected_sns_topic, cli=False) # pylint: disable=E1103
def test_perform_snapshot_ignore_volume(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {
'minimum': 5,
'frequency': '2 hours',
'retention': '5 days'
},
'match': {
'tag:backup': 'yes'
},
'ignore': []
}
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', AWS_MOCK_ACCOUNT,
snapshot_settings)
# figure out the EBS volume that came with our instance
instance_details = utils.get_instance(instance_id, region)
block_devices = instance_details.get('BlockDeviceMappings', [])
volume_id = block_devices[0]['Ebs']['VolumeId']
snapshot_settings['ignore'].append(volume_id)
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results
utils.snapshot_and_tag.assert_not_called() # pylint: disable=E1103
def shell_configure(*args):
"""Get, set, or delete configuration in DynamoDB."""
# lazy retrieve the account id one way or another
if args[0].aws_account_id is None:
aws_account_id = utils.get_owner_id(CTX)[0]
else:
aws_account_id = args[0].aws_account_id
LOG.debug("Account: %s", aws_account_id)
object_id = args[0].object_id
action = args[0].conf_action
installed_region = args[0].conf_toolregion
extra = args[0].extra
if action == 'check':
LOG.debug('Sanity checking configurations for %s', aws_account_id)
findings = deploy.sanity_check(
CTX,
installed_region,
aws_account_id=aws_account_id) or []
if extra:
prefix = '{},{}'.format(extra, aws_account_id)
else:
prefix = '{}'.format(aws_account_id)
for f in findings:
print("{}: {}".format(prefix, f))
elif action == 'list':
LOG.info('Listing all object keys for %s', aws_account_id)
list_results = dynamo.list_ids(
CTX,
installed_region,
aws_account_id=aws_account_id)
if list_results is None or len(list_results) == 0:
print('No configurations found')
else:
print("aws_account_id,id")
for r in list_results:
print("{},{}".format(aws_account_id, r))
elif action == 'get':
if object_id is None:
raise Exception('must provide an object key id')
else:
LOG.debug("Object key: %s", object_id)
LOG.info('Retrieving %s', args[0])
single_result = dynamo.get_configuration(
CTX,
installed_region,
object_id=object_id,
aws_account_id=aws_account_id)
if single_result is None:
print('No configuration found')
else:
print(json.dumps(single_result))
elif action == 'set':
if object_id is None:
raise Exception('must provide an object key id')
else:
LOG.debug("Object key: %s", object_id)
config = json.loads(args[0].configuration_json)
LOG.debug("Configuration: %s", config)
dynamo.store_configuration(installed_region, object_id, aws_account_id, config)
print('Saved to key {} under account {}'
.format(object_id, aws_account_id))
elif action == 'del':
print(dynamo.delete_configuration(
installed_region,
object_id=object_id,
aws_account_id=aws_account_id))
else:
# should never get here, from argparse
raise Exception('invalid parameters', args)
LOG.info('Function shell_configure completed')
def test_perform_snapshot(mocker):
"""Test for method of the same name."""
# some default settings for this test
region = 'us-west-2'
snapshot_settings = {
'snapshot': {'minimum': 5, 'frequency': '2 hours', 'retention': '5 days'},
'match': {'tag:backup': 'yes'}
}
# create an instance and record the id
instance_id = mocks.create_instances(region, count=1)[0]
# need to filter instances, so need dynamodb present
mocks.create_dynamodb('us-east-1')
dynamo.store_configuration('us-east-1', 'some_unique_id', '111122223333', snapshot_settings)
# figure out the EBS volume that came with our instance
instance_details = utils.get_instance(instance_id, region)
block_devices = instance_details.get('BlockDeviceMappings', [])
volume_id = block_devices[0]['Ebs']['VolumeId']
# determine what we should be tagging the snapshot
ret, freq = utils.parse_snapshot_settings(snapshot_settings) # pylint: disable=unused-variable
now = datetime.datetime.now(dateutil.tz.tzutc())
delete_on_dt = now + ret
delete_on = delete_on_dt.strftime('%Y-%m-%d')
# apply some tags
client = boto3.client('ec2', region_name=region)
instance_tags = [
{'Key': 'Name', 'Value': 'Foo'},
{'Key': 'Service', 'Value': 'Bar'},
{'Key': 'backup', 'Value': 'yes'},
]
client.create_tags(DryRun=False, Resources=[instance_id], Tags=instance_tags)
# override one of the tags
volume_tags = [{'Key': 'Service', 'Value': 'Baz'}]
client.create_tags(DryRun=False, Resources=[volume_id], Tags=volume_tags)
# when combined, we expect tags to be this.
tags = [
{'Key': 'Name', 'Value': 'Foo'},
{'Key': 'Service', 'Value': 'Baz'},
{'Key': 'backup', 'Value': 'yes'},
]
# patch the final method that takes a snapshot
mocker.patch('ebs_snapper.utils.snapshot_and_tag')
# since there are no snapshots, we should expect this to trigger one
ctx = utils.MockContext()
snapshot.perform_snapshot(ctx, region)
# test results
utils.snapshot_and_tag.assert_any_call( # pylint: disable=E1103
instance_id,
'ami-123abc',
volume_id,
delete_on,
region,
additional_tags=tags)
def test_perform_replication(mocker):
"""Test for method of the same name."""
# some default settings for this test
region_a = 'us-west-1'
region_b = 'us-east-1'
ctx = utils.MockContext()
# setup some dummy configuration for snapshots
mocks.create_dynamodb('us-east-1')
snapshot_settings = {'snapshot': {'minimum': 5, 'frequency': '2 hours', 'retention': '5 days'},
'match': {'tag:backup': 'yes'}}
dynamo.store_configuration('us-east-1', 'some_unique_id', AWS_MOCK_ACCOUNT, snapshot_settings)
# clients
client_a = boto3.client('ec2', region_name=region_a)
client_b = boto3.client('ec2', region_name=region_b)
# create a volume in region_a
volume = client_a.create_volume(Size=100, AvailabilityZone=region_a + "a")
snapshot_description = "Something from EBS Snapper"
snapshot = client_a.create_snapshot(
VolumeId=volume['VolumeId'],
Description=snapshot_description
)
client_a.create_tags(
Resources=[snapshot['SnapshotId']],
Tags=[{'Key': 'replication_dst_region', 'Value': region_b}]
)
# trigger replication, assert that we copied a snapshot to region_b
mocker.patch('ebs_snapper.utils.copy_snapshot_and_tag')
replication.perform_replication(ctx, region_a)
utils.copy_snapshot_and_tag.assert_any_call( # pylint: disable=E1103
ctx,
region_a,
region_b,
snapshot['SnapshotId'],
snapshot_description)
# now create that snapshot manually
replica_volume = client_b.create_volume(Size=100, AvailabilityZone=region_b + "a")
replica_snapshot_description = "Something from EBS Snapper, copied to region b"
replica_snapshot = client_b.create_snapshot(
VolumeId=replica_volume['VolumeId'],
Description=replica_snapshot_description
)
client_b.create_tags(
Resources=[replica_snapshot['SnapshotId']],
Tags=[
{'Key': 'replication_src_region', 'Value': region_a},
{'Key': 'replication_snapshot_id', 'Value': snapshot['SnapshotId']}
]
)
# trigger replication a second time, and confirm still one copy (not 2)
utils.copy_snapshot_and_tag.reset_mock()
replication.perform_replication(ctx, region_a)
utils.copy_snapshot_and_tag.assert_not_called() # pylint: disable=E1103
# now delete the original and trigger replication, confirm replica deleted
utils.delete_snapshot(snapshot['SnapshotId'], region_a)
mocker.patch('ebs_snapper.utils.delete_snapshot')
replication.perform_replication(ctx, region_b)
utils.delete_snapshot.assert_any_call( # pylint: disable=E1103
replica_snapshot['SnapshotId'],
region_b
)
