def test_get_owner_id():
"""Test for method of the same name."""
# make some dummy instances
client = boto3.client('ec2', region_name='us-west-2')
client.run_instances(ImageId='ami-123abc', MinCount=1, MaxCount=5)
# show that get_owner_id can get the dummy owner id
assert [AWS_MOCK_ACCOUNT] == utils.get_owner_id(utils.MockContext())
def deploy(context,
aws_account_id=None,
no_build=None,
no_upload=None,
no_stack=None):
"""Main function that does the deploy to an aws account"""
# lambda-uploader configuration step
lambda_zip_filename = 'ebs_snapper.zip'
if not no_build:
LOG.info("Building package using lambda-uploader")
build_package(lambda_zip_filename)
# get security credentials from EC2 API, if we are going to use them
needs_owner_id = (not no_upload) or (not no_stack)
if needs_owner_id:
if aws_account_id is None:
found_owners = utils.get_owner_id(context)
else:
found_owners = [aws_account_id]
if len(found_owners) <= 0:
LOG.warn('There are no instances I could find on this account.')
LOG.warn(
'I cannot figure out the account number without any instances.'
)
LOG.warn(
'Without account number, I do not know what to name the S3 bucket or stack.'
)
LOG.warn(
'You may provide it on the commandline to bypass this error.')
return
else:
aws_account = found_owners[0]
# freshen the S3 bucket
if not no_upload:
ebs_bucket_name = create_or_update_s3_bucket(
aws_account, lambda_zip_filename)
# freshen the stack
if not no_stack:
create_or_update_stack(aws_account, DEFAULT_REGION,
ebs_bucket_name)
# freshen up lambda jobs themselves
if not no_upload:
update_function_and_version(ebs_bucket_name, lambda_zip_filename)
ensure_cloudwatch_logs_retention(aws_account)
def list_ids(context, installed_region, aws_account_id=None):
"""Retrieve configuration from DynamoDB and return array of dictionary objects"""
found_configurations = {}
if aws_account_id is None:
aws_account_id = utils.get_owner_id(context)[0]
dynamodb = boto3.resource('dynamodb', region_name=installed_region)
table = dynamodb.Table('ebs_snapshot_configuration')
results = table.query(
KeyConditionExpression=Key('aws_account_id').eq(aws_account_id))
for item in results.get('Items', []):
str_item = item.get('configuration', None)
found_configurations[str_item] = item['id']
return found_configurations.values()
def get_configuration(context,
installed_region,
object_id,
aws_account_id=None):
"""Retrieve configuration from DynamoDB and return single object"""
if aws_account_id is None:
aws_account_id = utils.get_owner_id(context)[0]
dynamodb = boto3.resource('dynamodb', region_name=installed_region)
table = dynamodb.Table('ebs_snapshot_configuration')
expr = Key('aws_account_id').eq(aws_account_id) & Key('id').eq(object_id)
results = table.query(KeyConditionExpression=expr)
for item in results['Items']:
str_item = item['configuration']
json_item = json.loads(str_item)
return json_item
return None
def get_configuration(context,
installed_region,
object_id,
aws_account_id=None):
"""Retrieve configuration from DynamoDB and return single object"""
if aws_account_id is None:
aws_account_id = utils.get_owner_id(context)[0]
dynamodb = boto3.resource('dynamodb', region_name=installed_region)
table = dynamodb.Table('ebs_snapshot_configuration')
expr = Key('aws_account_id').eq(aws_account_id) & Key('id').eq(object_id)
results = table.query(KeyConditionExpression=expr)
for item in results.get('Items', []):
str_item = item.get('configuration', None)
try:
json_item = json.loads(str_item)
return json_item
except Exception as e:
raise EbsSnapperError('error loading configuration', e)
return None
def shell_configure(*args):
"""Get, set, or delete configuration in DynamoDB."""
# lazy retrieve the account id one way or another
if args[0].aws_account_id is None:
aws_account_id = utils.get_owner_id(CTX)[0]
else:
aws_account_id = args[0].aws_account_id
LOG.debug("Account: %s", aws_account_id)
object_id = args[0].object_id
action = args[0].conf_action
installed_region = args[0].conf_toolregion
extra = args[0].extra
if action == 'check':
LOG.debug('Sanity checking configurations for %s', aws_account_id)
findings = deploy.sanity_check(
CTX,
installed_region,
aws_account_id=aws_account_id) or []
if extra:
prefix = '{},{}'.format(extra, aws_account_id)
else:
prefix = '{}'.format(aws_account_id)
for f in findings:
print("{}: {}".format(prefix, f))
elif action == 'list':
LOG.info('Listing all object keys for %s', aws_account_id)
list_results = dynamo.list_ids(
CTX,
installed_region,
aws_account_id=aws_account_id)
if list_results is None or len(list_results) == 0:
print('No configurations found')
else:
print("aws_account_id,id")
for r in list_results:
print("{},{}".format(aws_account_id, r))
elif action == 'get':
if object_id is None:
raise Exception('must provide an object key id')
else:
LOG.debug("Object key: %s", object_id)
LOG.info('Retrieving %s', args[0])
single_result = dynamo.get_configuration(
CTX,
installed_region,
object_id=object_id,
aws_account_id=aws_account_id)
if single_result is None:
print('No configuration found')
else:
print(json.dumps(single_result))
elif action == 'set':
if object_id is None:
raise Exception('must provide an object key id')
else:
LOG.debug("Object key: %s", object_id)
config = json.loads(args[0].configuration_json)
LOG.debug("Configuration: %s", config)
dynamo.store_configuration(installed_region, object_id, aws_account_id, config)
print('Saved to key {} under account {}'
.format(object_id, aws_account_id))
elif action == 'del':
print(dynamo.delete_configuration(
installed_region,
object_id=object_id,
aws_account_id=aws_account_id))
else:
# should never get here, from argparse
raise Exception('invalid parameters', args)
LOG.info('Function shell_configure completed')
def sanity_check(context, installed_region='us-east-1', aws_account_id=None):
"""Retrieve configuration from DynamoDB and return array of dictionary objects"""
findings = []
# determine aws account id
if aws_account_id is None:
found_owners = utils.get_owner_id(context)
else:
found_owners = [aws_account_id]
if len(found_owners) <= 0:
findings.append(
'There are no instances I could find on this account. ' +
'Cannot figure out the account number without any instances. ' +
'Without account number, cannot figure out what to name the S3 bucket or stack.'
)
return findings
else:
aws_account = found_owners[0]
# The bucket does not exist or you have no access
bucket_exists = None
try:
s3_client = boto3.client('s3', region_name=installed_region)
ebs_bucket_name = 'ebs-snapper-{}'.format(aws_account)
s3_client.head_bucket(Bucket=ebs_bucket_name)
bucket_exists = True
except ClientError:
bucket_exists = False
# Configurations exist but tags do not
configurations = []
dynamodb_exists = None
try:
configurations = dynamo.list_configurations(context, installed_region)
dynamodb_exists = True
except ClientError:
configurations = []
dynamodb_exists = False
# we're going across all regions, but store these in one
regions = utils.get_regions(must_contain_instances=True)
ignored_tag_values = ['false', '0', 'no']
found_config_tag_values = []
found_backup_tag_values = []
# check out all the configs in dynamodb
for config in configurations:
# if it's missing the match section, ignore it
if not utils.validate_snapshot_settings(config):
findings.append(
"Found a snapshot configuration that isn't valid: {}".format(
str(config)))
continue
# build a boto3 filter to describe instances with
configuration_matches = config['match']
filters = utils.convert_configurations_to_boto_filter(
configuration_matches)
for k, v in configuration_matches.iteritems():
if str(v).lower() in ignored_tag_values:
continue
to_add = '{}, value:{}'.format(k, v)
found_config_tag_values.append(to_add)
# if we ended up with no boto3 filters, we bail so we don't snapshot everything
if len(filters) <= 0:
LOG.warn('Could not convert configuration match to a filter: %s',
configuration_matches)
findings.append(
"Found a snapshot configuration that couldn't be converted to a filter"
)
continue
filters.append({
'Name': 'instance-state-name',
'Values': ['running', 'stopped']
})
found_instances = None
for r in regions:
ec2 = boto3.client('ec2', region_name=r)
instances = ec2.describe_instances(Filters=filters)
res_list = instances.get('Reservations', [])
for reservation in res_list:
inst_list = reservation.get('Instances', [])
if len(inst_list) > 0:
found_instances = True
break
# Look at all the tags on instances
found_tag_data = ec2.describe_tags(Filters=[{
'Name': 'resource-type',
'Values': ['instance']
}])
for tag in found_tag_data.get('Tags', []):
k = tag['Key']
v = tag['Value']
if str(v).lower() in ignored_tag_values:
continue
to_add = 'tag:{}, value:{}'.format(k, v)
if k.lower() in ['backup'
] and to_add not in found_backup_tag_values:
found_backup_tag_values.append(to_add)
if not found_instances:
long_config = []
for k, v in configuration_matches.iteritems():
long_config.append('{}, value:{}'.format(k, v))
findings.append(
"{} was configured, but didn't match any instances".format(
", ".join(long_config)))
if len(found_backup_tag_values) > 0 or len(found_config_tag_values) > 0:
if not (bucket_exists and dynamodb_exists):
findings.append(
'Configuations or tags are present, but EBS snapper not fully deployed'
)
if bucket_exists and dynamodb_exists and len(configurations) == 0:
findings.append(
'No configurations existed for this account, but ebs-snapper was deployed'
)
# tagged instances without any config
for s in found_backup_tag_values:
if s not in found_config_tag_values:
findings.append(
'{} was tagged on an instance, but no configuration exists'.
format(s))
LOG.debug("configs: %s", str(found_config_tag_values))
LOG.debug("tags: %s", str(found_backup_tag_values))
return findings