def parse_profiles(profiles):
"""
Summary:
Parse awscli profile_names given as parameter in 1 of 2 forms:
1. single profilename given
2. list of profile_names
Also, function prepends profile_name(s) with a prefix when it detects
profile_name refers to temp credentials in the local awscli configuration
Args:
profiles (str or file): Profiles parameter can be either:
- a single profile_name (str)
- a file containing multiple profile_names, 1 per line
Returns:
- list of awscli profilenames, TYPE: list
OR
- single profilename, TYPE: str
"""
profile_list = []
try:
if isinstance(profiles, list):
return [_profile_prefix(x.strip()) for x in profiles]
elif os.path.isfile(profiles):
with open(profiles) as f1:
for line in f1:
profile_list.append(_profile_prefix(line.strip()))
else:
return _profile_prefix(profiles.strip())
except Exception as e:
logger.exception(
f'{inspect.stack()[0][3]}: Unknown error while converting profile_names from local awscli config: {e}'
)
raise
return profile_list
def client_wrapper(service, profile='default', region=DEFAULT_REGION):
"""
Summary.
Single caller boto3 service wrapper. Instantiates client object while
using temporary credientials for profile_name, if available in
local configuration. Tests authentication prior to returning any
client object.
Args:
:service (str): boto3 service abbreviation ('ec2', 's3', etc)
:profile (str): profile_name of an iam user from local awscli config
:region (str): AWS region code, optional
Returns:
client (boto3 object)
"""
profile_name = _profile_prefix(profile)
try:
if authenticated(profile_name):
return boto3_session(service=service,
profile=profile_name,
region=region)
except ClientError as e:
logger.exception(
"%s: Unknown boto3 failure while establishing session (Code: %s Message: %s)"
% (inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
return None
def _profile_prefix(profile, prefix='gcreds'):
"""
Summary:
Determines if temporary STS credentials provided via
local awscli config;
- if yes, returns profile with correct prefix
- if no, returns profile (profile_name) unaltered
- Note: Caller is parse_profiles(), Not to be called directly
Args:
profile (str): profile_name of a valid profile from local awscli config
prefix (str): prefix prepended to profile containing STS temporary credentials
Returns:
awscli profilename, TYPE str
"""
stderr = ' 2>/dev/null'
tempProfile = prefix + '-' + profile
try:
if subprocess.getoutput(
f'aws configure get profile.{profile}.aws_access_key_id {stderr}'
):
return profile
elif subprocess.getoutput(
f'aws configure get profile.{tempProfile}.aws_access_key_id {stderr}'
):
return tempProfile
except Exception as e:
logger.exception(
f'{inspect.stack()[0][3]}: Unknown error while interrogating local awscli config: {e}'
)
raise
return None
def authenticated(profile):
"""
Summary:
Tests generic authentication status to AWS Account
Args:
:profile (str): iam user name from local awscli configuration
Returns:
TYPE: bool, True (Authenticated)| False (Unauthenticated)
"""
try:
sts_client = boto3_session(service='sts', profile=profile)
httpstatus = sts_client.get_caller_identity(
)['ResponseMetadata']['HTTPStatusCode']
if httpstatus == 200:
return True
except ClientError as e:
if e.response['Error']['Code'] == 'InvalidClientTokenId':
logger.info(
'%s: Invalid credentials to authenticate for profile user (%s). Exit. [Code: %d]'
% (inspect.stack()[0][3], profile,
exit_codes['EX_NOPERM']['Code']))
elif e.response['Error']['Code'] == 'ExpiredToken':
logger.info(
'%s: Expired temporary credentials detected for profile user (%s) [Code: %d]'
% (inspect.stack()[0][3], profile,
exit_codes['EX_CONFIG']['Code']))
else:
logger.exception(
'%s: Unknown Boto3 problem. Error: %s' %
(inspect.stack()[0][3], e.response['Error']['Message']))
except Exception as e:
return False
return False
def boto3_session(service, region=DEFAULT_REGION, profile=None):
"""
Summary:
Establishes boto3 sessions, client
Args:
:service (str): boto3 service abbreviation ('ec2', 's3', etc)
:profile (str): profile_name of an iam user from local awscli config
Returns:
TYPE: boto3 client object
"""
try:
if profile and profile != 'default':
session = boto3.Session(profile_name=profile)
return session.client(service, region_name=region)
except ClientError as e:
logger.exception(
"%s: IAM user or role not found (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise
except ProfileNotFound:
msg = ('%s: The profile (%s) was not found in your local config' %
(inspect.stack()[0][3], profile))
stdout_message(msg, 'FAIL')
logger.warning(msg)
return boto3.client(service, region_name=region)
def stopped_instances(region, profile=None, ids=False, debug=False):
"""
Summary.
Determines state of all ec2 machines in a region
Returns:
:stopped ec2 instances, TYPE: ec2 objects
OR
:stopped ec2 instance ids, TYPE: str
"""
try:
if profile and profile != 'default':
session = boto3.Session(profile_name=profile)
ec2 = session.resource('ec2', region_name=region)
else:
ec2 = boto3.resource('ec2', region_name=region)
instances = ec2.instances.all()
if ids:
return [x.id for x in instances if x.state['Name'] == 'stopped']
except ClientError as e:
logger.exception(
"%s: IAM user or role not found (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise
except ProfileNotFound:
msg = ('%s: The profile (%s) was not found in your local config' %
(inspect.stack()[0][3], profile))
stdout_message(msg, 'FAIL')
logger.warning(msg)
return [x for x in instances if x.state['Name'] == 'stopped']
def query_dynamodb(self, partition_key, key_value):
"""
Queries DynamoDB table using partition key,
returns the item matching key value
"""
try:
resource_dynamodb = self.boto_dynamodb_resource(self.region)
table = resource_dynamodb.Table(self.tablename)
logger.info('Table %s: Table Item Count is: %s' %
(table.table_name, table.item_count))
# query on parition key
response = table.query(
KeyConditionExpression=Key(partition_key).eq(str(key_value)))
if response['Items']:
item = response['Items'][0]['Account Name']
else:
item = 'unIdentified'
except ClientError as e:
logger.exception(
"Couldn\'t query DynamoDB table (Code: %s Message: %s)" %
(e.response['Error']['Code'], e.response['Error']['Message']))
return 1
return item
def assume_role(self, aws_account_id, service_role):
"""
Summary.
Assumes a DynamoDB role in 'destination' AWS account
Args:
aws_account_id (str): 12 digit AWS Account number containing dynamodb table
service_role (str): IAM role dynamodb service containing permissions
allowing interaction with dynamodb
Returns:
temporary credentials for service_role when assumed, TYPE: json
"""
session = boto3.Session()
sts_client = session.client('sts')
try:
# assume role in destination account
assumed_role = sts_client.assume_role(
RoleArn="arn:aws:iam::%s:role/%s" %
(str(aws_account_id), service_role),
RoleSessionName="DynamoDBReaderSession")
except ClientError as e:
logger.exception(
"Couldn't assume role to read DynamoDB, account " +
str(aws_account_id) +
" (switching role) (Code: %s Message: %s)" %
(e.response['Error']['Code'], e.response['Error']['Message']))
raise e
return assumed_role['Credentials']
def sns_notification(topic_arn,
subject,
message,
account_id=None,
account_name=None):
"""
Summary.
Sends message to AWS sns service topic provided as a
parameter
Args:
topic_arn (str): sns topic arn
subject (str): subject of sns message notification
message (str): message body
Returns:
TYPE: Boolean | Success or Failure
"""
if not (account_id or account_name):
account_id, account_name = get_account_info()
# assemble msg
header = 'AWS Account: %s (%s) | %s' % \
(str(account_name).upper(), str(account_id), subject)
msg = '\n%s\n\n%s' % (time.strftime('%c'), message)
msg_dict = {'default': msg}
# client
region = (topic_arn.split('sns:', 1)[1]).split(":", 1)[0]
client = boto3.client('sns', region_name=region)
try:
# sns publish
response = client.publish(TopicArn=topic_arn,
Subject=header,
Message=json.dumps(msg_dict),
MessageStructure='json')
if response['ResponseMetadata']['HTTPStatusCode'] == '200':
return True
else:
return False
except ClientError as e:
logger.exception('%s: problem sending sns msg (Code: %s Message: %s)' %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
return False
def boto_dynamodb_resource(self, region):
"""
Initiates boto resource to communicate with AWS API
"""
try:
dynamodb_resource = boto3.resource(
'dynamodb',
aws_access_key_id=self.aws_credentials['AccessKeyId'],
aws_secret_access_key=self.aws_credentials['SecretAccessKey'],
aws_session_token=self.aws_credentials['SessionToken'],
region_name=region)
except ClientError as e:
logger.exception(
"Unknown problem creating boto3 resource (Code: %s Message: %s)"
%
(e.response['Error']['Code'], e.response['Error']['Message']))
return 1
return dynamodb_resource
def scan_accounts(self, account_type):
"""
Read method for DynamoDB table
"""
accounts, account_ids = [], []
valid_mpc_pkgs = [
'B', 'RA-PKG-B', 'RA-PKG-C', 'P', 'ATA', 'BUP', 'DXA'
]
types = [x.strip(' ') for x in account_type.split(',')] # parse types
try:
resource_dynamodb = self.boto_dynamodb_resource(self.region)
table = resource_dynamodb.Table(self.tablename)
# scan table
if set(types).issubset(set(valid_mpc_pkgs)):
for type in types:
response = table.scan(
FilterExpression=Attr('MPCPackage').eq(type))
for account_dict in response['Items']:
accounts.append(account_dict)
elif types[0] == 'All':
# all valid_mpc_pkgs accounts (commercial accounts)
response = table.scan(
FilterExpression=Attr('MPCPackage').ne("P"))
for account_dict in response['Items']:
accounts.append(account_dict)
except ClientError as e:
logger.exception(
"Couldn\'t scan DynamoDB table (Code: %s Message: %s)" %
(e.response['Error']['Code'], e.response['Error']['Message']))
return 1
if accounts:
for account in accounts:
account_info = {}
account_info['AccountName'] = account['Account Name']
account_info['AccountId'] = account['Account ID']
account_ids.append(account_info)
else:
logger.info('No items returned from DyanamoDB')
return account_ids
def read_env_variable(arg, default=None, patterns=None):
"""
Summary.
Parse environment variables, validate characters, convert
type(s). default should be used to avoid conversion of an
variable and retain string type
Usage:
>>> from lambda_utils import read_env_variable
>>> os.environ['DBUGMODE'] = 'True'
>>> myvar = read_env_variable('DBUGMODE')
>>> type(myvar)
True
>>> from lambda_utils import read_env_variable
>>> os.environ['MYVAR'] = '1345'
>>> myvar = read_env_variable('MYVAR', 'default')
>>> type(myvar)
str
Args:
:arg (str): Environment variable name (external name)
:default (str): Default if no variable found in the environment under
name in arg parameter
:patterns (None): Unused; not user callable. Used preservation of the
patterns tuple between calls during runtime
Returns:
environment variable value, TYPE str
"""
if patterns is None:
patterns = (
(re.compile('^[-+]?[0-9]+$'), int),
(re.compile('\d+\.\d+'), float),
(re.compile(r'^(true|false)$',
flags=re.IGNORECASE), lambda x: x.lower() == 'true'),
(re.compile('[a-z/]+', flags=re.IGNORECASE), str),
(re.compile('[a-z/]+\.[a-z/]+', flags=re.IGNORECASE), str),
)
if arg in os.environ:
var = os.environ[arg]
if var is None:
ex = KeyError('environment variable %s not set' % arg)
logger.exception(ex)
raise ex
else:
if default:
return str(var) # force default type (str)
else:
for pattern, func in patterns:
if pattern.match(var):
return func(var)
# type not identified
logger.warning(
'%s: failed to identify environment variable [%s] type. May contain \
special characters' % (inspect.stack()[0][3], arg))
return str(var)
else:
ex = KeyError('environment variable %s not set' % arg)
logger.exception(ex)
raise ex
def main():
""" copies ec2 instance tags to attached resources """
for profile in profiles:
# derive account alias from profile
account = '-'.join(profile.split('-')[1:])
for region in regions:
#instances = []
session = boto3.Session(profile_name=profile, region_name=region)
client = session.client('ec2')
ec2 = session.resource('ec2')
instances = get_instances(profile, region)
volumes = get_volumes(profile, region)
# print summary
if SUMMARY_REPORT:
print('\nFor AWS Account %s, region %s, Found %d Instances\n' %
(account, region, len(instances)))
continue
# copy tags
if instances:
try:
base = ec2.instances.filter(InstanceIds=instances)
ct = 0
for instance in base:
ids, after_tags = [], []
ct += 1
if instance.tags:
# filter out tags to prohibited from copy
filtered_tags = filter_tags(
instance.tags, *NO_COPY_LIST)
else:
# no tags on instance to copy
continue
if not valid_tags(filtered_tags):
print('\nWARNING:')
logger.warning(
'Skipping instance ID %s, Invalid Tags\n' %
instance.id)
continue
# collect attached resource ids to be tagged
for vol in instance.volumes.all():
ids.append(vol.id)
for eni in instance.network_interfaces:
ids.append(eni.id)
logger.info('InstanceID %s, instance %d of %d:' %
(instance.id, ct, len(instances)))
logger.info('Resource Ids to tag is:')
logger.info(str(ids) + '\n')
if DEBUGMODE:
# BEFORE tag copy
logger.info('BEFORE list of %d tags is:' %
(len(instance.tags)))
pretty_print_tags(instance.tags)
# AFTER tag copy | put Name tag back into apply tags, ie, after_tags
retain_tags = select_tags(instance.tags,
PRESERVE_TAGS)
for tag in (*retain_tags, *filtered_tags):
after_tags.append(tag)
logger.info(
'For InstanceID %s, the AFTER FILTERING list of %d tags is:'
% (instance.id, len(after_tags)))
logger.info('Tags to apply are:')
pretty_print_tags(after_tags)
else:
logger.info('InstanceID %s, instance %d of %d:' %
(instance.id, ct, len(instances)))
if filtered_tags: # we must have something to apply
# apply tags
for resourceId in ids:
# retain a copy of tags to preserve if is a volume
if resourceId.startswith('vol-'):
r = client.describe_tags(Filters=[
{
'Name': 'resource-id',
'Values': [resourceId],
},
])
retain_tags = select_tags(
r['Tags'], PRESERVE_TAGS)
# add retained tags before appling to volume
if retain_tags:
for tag in retain_tags:
filtered_tags.append(tag)
# clear tags
print('\n')
logger.info(
'Clearing tags on resource: %s' %
str(resourceId))
client.delete_tags(Resources=[resourceId],
Tags=[])
# create new tags
logger.info(
'Applying tags to resource %s\n' %
resourceId)
ec2.create_tags(Resources=[resourceId],
Tags=filtered_tags)
# delay to throttle API requests
sleep(1)
except ClientError as e:
logger.exception(
"%s: Problem (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise
