def boto3_session(service, region=DEFAULT_REGION, profile=None):
"""
Summary:
Establishes boto3 sessions, client
Args:
:service (str): boto3 service abbreviation ('ec2', 's3', etc)
:profile (str): profile_name of an iam user from local awscli config
Returns:
TYPE: boto3 client object
"""
try:
if profile and profile != 'default':
session = boto3.Session(profile_name=profile)
return session.client(service, region_name=region)
except ClientError as e:
logger.exception(
"%s: IAM user or role not found (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise
except ProfileNotFound:
msg = ('%s: The profile (%s) was not found in your local config' %
(inspect.stack()[0][3], profile))
stdout_message(msg, 'FAIL')
logger.warning(msg)
return boto3.client(service, region_name=region)
def stopped_instances(region, profile=None, ids=False, debug=False):
"""
Summary.
Determines state of all ec2 machines in a region
Returns:
:stopped ec2 instances, TYPE: ec2 objects
OR
:stopped ec2 instance ids, TYPE: str
"""
try:
if profile and profile != 'default':
session = boto3.Session(profile_name=profile)
ec2 = session.resource('ec2', region_name=region)
else:
ec2 = boto3.resource('ec2', region_name=region)
instances = ec2.instances.all()
if ids:
return [x.id for x in instances if x.state['Name'] == 'stopped']
except ClientError as e:
logger.exception(
"%s: IAM user or role not found (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise
except ProfileNotFound:
msg = ('%s: The profile (%s) was not found in your local config' %
(inspect.stack()[0][3], profile))
stdout_message(msg, 'FAIL')
logger.warning(msg)
return [x for x in instances if x.state['Name'] == 'stopped']
def log_message(label, msg):
""" logs all messages sent to stdout """
if label in critical_status:
logger.critical(msg)
elif label in warning_status:
logger.warning(msg)
else:
logger.info(msg)
return True
def init_cli():
# parser = argparse.ArgumentParser(add_help=False, usage=help_menu())
parser = argparse.ArgumentParser(add_help=False)
try:
args = options(parser)
except Exception as e:
help_menu()
stdout_message(str(e), 'ERROR')
sys.exit(exit_codes['EX_OK']['Code'])
if len(sys.argv) == 1:
help_menu()
sys.exit(exit_codes['EX_OK']['Code'])
elif args.help:
help_menu()
sys.exit(exit_codes['EX_OK']['Code'])
elif args.version:
package_version()
elif args.configure:
r = option_configure(args.debug,
local_config['PROJECT']['CONFIG_PATH'])
return r
else:
if precheck(): # if prereqs set, run
if authenticated(profile=args.profile):
# execute keyset operation
success = main(operation=args.operation,
profile=args.profile,
user_name=args.username,
auto=args.auto,
debug=args.debug)
if success:
logger.info('IAM access keyset operation complete')
sys.exit(exit_codes['EX_OK']['Code'])
else:
stdout_message(
'Authenication Failed to AWS Account for user %s' %
args.profile,
prefix='AUTH',
severity='WARNING')
sys.exit(exit_codes['E_AUTHFAIL']['Code'])
failure = """ : Check of runtime parameters failed for unknown reason.
Please ensure local awscli is configured. Then run keyconfig to
configure keyup runtime parameters. Exiting. Code: """
logger.warning(failure +
'Exit. Code: %s' % sys.exit(exit_codes['E_MISC']['Code']))
print(failure)
def boto3_session(service, region=DEFAULT_REGION, profile=None):
"""
Summary:
Establishes boto3 sessions, client
Args:
:service (str): boto3 service abbreviation ('ec2', 's3', etc)
:profile (str): profile_name of an iam user from local awscli config
:region (str): AWS region code, optional
Returns:
client (boto3 object)
"""
fx = inspect.stack()[0][3]
try:
if (not profile or profile == 'default') and service != 'iam':
return boto3.client(service, region_name=region)
elif (not profile or profile == 'default') and service == 'iam':
return boto3.client(service)
elif profile and profile != 'default':
session = boto3.Session(profile_name=profile)
return session.client(service, region_name=region)
except ClientError as e:
if e.response['Error']['Code'] == 'InvalidClientTokenId':
logger.warning(
'{}: Invalid credentials used by profile user {}'.format(
fx, profile or 'default'))
elif e.response['Error']['Code'] == 'ExpiredToken':
logger.info(
'%s: Expired temporary credentials detected for profile user (%s) [Code: %d]'
% (fx, profile, exit_codes['EX_CONFIG']['Code']))
except ProfileNotFound:
msg = (
'{}: Profile name {} was not found in your local config.'.format(
fx, profile))
stdout_message(msg, 'WARN')
logger.warning(msg)
return None
return boto3.client(service, region_name=region)
def get_account_info(account_profile=None):
"""
Summary.
Queries AWS iam and sts services to discover account id information
in the form of account name and account alias (if assigned)
Returns:
TYPE: tuple
Example usage:
>>> account_number, account_name = lambda_utils.get_account_info()
>>> print(account_number, account_name)
103562488773 tooling-prod
"""
if account_profile:
session = boto3.Session(profile_name=account_profile)
sts_client = session.client('sts')
iam_client = session.client('iam')
else:
sts_client = boto3.client('sts')
iam_client = boto3.client('iam')
try:
number = sts_client.get_caller_identity()['Account']
name = iam_client.list_account_aliases()['AccountAliases'][0]
except IndexError as e:
name = '<no_alias_assigned>'
logger.info(
'Error: %s. No account alias defined. account_name set to %s' %
(e, name))
return (number, name)
except ClientError as e:
logger.warning(
"%s: problem retrieving caller identity (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise e
return (number, name)
def delete_tags(resourceIds, region, tags):
""" Removes tags from an EC2 resource """
client = boto3_session('ec2', region)
try:
for resourceid in resourceIds:
response = client.delete_tags(
Resources=[resourceid],
Tags=tags
)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
logger.info('Existing Tags deleted from vol id %s' % resourceid)
return True
else:
logger.warning('Problem deleting existing tags from vol id %s' % resourceid)
return False
except ClientError as e:
logger.critical(
"%s: Problem apply tags to ec2 instances (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'], e.response['Error']['Message']))
return False
def export_json_object(dict_obj, filename=None, logging=True):
"""
Summary:
exports object to block filesystem object
Args:
:dict_obj (dict): dictionary object
:filename (str): name of file to be exported (optional)
Returns:
True | False Boolean export status
"""
try:
if filename:
try:
with open(filename, 'w') as handle:
handle.write(json.dumps(dict_obj, indent=4, sort_keys=True))
logger.info(
'%s: Wrote %s to local filesystem location' %
(inspect.stack()[0][3], filename))
handle.close()
except TypeError as e:
logger.warning(
'%s: object in dict not serializable: %s' %
(inspect.stack()[0][3], str(e)))
else:
json_str = json.dumps(dict_obj, indent=4, sort_keys=True)
print(highlight(json_str, lexers.JsonLexer(), formatters.TerminalFormatter()).strip())
if logging:
logger.info('%s: successful export to stdout' % inspect.stack()[0][3])
return True
except OSError as e:
logger.critical(
'%s: export_file_object: error writing to %s to filesystem. Error: %s' %
(inspect.stack()[0][3], filename, str(e)))
return False
if logging:
logger.info('export_file_object: successful export to %s' % filename)
return True
def authenticated(profile):
"""
Tests generic authentication status to AWS Account
Args:
:profile (str): iam user name from local awscli configuration
Returns:
TYPE: bool, True (Authenticated)| False (Unauthenticated)
"""
try:
sts_client = boto3_session(service='sts', profile=profile)
httpstatus = sts_client.get_caller_identity(
)['ResponseMetadata']['HTTPStatusCode']
if httpstatus == 200:
return True
except ClientError as e:
if e.response['Error']['Code'] == 'InvalidClientTokenId':
logger.warning(
'%s: Invalid credentials to authenticate for profile user (%s). Exit. [Code: %d]'
% (inspect.stack()[0][3], profile,
exit_codes['EX_NOPERM']['Code']))
elif e.response['Error']['Code'] == 'ExpiredToken':
logger.info(
'%s: Expired temporary credentials detected for profile user (%s) [Code: %d]'
% (inspect.stack()[0][3], profile,
exit_codes['EX_CONFIG']['Code']))
else:
logger.exception(
'%s: Unknown Boto3 problem. Error: %s' %
(inspect.stack()[0][3], e.response['Error']['Message']))
except Exception as e:
fx = inspect.stack()[0][3]
logger.exception('{}: Unknown error: {}'.format(fx, e))
return False
return False
def read_env_variable(arg, default=None, patterns=None):
"""
Summary.
Parse environment variables, validate characters, convert
type(s). default should be used to avoid conversion of an
variable and retain string type
Usage:
>>> from lambda_utils import read_env_variable
>>> os.environ['DBUGMODE'] = 'True'
>>> myvar = read_env_variable('DBUGMODE')
>>> type(myvar)
True
>>> from lambda_utils import read_env_variable
>>> os.environ['MYVAR'] = '1345'
>>> myvar = read_env_variable('MYVAR', 'default')
>>> type(myvar)
str
Args:
:arg (str): Environment variable name (external name)
:default (str): Default if no variable found in the environment under
name in arg parameter
:patterns (None): Unused; not user callable. Used preservation of the
patterns tuple between calls during runtime
Returns:
environment variable value, TYPE str
"""
if patterns is None:
patterns = (
(re.compile('^[-+]?[0-9]+$'), int),
(re.compile('\d+\.\d+'), float),
(re.compile(r'^(true|false)$',
flags=re.IGNORECASE), lambda x: x.lower() == 'true'),
(re.compile('[a-z/]+', flags=re.IGNORECASE), str),
(re.compile('[a-z/]+\.[a-z/]+', flags=re.IGNORECASE), str),
)
if arg in os.environ:
var = os.environ[arg]
if var is None:
ex = KeyError('environment variable %s not set' % arg)
logger.exception(ex)
raise ex
else:
if default:
return str(var) # force default type (str)
else:
for pattern, func in patterns:
if pattern.match(var):
return func(var)
# type not identified
logger.warning(
'%s: failed to identify environment variable [%s] type. May contain \
special characters' % (inspect.stack()[0][3], arg))
return str(var)
else:
ex = KeyError('environment variable %s not set' % arg)
logger.exception(ex)
raise ex
def main():
""" copies ec2 instance tags to attached resources """
for profile in profiles:
# derive account alias from profile
account = '-'.join(profile.split('-')[1:])
for region in regions:
#instances = []
session = boto3.Session(profile_name=profile, region_name=region)
client = session.client('ec2')
ec2 = session.resource('ec2')
instances = get_instances(profile, region)
volumes = get_volumes(profile, region)
# print summary
if SUMMARY_REPORT:
print('\nFor AWS Account %s, region %s, Found %d Instances\n' %
(account, region, len(instances)))
continue
# copy tags
if instances:
try:
base = ec2.instances.filter(InstanceIds=instances)
ct = 0
for instance in base:
ids, after_tags = [], []
ct += 1
if instance.tags:
# filter out tags to prohibited from copy
filtered_tags = filter_tags(
instance.tags, *NO_COPY_LIST)
else:
# no tags on instance to copy
continue
if not valid_tags(filtered_tags):
print('\nWARNING:')
logger.warning(
'Skipping instance ID %s, Invalid Tags\n' %
instance.id)
continue
# collect attached resource ids to be tagged
for vol in instance.volumes.all():
ids.append(vol.id)
for eni in instance.network_interfaces:
ids.append(eni.id)
logger.info('InstanceID %s, instance %d of %d:' %
(instance.id, ct, len(instances)))
logger.info('Resource Ids to tag is:')
logger.info(str(ids) + '\n')
if DEBUGMODE:
# BEFORE tag copy
logger.info('BEFORE list of %d tags is:' %
(len(instance.tags)))
pretty_print_tags(instance.tags)
# AFTER tag copy | put Name tag back into apply tags, ie, after_tags
retain_tags = select_tags(instance.tags,
PRESERVE_TAGS)
for tag in (*retain_tags, *filtered_tags):
after_tags.append(tag)
logger.info(
'For InstanceID %s, the AFTER FILTERING list of %d tags is:'
% (instance.id, len(after_tags)))
logger.info('Tags to apply are:')
pretty_print_tags(after_tags)
else:
logger.info('InstanceID %s, instance %d of %d:' %
(instance.id, ct, len(instances)))
if filtered_tags: # we must have something to apply
# apply tags
for resourceId in ids:
# retain a copy of tags to preserve if is a volume
if resourceId.startswith('vol-'):
r = client.describe_tags(Filters=[
{
'Name': 'resource-id',
'Values': [resourceId],
},
])
retain_tags = select_tags(
r['Tags'], PRESERVE_TAGS)
# add retained tags before appling to volume
if retain_tags:
for tag in retain_tags:
filtered_tags.append(tag)
# clear tags
print('\n')
logger.info(
'Clearing tags on resource: %s' %
str(resourceId))
client.delete_tags(Resources=[resourceId],
Tags=[])
# create new tags
logger.info(
'Applying tags to resource %s\n' %
resourceId)
ec2.create_tags(Resources=[resourceId],
Tags=filtered_tags)
# delay to throttle API requests
sleep(1)
except ClientError as e:
logger.exception(
"%s: Problem (Code: %s Message: %s)" %
(inspect.stack()[0][3], e.response['Error']['Code'],
e.response['Error']['Message']))
raise