def _session(self, region, account_id=None):
# instantiate EC2 session
if account_id is None:
return EC2(self.logger, region)
else:
return EC2(self.logger,
region,
credentials=self.assume_role(self.logger, account_id))
def create_vpc_peering_connection(self):
try:
ssm = SSM(self.logger)
requester_account_id = self.params.get('PeeringAccountID')
requester_vpc_id = self.params.get('PeeringVPCID')
requester_region = self.params.get('PeeringRegion')
accepter_region = self.params.get('Region')
accepter_account_id = self.params.get('AccountID')
accepter_vpc_id = self.params.get('VPCID')
# instantiate EC2 sessions
ec2_peer_requester = EC2(self.logger, requester_region, credentials=self.assume_role(requester_account_id))
ec2_peer_accepter = EC2(self.logger, accepter_region, credentials=self.assume_role(accepter_account_id))
# request vpc peering connection
response = ec2_peer_requester.create_vpc_peering_connection(accepter_vpc_id,
requester_vpc_id,
accepter_account_id,
accepter_region)
#
peer_connection_id = response.get('VpcPeeringConnection', {}).get('VpcPeeringConnectionId')
self.check_peering_status(ec2_peer_requester, peer_connection_id, ['pending-acceptance', 'active'], ['failed', 'rejected'])
# accept vpc peering
resp_peer_connection_accept = ec2_peer_accepter.accept_vpc_peering_connection(peer_connection_id)
accepter_peer_connection_info = resp_peer_connection_accept.get('VpcPeeringConnection')
if accepter_peer_connection_info.get('Status').get('Code') is not 'active':
self.check_peering_status(ec2_peer_requester, peer_connection_id, ['active'], ['failed', 'rejected'])
# get vpc details
requester_vpc_cidr = response.get('VpcPeeringConnection', {}).get('RequesterVpcInfo').get('CidrBlock')
accepter_vpc_cidr = accepter_peer_connection_info.get('AccepterVpcInfo').get('CidrBlock')
# write peering connection id to SSM Parameter store
requester_account_name = self.params.get('AccountName')
prefix = self.params.get('PeeringConnectionKeyPrefix')
suffix = 'peering_connections'
peering_connection_id_parameter_key = '{}/{}/{}'.format(prefix, suffix, requester_account_name)
self.logger.info("SSM Parameter key: {} has been written".format(peering_connection_id_parameter_key))
ssm.put_parameter(peering_connection_id_parameter_key, peer_connection_id)
return {
'PeerConnectionID': peer_connection_id,
'RequesterAccountID': requester_account_id,
'AccepterAccountID': accepter_account_id,
'RequesterVPCCIDR': requester_vpc_cidr,
'AccepterVPCCIDR': accepter_vpc_cidr
}
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def expunge_default_vpc(self):
ec2 = EC2(self.logger, self.params.get('Region'))
account_list = self.params.get('AccountList')
# checking if we member account IDs are present in the parameters
if account_list is not None:
for region in ec2.describe_regions():
try:
self.logger.info('~' * 75)
self.logger.info(
"Deleting default VPCs from the member account in "
"region: {}".format(region.get('RegionName')))
# Calling method to delete VPCs in the member account
self.member_expunge_default_vpc(account_list,
region.get('RegionName'))
except Exception as e:
message = {
'FILE': __file__.split('/')[-1],
'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3],
'EXCEPTION': str(e)
}
self.logger.exception(message)
raise
else:
self.logger.info("'AccountList' key not found in the properties")
raise Exception("'AccountList' key not found in the properties")
def get_azs_from_member_account(self, region, qty, account, key_az=None):
"""gets a predefined quantity of (random) az's from a specified region
Args:
region (str): region name
qty: quantity of az's to return
account: account id of the member account
Returns:
list: availability zone names
"""
try:
if key_az:
self.logger.info("Looking up values in SSM parameter:{}".format(key_az))
existing_param = self.ssm.describe_parameters(key_az)
if existing_param:
self.logger.info('Found existing SSM parameter, returning exising AZ list.')
return self.ssm.get_parameter(key_az)
if account is not None:
ec2 = self._session(region, account)
self.logger.info("Getting list of AZs in region: {} from account: {}".format(region, account))
return self._get_az(ec2, key_az, qty)
else:
self.logger.info("Creating EC2 Session in {} region".format(region))
ec2 = EC2(self.logger, region)
return self._get_az(ec2, key_az, qty)
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def get_azs_from_member_account(self, region, qty, account, key_az=None):
"""gets a predefined quantity of (random) az's from a specified region
Args:
region (str): region name
qty: quantity of az's to return
account: account id of the member account
key_az (str): ssm parameter store key where existing
AZ list is stored
Returns:
list: availability zone names
"""
if key_az:
self.logger.info(
"Looking up values in SSM parameter:{}".format(key_az))
existing_param = self.ssm.describe_parameters(key_az)
if existing_param:
self.logger.info('Found existing SSM parameter, returning'
' existing AZ list.')
return self.ssm.get_parameter(key_az)
if account is not None:
acct = account[0] if isinstance(account, list) else account
ec2 = self._session(region, acct)
self.logger.info("Getting list of AZs in region: {} from"
" account: {}".format(region, acct))
return self._get_az(ec2, key_az, qty)
else:
self.logger.info(
"Creating EC2 Session in {} region".format(region))
ec2 = EC2(self.logger, region)
return self._get_az(ec2, key_az, qty)
def delete_vpc_peering_routing(self):
try:
# declare variables
peer_connection_id = self.params.get('PeerConnectionID')
vpc_cidr = self.params.get('VPCCIDR')
account_id = self.params.get('AccountID')
route_table_ids_str = self.params.get('RouteTableIDs')
route_table_ids = route_table_ids_str.split(",")
region = self.params.get('Region')
# instantiate EC2 sessions
ec2 = EC2(self.logger, region, credentials=self.assume_role(account_id))
# change routes in all the peer vpc's route tables
for id in route_table_ids:
# Retrieve routes, from the route table, related to only this
# peer connection
response = ec2.describe_route_tables(
id, peer_connection_id).get('RouteTables')
if len(response) > 0:
for item in response[0].get('Routes'):
if item.get('VpcPeeringConnectionId') == peer_connection_id:
response = ec2.delete_route(vpc_cidr, id)
self.logger.debug(response.get('Return'))
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def update_vpc_peering_routing(self):
try:
# declare variables
peer_connection_id = self.params.get('PeerConnectionID')
vpc_cidr = self.params.get('VPCCIDR')
account_id = self.params.get('AccountID')
route_table_ids_str = self.params.get('RouteTableIDs')
route_table_ids = route_table_ids_str.split(",")
region = self.params.get('Region')
# instantiate EC2 sessions
ec2 = EC2(self.logger, region, credentials=self.assume_role(account_id))
# change routes in all the peer vpc's route tables
for id in route_table_ids:
response = ec2.update_route(vpc_cidr, id, peer_connection_id)
if response.get('Return'):
self.logger.info('Route table {} updated successfully'.format(id))
else:
raise Exception("Failed to update the Route table : {} with route to PeerConnectionID : {} "
"for VPC CIDR: {}".format(id, peer_connection_id, vpc_cidr))
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def delete_vpc_peering_connection(self):
try:
ssm = SSM(self.logger)
# read values from SSM Parameter Store
requester_account_name = self.params.get('AccountName')
requester_account_id = self.params.get('PeeringAccountID')
requester_region = self.params.get('Region')
self.logger.debug(requester_account_id)
self.logger.info("Peering Account ID: {}".format(requester_account_id))
prefix = self.params.get('PeeringConnectionKeyPrefix')
suffix = 'peering_connections'
peer_connection_id_parameter_key = '{}/{}/{}'.format(prefix, suffix, requester_account_name)
peer_connection_id = ssm.get_parameter(peer_connection_id_parameter_key)
self.logger.debug(peer_connection_id)
self.logger.info("Peering Connection ID: {}".format(peer_connection_id))
# instantiate EC2 session
ec2_peer_requester = EC2(self.logger, requester_region, credentials=self.assume_role(requester_account_id))
# delete vpc peering connection id
response = ec2_peer_requester.delete_vpc_peering_connection(peer_connection_id)
self.logger.info("Peering Connection ID: {} deleted".format(peer_connection_id))
self.logger.debug(response)
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def update_vpc_peering_connection(self):
try:
requester_account_id = self.params.get('PeeringAccountID')
requester_vpc_id = self.params.get('PeeringVPCID')
requester_region = self.params.get('PeeringRegion')
accepter_account_id = self.params.get('AccountID')
accepter_vpc_id = self.params.get('VPCID')
# instantiate EC2 sessions
ec2_peer_requester = EC2(self.logger, requester_region, credentials=self.assume_role(requester_account_id))
response = ec2_peer_requester.describe_vpc_peering_connections_by_filters(requester_account_id, requester_vpc_id, accepter_account_id, accepter_vpc_id)
if response.get('VpcPeeringConnections'):
peer_conn = response.get('VpcPeeringConnections')[0]
existing_peering_id = peer_conn.get('VpcPeeringConnectionId')
requester_vpc_cidr = peer_conn.get('RequesterVpcInfo').get('CidrBlock')
accepter_vpc_cidr = peer_conn.get('AccepterVpcInfo').get('CidrBlock')
else:
return self.create_vpc_peering_connection()
return {
'PeerConnectionID': existing_peering_id,
'RequesterAccountID': requester_account_id,
'AccepterAccountID': accepter_account_id,
'RequesterVPCCIDR': requester_vpc_cidr,
'AccepterVPCCIDR': accepter_vpc_cidr
}
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def member_expunge_default_vpc(self, accounts, region):
# instantiate STS class
sts = STS(self.logger)
# iterate through all the members in the list
for account in accounts:
#role_arn = "arn:aws:iam::" + str(account) + ":role/AWSCloudFormationStackSetExecutionRole"
role_arn = "arn:aws:iam::" + str(account) + ":role/pac-devops"
session_name = "expunge_default_vpc_role"
# assume role
credentials = sts.assume_role(role_arn, session_name)
self.logger.info("Assuming IAM role: {}".format(role_arn))
# instantiate EC2 class using temporary security credentials
self.logger.debug("Creating EC2 Session in {} for account: {}".format(region, account))
ec2 = EC2(self.logger, region, credentials=credentials)
if type(credentials) == dict:
response = ec2.describe_vpcs()
self.logger.info(response)
if not response.get('Vpcs'):
self.logger.info(
"There is no default VPC to delete in {} (member account: {}).".format(region, account))
else:
for vpc in response.get('Vpcs'):
vpc_id = vpc.get('VpcId')
default = True if vpc.get('IsDefault') is True else False
if default:
self.logger.info("Found the default VPC: {}".format(vpc_id))
# Delete dependencies (calling method)
self.logger.info(
"Deleting dependencies for member account ID: {} in {}".format(account, region))
self.delete_vpc_dependencies(ec2, region, vpc_id)
# Delete VPC
self.logger.info(
"Deleting VPC: {} in member account ID: {} in {}".format(vpc_id, account, region))
self.logger.info(ec2.delete_vpc(vpc_id))
else:
self.logger.info("{} is not the default VPC, skipping...".format(vpc_id))
else:
self.logger.error("Unable to obtain credentials")
def get_azs(self, region, qty, key_az=None):
"""gets a predefined quantity of (random) az's from a specified region
Args:
region (str): region name
qty: quantity of az's to return
Returns:
list: availability zone names
"""
try:
if key_az:
self.logger.info(
"Looking up values in SSM parameter:{}".format(key_az))
existing_param = self.ssm.describe_parameters(key_az)
if existing_param:
return self.ssm.get_parameter(key_az)
self.logger.info(
"Creating EC2 Session in {} region".format(region))
ec2 = EC2(self.logger, region)
# Get AZs
self.logger.info(
"Getting list of AZs in region: {}".format(region))
az_list = ec2.describe_availability_zones()
self.logger.info("_get_azs output: %s" % az_list)
random_az_list = ','.join(random.sample(az_list, qty))
description = "Contains random AZs selected by Landing Zone Solution"
if key_az:
self.ssm.put_parameter(key_az, random_az_list, description)
return random_az_list
except Exception as e:
message = {
'FILE': __file__.split('/')[-1],
'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3],
'EXCEPTION': str(e)
}
self.logger.exception(message)
raise
def delete_vpc_peering_routing(self):
try:
# declare variables
vpc_cidr = self.params.get('VPCCIDR')
account_id = self.params.get('AccountID')
route_table_ids_str = self.params.get('RouteTableIDs')
route_table_ids = route_table_ids_str.split(",")
region = self.params.get('Region')
# instantiate EC2 sessions
ec2 = EC2(self.logger, region, credentials=self.assume_role(account_id))
# change routes in all the peer vpc's route tables
for id in route_table_ids:
response = ec2.delete_route(vpc_cidr, id)
self.logger.debug(response.get('Return'))
except Exception as e:
message = {'FILE': __file__.split('/')[-1], 'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3], 'EXCEPTION': str(e)}
self.logger.exception(message)
raise
def _session(self, region, account_id):
# instantiate EC2 sessions
return EC2(self.logger,
region,
credentials=self.assume_role(self.logger, account_id))
def calculate_vpc_parameters(self):
# This function calculates the CIDR ranges based on the number of subnets inputted by the user
# The subnet name parameters match the subnet parameters provided in the Scalable VPC QuickStart
# declare variables
# set the mask based on the number of subnet combinations
vpc_cidr = self.params.get('VPCCidr')
number_of_azs = int(self.params.get('AvailabilityZones'))
public_subnets = self.params.get('PublicSubnets')
private_subnets = self.params.get('PrivateSubnets')
region = self.params.get('Region')
account_list = self.params.get('AccountList')
# join the public and private subnet parameter names
vpc_subnet_parameter_names = public_subnets + private_subnets
all_subnet_parameter_names = [
'PrivateSubnet1ACIDR', 'PrivateSubnet2ACIDR',
'PrivateSubnet3ACIDR', 'PrivateSubnet4ACIDR', 'PublicSubnet1CIDR',
'PublicSubnet2CIDR', 'PublicSubnet3CIDR', 'PublicSubnet4CIDR',
'PrivateSubnet1BCIDR', 'PrivateSubnet2BCIDR',
'PrivateSubnet3BCIDR', 'PrivateSubnet4BCIDR'
]
# calculate the difference between all the subnet parameter names so that null cidr values are returned
# to the template. This makes it easier for the admin looking at the parameters of the stack to discern between
# used and unused subnets
excluded_names = list(
set(all_subnet_parameter_names) - set(vpc_subnet_parameter_names))
# calculate the subnet mask based on the number of subnets required
number_of_subnets = len(vpc_subnet_parameter_names)
vpc_mask = int(vpc_cidr[vpc_cidr.find('/') + 1:])
layer = 0
x = float(number_of_subnets)
while x > 1:
x = x / 2
layer += 1
mask = vpc_mask + layer
if mask > 28:
raise Exception(
'The number of subnets requested ({}) does not fit in the VPC CIDR provided ({}).'
'Try increasing your CIDR range or decreasing '
'the number of subnets required.'.format(
number_of_subnets, vpc_cidr))
# get the available availability_zones and extract the amount needed based on the user input
if account_list is None:
ec2 = EC2(self.logger, region)
self.logger.info(
"Getting list of AZs in region: {} from master account".format(
region))
else:
account_id = account_list[
0] # AVM can only create 1 account at a time
ec2 = self._session(region, account_id)
self.logger.info(
"Getting list of AZs in region: {} from account: {}".format(
region, account_id))
# get AZs
availability_zones = ec2.describe_availability_zones()
# check that there are enough AZs which are available at this time to satisfy the request
if len(availability_zones) < number_of_azs:
self.logger.info('Available availability zones: {}'.format(
str(availability_zones)))
self.logger.info('Availability zones requested: {}'.format(
str(number_of_azs)))
raise Exception(
'Not enough availability zones are available right now to fulfill this request.'
'Reduce the number of AZ\'s or try again later')
else:
usable_azs = availability_zones[:number_of_azs]
# calculate the subnet cidrs
vpc_ip = netaddr.IPNetwork(vpc_cidr)
vpc_subnets = list(
vpc_ip.subnet(mask, count=len(vpc_subnet_parameter_names)))
# create the parameter set
parameters = {}
parameters['AvailabilityZones'] = usable_azs
parameters['NumberOfAZs'] = str(number_of_azs)
parameters['VPCCIDR'] = vpc_cidr
parameters['CreatePrivateSubnets'] = self.params.get(
'CreatePrivateSubnets')
parameters['CreatePublicSubnets'] = self.params.get(
'CreatePublicSubnets')
parameters['CreateAdditionalPrivateSubnets'] = self.params.get(
'CreateAdditionalPrivateSubnets')
for index, subnet in enumerate(vpc_subnet_parameter_names):
parameters[subnet] = str(vpc_subnets[index])
# add the null subnet cidr values for the excluded parameter names
for name in excluded_names:
parameters[name] = 'None'
p = {'Parameters': parameters}
# Adding the parameters in primary dict
response = {**p, **parameters}
self.logger.info(response)
return response
def _create_service_catalog_state_machine_input_map(
self, portfolio, product):
input_params = {}
sc_portfolio = {}
sc_portfolio.update({'PortfolioName': sanitize(portfolio.name, True)})
sc_portfolio.update(
{'PortfolioDescription': sanitize(portfolio.description, True)})
sc_portfolio.update(
{'PortfolioProvider': sanitize(portfolio.owner, True)})
ssm_value = self.param_handler.update_params(
transform_params({'principal_role': portfolio.principal_role}))
sc_portfolio.update({'PrincipalArn': ssm_value.get('principal_role')})
sc_product = {}
sc_product.update({'ProductName': sanitize(product.name, True)})
sc_product.update({'ProductDescription': product.description})
sc_product.update({'ProductOwner': sanitize(portfolio.owner, True)})
if product.hide_old_versions is True:
sc_product.update({'HideOldVersions': 'Yes'})
else:
sc_product.update({'HideOldVersions': 'No'})
ssm_value = self.param_handler.update_params(
transform_params(
{'launch_constraint_role': product.launch_constraint_role}))
sc_product.update({'RoleArn': ssm_value.get('launch_constraint_role')})
ec2 = EC2(self.logger, environ.get('AWS_REGION'))
region_list = []
for region in ec2.describe_regions():
region_list.append(region.get('RegionName'))
if os.path.isfile(
os.path.join(self.manifest_folder, product.skeleton_file)):
lambda_arn_param = get_env_var('lambda_arn_param_name')
lambda_arn = self.ssm.get_parameter(lambda_arn_param)
portfolio_index = self.manifest.portfolios.index(portfolio)
product_index = self.manifest.portfolios[
portfolio_index].products.index(product)
product_name = self.manifest.portfolios[portfolio_index].products[
product_index].name
logger.info(
"Generating the product template for {} from {}".format(
product_name,
os.path.join(self.manifest_folder, product.skeleton_file)))
j2loader = jinja2.FileSystemLoader(self.manifest_folder)
j2env = jinja2.Environment(loader=j2loader)
j2template = j2env.get_template(product.skeleton_file)
template_url = None
if product.product_type.lower() == 'baseline':
# j2result = j2template.render(manifest=self.manifest, portfolio_index=portfolio_index,
# product_index=product_index, lambda_arn=lambda_arn, uuid=uuid.uuid4(),
# regions=region_list)
template_url = self._stage_template(product.skeleton_file +
".template")
elif product.product_type.lower() == 'optional':
if len(product.template_file) > 0:
template_url = self._stage_template(product.template_file)
j2result = j2template.render(
manifest=self.manifest,
portfolio_index=portfolio_index,
product_index=product_index,
lambda_arn=lambda_arn,
uuid=uuid.uuid4(),
template_url=template_url)
generated_avm_template = os.path.join(
self.manifest_folder,
product.skeleton_file + ".generated.template")
logger.info(
"Writing the generated product template to {}".format(
generated_avm_template))
with open(generated_avm_template, "w") as fh:
fh.write(j2result)
template_url = self._stage_template(generated_avm_template)
else:
raise Exception(
"Missing template_file location for portfolio:{} and product:{} in Manifest file"
.format(portfolio.name, product.name))
else:
raise Exception(
"Missing skeleton_file for portfolio:{} and product:{} in Manifest file"
.format(portfolio.name, product.name))
artifact_params = {}
artifact_params.update({'Info': {'LoadTemplateFromURL': template_url}})
artifact_params.update({'Type': 'CLOUD_FORMATION_TEMPLATE'})
artifact_params.update({'Description': product.description})
sc_product.update({'ProvisioningArtifactParameters': artifact_params})
try:
if product.rules_file:
rules = self._load_template_rules(product.rules_file)
sc_product.update({'Rules': rules})
except Exception as e:
logger.error(e)
input_params.update({'SCPortfolio': sc_portfolio})
input_params.update({'SCProduct': sc_product})
return self._create_state_machine_input_map(input_params)
def create_key_pair(self,
account,
region,
param_key_material=None,
param_key_fingerprint=None,
param_key_name=None):
if param_key_name:
self.logger.info(
"Looking up values in SSM parameter:{}".format(param_key_name))
existing_param = self.ssm.describe_parameters(param_key_name)
if existing_param:
return self.ssm.get_parameter(param_key_name)
sts = STS(self.logger)
key_name = sanitize(
"%s_%s_%s_%s" %
('lz', account, region, time.strftime("%Y-%m-%dT%H-%M-%S")))
try:
role_arn = "arn:aws:iam::" + str(
account) + ":role/AWSCloudFormationStackSetExecutionRole"
session_name = "create_key_pair_role"
# assume role
credentials = sts.assume_role(role_arn, session_name)
self.logger.info("Assuming IAM role: {}".format(role_arn))
# instantiate EC2 class
self.logger.debug(
"Creating EC2 Session in {} for account: {}".format(
region, account))
ec2 = EC2(self.logger, region, credentials=credentials)
if type(credentials) == dict:
# create EC2 key pair in member account
self.logger.info(
"Create key pair in the member account {} in region: {}".
format(account, region))
response = ec2.create_key_pair(key_name)
self.logger.debug(response)
# add key material and fingerprint in the SSM Parameter Store
self.logger.info(
"Adding Key Material and Fingerprint to SSM PS")
description = "Contains EC2 key pair asset created by Landing Zone Solution: " \
"EC2 Key Pair Custom Resource."
# Get Landing Zone KMS Key ID
key_id = self._get_kms_key_id()
if param_key_fingerprint:
self.ssm.put_parameter_use_cmk(
param_key_fingerprint, response.get('KeyFingerprint'),
key_id, description)
if param_key_material:
self.ssm.put_parameter_use_cmk(param_key_material,
response.get('KeyMaterial'),
key_id, description)
if param_key_name:
self.ssm.put_parameter(param_key_name, key_name,
description)
return key_name
else:
self.logger.error("Unable to obtain credentials.")
except Exception as e:
message = {
'FILE': __file__.split('/')[-1],
'CLASS': self.__class__.__name__,
'METHOD': inspect.stack()[0][3],
'EXCEPTION': str(e)
}
self.logger.exception(message)
raise
