def share_to_another_region(ami_id,
regn,
ami_name,
account_ids,
timeout_sec=900):
if not regn == region():
resp = ec2(region=regn).copy_image(SourceRegion=region(),
SourceImageId=ami_id,
Name=ami_name)
ami_id = resp['ImageId']
status = "initial"
start = time.time()
while status != 'available':
time.sleep(2)
if time.time() - start > timeout_sec:
raise Exception("Failed waiting for status 'available' for " +
ami_id + " (timeout: " + str(timeout_sec) + ")")
try:
images_resp = ec2(region=regn).describe_images(ImageIds=[ami_id])
status = images_resp['Images'][0]['State']
except ClientError:
print("Did not find image " + ami_id)
perms = {"Add": []}
my_acco = resolve_account()
for acco in account_ids:
if not acco == my_acco:
perms['Add'].append({"UserId": acco})
if len(perms['Add']) > 0:
ec2(region=regn).modify_image_attribute(ImageId=ami_id,
LaunchPermission=perms)
def _get_params(data, template):
params = OrderedDict()
# first load defaults for all parameters in "Parameters"
if 'Parameters' in data:
_add_params(params, data, 'Parameters', True)
if 'Fn::Merge' in data['Parameters'] and 'Result' in data['Parameters']['Fn::Merge']:
_add_params(params, data['Parameters']['Fn::Merge'], 'Result', True)
if 'Fn::ImportYaml' in data['Parameters'] and 'Result' in data['Parameters']['Fn::ImportYaml']:
_add_params(params, data['Parameters']['Fn::ImportYaml'], 'Result', True)
if "resources" in data and 'Parameters' in data['resources']:
params['ServerlessDeploymentBucket'] = PARAM_NOT_AVAILABLE
_add_params(params, data['resources'], 'Parameters', True)
if 'Fn::Merge' in data['resources']['Parameters'] and 'Result' in data['resources']['Parameters']['Fn::Merge']:
_add_params(params, data['resources']['Parameters']['Fn::Merge'], 'Result', True)
if 'Fn::ImportYaml' in data['resources']['Parameters'] and 'Result' in data['resources']['Parameters']['Fn::ImportYaml']:
_add_params(params, data['resources']['Parameters']['Fn::ImportYaml'], 'Result', True)
params['STACK_NAME'] = PARAM_NOT_AVAILABLE
if 'REGION' not in os.environ:
os.environ['REGION'] = region()
params['REGION'] = os.environ['REGION']
if 'ACCOUNT_ID' not in os.environ:
if resolve_account():
os.environ['ACCOUNT_ID'] = resolve_account()
else:
os.environ['ACCOUNT_ID'] = "None"
params['ACCOUNT_ID'] = os.environ['ACCOUNT_ID']
global SOURCED_PARAMS
if not SOURCED_PARAMS:
SOURCED_PARAMS = {}
# then override them with values from infra
template_dir = os.path.dirname(os.path.abspath(template))
image_dir = os.path.dirname(template_dir)
image_name = os.path.basename(image_dir)
stack_name = os.path.basename(template_dir)
stack_name = re.sub('^stack-', '', stack_name)
SOURCED_PARAMS = load_parameters(component=image_name, stack=stack_name)
SOURCED_PARAMS.update(os.environ)
params.update(SOURCED_PARAMS)
# source_infra_properties.sh always resolves a region, account id and stack
# name
params["AWS::Region"] = params['REGION']
params["AWS::AccountId"] = params['ACCOUNT_ID']
params["AWS::StackName"] = params['STACK_NAME']
# finally load AWS-provided and "Resources"
params["AWS::NotificationARNs"] = PARAM_NOT_AVAILABLE
params["AWS::NoValue"] = PARAM_NOT_AVAILABLE
params["AWS::StackId"] = PARAM_NOT_AVAILABLE
_add_params(params, data, 'Resources', False)
if "resources" in data:
_add_params(params, data['resources'], 'Resources', False)
return params
def show_stack_params_and_outputs():
""" Show stack parameters and outputs as a single json documents
"""
parser = get_parser()
parser.add_argument("-r", "--region", help="Region for the stack to show",
default=region()).completer = ChoicesCompleter(regions())
parser.add_argument("-p", "--parameter", help="Name of paremeter if only" +
" one parameter required")
parser.add_argument("stack_name", help="The stack name to show").completer = \
ChoicesCompleter(best_effort_stacks())
argcomplete.autocomplete(parser)
args = parser.parse_args()
resp, _ = stack_params_and_outputs_and_stack(stack_name=args.stack_name, stack_region=args.region)
if args.parameter:
if args.parameter in resp:
print(resp[args.parameter])
else:
parser.error("Parameter " + args.parameter + " not found")
else:
print(json.dumps(resp, indent=2))
def create_account(email,
account_name,
role_name="OrganizationAccountAccessRole",
trust_role="TrustedAccountAccessRole",
access_to_billing=True,
trusted_accounts=None,
mfa_token=None):
if access_to_billing:
access = "ALLOW"
else:
access = "DENY"
timeout = 300
if trusted_accounts:
trusted_roles = {}
for trusted_account in trusted_accounts:
role_arn = find_role_arn(trusted_account)
if not role_arn:
raise Exception("Failed to resolve trusted account " +
trusted_account)
trusted_roles[trusted_account] = role_arn
response = organizations().create_account(Email=email,
AccountName=account_name,
RoleName=role_name,
IamUserAccessToBilling=access)
if 'CreateAccountStatus' in response and 'Id' in response[
'CreateAccountStatus']:
create_account_id = response['CreateAccountStatus']['Id']
startTime = time()
status = response['CreateAccountStatus']['State']
while time() - startTime < timeout and not status == "SUCCEEDED":
if response['CreateAccountStatus']['State'] == "FAILED":
raise Exception(
"Account creation failed: " +
response['CreateAccountStatus']['FailureReason'])
print("Waiting for account creation to finish")
sleep(2)
response = organizations().describe_create_account_status(
CreateAccountRequestId=create_account_id)
status = response['CreateAccountStatus']['State']
if time() - startTime > timeout and not status == "SUCCEEDED":
raise Exception("Timed out waiting to create account " +
response['CreateAccountStatus']['State'])
account_id = response['CreateAccountStatus']['AccountId']
os.environ['paramManagedAccount'] = account_id
os.environ['paramManageRole'] = role_name
template = find_include("manage-account.yaml")
cf_deploy.deploy("managed-account-" + account_name, template, region())
if trusted_accounts:
role_arn = "arn:aws:iam::" + account_id + ":role/" + role_name
assumed_creds = utils.assume_role(role_arn, mfa_token_name=mfa_token)
sess = session(aws_access_key_id=assumed_creds['AccessKeyId'],
aws_secret_access_key=assumed_creds['SecretAccessKey'],
aws_session_token=assumed_creds['SessionToken'])
for trusted_account in trusted_accounts:
os.environ['paramTrustedAccount'] = trusted_roles[
trusted_account].split(":")[4]
os.environ['paramRoleName'] = trust_role
template = find_include("trust-account-role.yaml")
cf_deploy.deploy("trust-" + trusted_account,
template,
utils.region(),
session=sess)
template = find_include("manage-account.yaml")
for trusted_account in trusted_accounts:
role_arn = trusted_roles[trusted_account]
print("Assuming role " + role_arn)
assumed_creds = utils.assume_role(role_arn,
mfa_token_name=mfa_token)
sess = session(
aws_access_key_id=assumed_creds['AccessKeyId'],
aws_secret_access_key=assumed_creds['SecretAccessKey'],
aws_session_token=assumed_creds['SessionToken'])
os.environ['paramManagedAccount'] = account_id
os.environ['paramRoleName'] = trust_role
cf_deploy.deploy("managed-account-" + account_name + "-" +
account_id,
template,
region(),
session=sess)
def load_parameters(component=None, stack=None, serverless=None, docker=None, image=None,
cdk=None, terraform=None, branch=None, resolve_images=False,
git=None):
subc_name = ""
if stack:
subc_name = "stack=" + stack
if serverless:
subc_name = "serverless=" + serverless
if docker:
subc_name = "docker=" + docker
if isinstance(image, six.string_types):
subc_name = "image=" + image
if cdk:
subc_name = "cdk=" + cdk
if terraform:
subc_name = "terraform=" + terraform
if not git:
git = Git()
with git:
current_branch = git.get_current_branch()
if not branch:
branch = current_branch
branch = branch.strip().split("origin/")[-1:][0]
params_key = (component, subc_name, branch)
if params_key in parameters:
return parameters[params_key]
ret = {
"GIT_BRANCH": branch
}
account = resolve_account()
if account:
ret["ACCOUNT_ID"] = account
if component:
ret["COMPONENT"] = component
prefix = ""
if current_branch != branch:
prefix = git.export_branch(branch) + os.sep
files = [prefix + "infra.properties", prefix + "infra-" + branch + ".properties"]
if component:
files.append(prefix + component + os.sep + "infra.properties")
files.append(prefix + component + os.sep + "infra-" + branch + ".properties")
_add_subcomponent_file(prefix + component, branch, "stack", stack, files)
_add_subcomponent_file(prefix + component, branch, "serverless", serverless, files)
_add_subcomponent_file(prefix + component, branch, "cdk", cdk, files)
_add_subcomponent_file(prefix + component, branch, "terraform", terraform, files)
_add_subcomponent_file(prefix + component, branch, "docker", docker, files)
_add_subcomponent_file(prefix + component, branch, "image", image, files)
if isinstance(image, six.string_types):
files.append(prefix + component + os.sep + "image" + os.sep + "infra.properties")
files.append(prefix + component + os.sep + "image" + os.sep + "infra-" + branch + ".properties")
initial_resolve = ret.copy()
os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"] = "true"
for file in files:
if os.path.exists(file):
import_parameter_file(file, initial_resolve)
del os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"]
if "REGION" not in initial_resolve:
ret["REGION"] = region()
else:
ret["REGION"] = initial_resolve["REGION"]
if not "AWS_DEFAULT_REGION" in os.environ:
os.environ["AWS_DEFAULT_REGION"] = ret["REGION"]
if "paramEnvId" not in initial_resolve:
ret["paramEnvId"] = branch
else:
ret["paramEnvId"] = initial_resolve["paramEnvId"]
for file in files:
if os.path.exists(file):
import_parameter_file(file, ret)
if (serverless or stack or cdk or terraform) and resolve_images:
image_branch = branch
if 'BAKE_IMAGE_BRANCH' in ret:
image_branch = ret['BAKE_IMAGE_BRANCH']
for docker in [dockerdir.split("/docker-")[1] for dockerdir in glob(component + os.sep + "docker-*")]:
try:
ret['paramDockerUri' + docker] = resolve_docker_uri(component, 'paramDockerUri' + docker, image_branch, git)
except ClientError:
# Best effor to load docker uris, but ignore errors since the repo might not
# actually be in use. Missing and used uris will result in an error later.
pass
for image_name in [imagedir.split("/image")[1].replace("-", "").lower() for imagedir in glob(component + os.sep + "image*")]:
try:
image = resolve_ami(ret, component, image_name, image_branch, branch, git)
if image:
ret['paramAmi' + image_name] = image['ImageId']
ret['paramAmiName' + image_name] = image['Name']
env_param_name = "AMI_ID"
if image_name:
env_param_name += "_" + image_name.upper()
ret[env_param_name] = image['ImageId']
except ClientError:
# Best effor to load ami info, but ignore errors since the image might not
# actually be in use. Missing and used images will result in an error later.
pass
if "REGION" not in ret:
ret["REGION"] = region()
if "paramEnvId" not in ret:
ret["paramEnvId"] = branch
if "ORIG_STACK_NAME" in os.environ:
ret["ORIG_STACK_NAME"] = os.environ["ORIG_STACK_NAME"]
if "STACK_NAME" not in ret:
ret["STACK_NAME"] = component + "-" + ret["ORIG_STACK_NAME"] + "-" + ret["paramEnvId"]
for k, v in list(os.environ.items()):
if k.startswith("ORIG_") and k.endswith("_NAME"):
ret[k] = v
if "ORIG_DOCKER_NAME" in os.environ:
if "DOCKER_NAME" not in ret:
ret["DOCKER_NAME"] = component + "/" + ret["paramEnvId"] + "-" + ret["ORIG_DOCKER_NAME"]
if "JENKINS_JOB_PREFIX" not in ret:
ret["JENKINS_JOB_PREFIX"] = "ndt" + ret["paramEnvId"]
parameters[params_key] = ret
return ret
def set_region():
"""Sets the environment variable AWS_DEFAULT_REGION if not already set
to a sensible default
"""
if "AWS_DEFAULT_REGION" not in os.environ:
os.environ["AWS_DEFAULT_REGION"] = region()