def conditionally_create_profile(role_name, service_type):
"""
Check that there is a 1:1 correspondence with an InstanceProfile having the same name
as the role, and that the role is contained in it. Create InstanceProfile and attach to role if needed.
"""
# make instance profile if this service_type gets an instance profile
if service_type not in INSTANCE_PROFILE_SERVICE_TYPES:
print_if_verbose("service type: {} not eligible for instance profile".format(service_type))
return
instance_profile = get_instance_profile(role_name)
if not instance_profile:
print("Create instance profile: {}".format(role_name))
if CONTEXT.commit:
try:
instance_profile = CLIENTS["iam"].create_instance_profile(InstanceProfileName=role_name)
except ClientError as error:
fail("Exception creating instance profile named: {} {}".format(role_name, sys.exc_info(), error))
else:
print_if_verbose("instance profile already exists: {}".format(role_name))
# attach instance profile to role; test 'if instance_profile' because we drop through to here in a dry run
if instance_profile and not instance_profile_contains_role(instance_profile, role_name):
print("Add role: {} to instance profile: {}".format(role_name, role_name))
if CONTEXT.commit:
try:
CLIENTS["iam"].add_role_to_instance_profile(InstanceProfileName=role_name, RoleName=role_name)
except ClientError as error:
fail("Exception adding role to instance profile: {} {}".format(role_name, sys.exc_info(), error))
else:
print_if_verbose("instance profile already contains role: {}".format(role_name))
def handle_args_and_set_context(args):
"""
Args:
args: the command line args, probably passed from main() as sys.argv[1:]
Returns:
a populated EFContext object
Raises:
IOError: if service registry file can't be found or can't be opened
RuntimeError: if repo or branch isn't as spec'd in ef_config.EF_REPO and ef_config.EF_REPO_BRANCH
CalledProcessError: if 'git rev-parse' command to find repo root could not be run
"""
parser = argparse.ArgumentParser()
parser.add_argument("env", help=", ".join(EFConfig.ENV_LIST))
parser.add_argument("--sr", help="optional /path/to/service_registry_file.json", default=None)
parser.add_argument("--commit", help="Make changes in AWS (dry run if omitted)", action="store_true", default=False)
parser.add_argument("--verbose", help="Print additional info", action="store_true", default=False)
parser.add_argument("--devel", help="Allow running from branch; don't refresh from origin", action="store_true",
default=False)
parsed_args = vars(parser.parse_args(args))
context = EFContext()
context.commit = parsed_args["commit"]
context.devel = parsed_args["devel"]
try:
context.env = parsed_args["env"]
except ValueError as e:
fail("Error in env: {}".format(e.message))
# Set up service registry and policy template path which depends on it
context.service_registry = EFServiceRegistry(parsed_args["sr"])
context.policy_template_path = normpath(dirname(context.service_registry.filespec)) + EFConfig.POLICY_TEMPLATE_PATH_SUFFIX
context.verbose = parsed_args["verbose"]
return context
def resolve_policy_document(policy_name):
policy_filename = "{}{}.json".format(CONTEXT.policy_template_path, policy_name)
print_if_verbose("Load policy: {} from file: {}".format(policy_name, policy_filename))
# retrieve policy template
try:
policy_file = file(policy_filename, 'r')
policy_template = policy_file.read()
policy_file.close()
except:
fail("error opening policy file: {}".format(policy_filename))
print_if_verbose("pre-resolution policy template:\n{}".format(policy_template))
# If running in EC2, do not set profile and set target_other=True
if CONTEXT.whereami == "ec2":
resolver = EFTemplateResolver(target_other=True, env=CONTEXT.env, region=EFConfig.DEFAULT_REGION,
service=CONTEXT.service, verbose=CONTEXT.verbose)
else:
resolver = EFTemplateResolver(profile=CONTEXT.account_alias, env=CONTEXT.env, region=EFConfig.DEFAULT_REGION,
service=CONTEXT.service, verbose=CONTEXT.verbose)
resolver.load(policy_template)
policy_document = resolver.render()
print_if_verbose("resolved policy document:\n{}".format(policy_document))
if not resolver.resolved_ok():
fail("policy template {} has unresolved symbols or extra {{ or }}: {}".format(
policy_filename, resolver.unresolved_symbols()))
return policy_document
def resolve_template(template, profile, env, region, service, verbose):
# resolve {{SYMBOLS}} in the passed template file
isfile(template) or fail("Not a file: {}".format(template))
resolver = EFTemplateResolver(profile=profile,
target_other=True,
env=env,
region=region,
service=service,
verbose=verbose)
with open(template) as template_file:
resolver.load(template_file)
resolver.render()
if verbose:
print(resolver.template)
dangling_left, dangling_right = resolver.count_braces()
if resolver.unresolved_symbols():
fail("Unable to resolve symbols: " +
",".join(["{{" + s + "}}"
for s in resolver.unresolved_symbols()]))
elif dangling_left > 0 or dangling_right > 0:
fail("Some {{ or }} were not resolved. left{{: {}, right}}: {}".format(
dangling_left, dangling_right))
else:
return resolver.template
def conditionally_inline_policies(role_name, sr_entry):
"""
If 'policies' key lists the filename prefixes of policies to bind to the role,
load them from the expected path and inline them onto the role
Args:
role_name: name of the role to attach the policies to
sr_entry: service registry entry
"""
service_type = sr_entry['type']
if not (service_type in SERVICE_TYPE_ROLE and "policies" in sr_entry):
print_if_verbose("not eligible for policies; service_type: {} is not valid for policies "
"or no 'policies' key in service registry for this role".format(service_type))
return
for policy_name in sr_entry['policies']:
print_if_verbose("loading policy: {} for role: {}".format(policy_name, role_name))
try:
policy_document = resolve_policy_document(policy_name)
except:
fail("Exception loading policy: {} for role: {}".format(policy_name, role_name), sys.exc_info())
# inline the policy onto the role
if CONTEXT.commit:
try:
CLIENTS["iam"].put_role_policy(RoleName=role_name, PolicyName=policy_name, PolicyDocument=policy_document)
except:
fail("Exception putting policy: {} onto role: {}".format(policy_name, role_name), sys.exc_info())
def conditionally_attach_customer_managed_policies(role_name, sr_entry):
"""
If 'customer_managed_policies' key lists the names of customer managed policies to bind to the role,
attach them to the role.
Note that this function will throw a warning without failing, if the managed policy does not exist in
the given account.
Args:
role_name: name of the role to attach the policies to
sr_entry: service registry entry
"""
service_type = sr_entry['type']
if not (service_type in SERVICE_TYPE_ROLE and "customer_managed_policies" in sr_entry):
print_if_verbose("not eligible for policies; service_type: {} is not valid for policies "
"or no 'customer_managed_policies' key in service registry for this role".format(service_type))
return
for policy_name in sr_entry['customer_managed_policies']:
print_if_verbose("loading policy: {} for role: {}".format(policy_name, role_name))
if CONTEXT.commit:
policy_arn = 'arn:aws:iam::{}:policy/{}'.format(CONTEXT.account_id, policy_name)
try:
CLIENTS["iam"].attach_role_policy(RoleName=role_name, PolicyArn=policy_arn)
except CLIENTS["iam"].exceptions.NoSuchEntityException as exc:
print("WARNING: {}".format(exc))
except:
fail("Exception putting policy: {} onto role: {}".format(policy_name, role_name), sys.exc_info())
def main():
context = handle_args_and_set_context(sys.argv[1:])
profile = None if context.whereami == "ec2" else context.account_alias
try:
clients = ef_utils.create_aws_clients(EFConfig.DEFAULT_REGION, profile,
"kms")
except RuntimeError as error:
ef_utils.fail(
"Exception creating clients in region {} with profile {}".format(
EFConfig.DEFAULT_REGION, profile), error)
if context.secret_file:
generate_secret_file(context.secret_file, context.match,
context.service, context.env, clients)
return
if context.decrypt:
decrypted = ef_utils.kms_decrypt(kms_client=clients['kms'],
secret=context.decrypt)
key_aliases = ef_utils.kms_key_alias(clients['kms'], decrypted.key_id)
print("Decrypted Secret: {}; Key: {}".format(decrypted.plaintext,
', '.join(key_aliases)))
return
if context.plaintext:
password = context.plaintext
else:
password = generate_secret(context.length)
print("Generated Secret: {}".format(password))
encrypted_password = ef_utils.kms_encrypt(clients['kms'], context.service,
context.env, password)
print(format_secret(encrypted_password))
return
def get_metadata_or_fail(metadata_key):
"""
Call get_metadata; halt with fail() if it raises an exception
"""
try:
return http_get_metadata(metadata_key)
except IOError as error:
fail("Exception in http_get_metadata {} {}".format(metadata_key, repr(error)))
def handle_args_and_set_context(args):
"""
Args:
args: the command line args, probably passed from main() as sys.argv[1:]
Returns:
a populated EFCFContext object (extends EFContext)
Raises:
IOError: if service registry file can't be found or can't be opened
RuntimeError: if repo or branch isn't as spec'd in ef_config.EF_REPO and ef_config.EF_REPO_BRANCH
CalledProcessError: if 'git rev-parse' command to find repo root could not be run
"""
parser = argparse.ArgumentParser()
parser.add_argument("template_file", help="/path/to/template_file.json")
parser.add_argument("env", help=", ".join(EFConfig.ENV_LIST))
parser.add_argument(
"--changeset",
help="create a changeset; cannot be combined with --commit",
action="store_true",
default=False)
parser.add_argument(
"--commit",
help=
"Make changes in AWS (dry run if omitted); cannot be combined with --changeset",
action="store_true",
default=False)
parser.add_argument(
"--poll",
help="Poll Cloudformation to check status of stack creation/updates",
action="store_true",
default=False)
parser.add_argument("--sr",
help="optional /path/to/service_registry_file.json",
default=None)
parser.add_argument("--verbose",
help="Print additional info + resolved template",
action="store_true",
default=False)
parser.add_argument(
"--devel",
help="Allow running from branch; don't refresh from origin",
action="store_true",
default=False)
parsed_args = vars(parser.parse_args(args))
context = EFCFContext()
try:
context.env = parsed_args["env"]
context.template_file = parsed_args["template_file"]
except ValueError as e:
fail("Error in argument: {}".format(e.message))
context.changeset = parsed_args["changeset"]
context.commit = parsed_args["commit"]
context.devel = parsed_args["devel"]
context.poll_status = parsed_args["poll"]
context.verbose = parsed_args["verbose"]
# Set up service registry and policy template path which depends on it
context.service_registry = EFServiceRegistry(parsed_args["sr"])
return context
def get_version_by_value(context, value):
"""
Get the latest version that matches the provided ami-id
Args:
context: a populated EFVersionContext object
value: the value of the version to look for
"""
versions = get_versions(context)
for version in versions:
if version.value == value:
return version
fail("Didn't find a matching version for: "
"{}:{} in env/service: {}/{}".format(context.key, value, context.env,
context.service_name))
def precheck_dist_hash(context):
"""
Is the dist in service the same as the dist marked current in the version records?
This tool won't update records unless the world state is coherent.
Args:
context: a populated EFVersionContext object
Returns:
True if ok to proceed
Raises:
RuntimeError if not ok to proceed
"""
# get the current dist-hash
key = "{}/{}/dist-hash".format(context.service_name, context.env)
print_if_verbose("precheck_dist_hash with key: {}".format(key))
try:
current_dist_hash = Version(
context.aws_client("s3").get_object(
Bucket=EFConfig.S3_VERSION_BUCKET, Key=key))
print_if_verbose("dist-hash found: {}".format(current_dist_hash.value))
except ClientError as error:
if error.response["Error"]["Code"] == "NoSuchKey":
# If bootstrapping (this will be the first entry in the version history)
# then we can't check it vs. current version, thus we cannot get the key
print_if_verbose(
"precheck passed without check because current dist-hash is None"
)
return True
else:
fail("Exception while prechecking dist_hash for {} {}: {}".format(
context.service_name, context.env, error))
# Otherwise perform a consistency check
# 1. get dist version in service for environment
try:
response = urllib2.urlopen(current_dist_hash.location, None, 5)
if response.getcode() != 200:
raise IOError("Non-200 response " + str(response.getcode()) +
" reading " + current_dist_hash.location)
dist_hash_in_service = response.read().strip()
except urllib2.URLError as error:
raise IOError("URLError in http_get_dist_version: " + repr(error))
# 2. dist version in service should be the same as "current" dist version
if dist_hash_in_service != current_dist_hash.value:
raise RuntimeError(
"{} dist-hash in service: {} but expected dist-hash: {}".format(
key, dist_hash_in_service, current_dist_hash.value))
# Check passed - all is well
return True
def handle_args_and_set_context(args):
"""
Args:
args: the command line args, probably passed from main() as sys.argv[1:]
Returns:
a populated EFPWContext object
Raises:
RuntimeError: if branch isn't as spec'd in ef_config.EF_REPO_BRANCH
ValueError: if a parameter is invalid
"""
parser = argparse.ArgumentParser()
parser.add_argument("service",
help="name of service password is being generated for")
parser.add_argument("env", help=", ".join(EFConfig.ENV_LIST))
group = parser.add_mutually_exclusive_group()
group.add_argument("--decrypt",
help="encrypted string to be decrypted",
default="")
group.add_argument(
"--plaintext",
help="secret to be encrypted rather than a randomly generated one",
default="")
group.add_argument("--secret_file",
help="json file containing secrets to be encrypted",
default="")
parser.add_argument(
"--match",
help=
"used in conjunction with --secret_file to match against keys to be encrypted",
default="")
parser.add_argument("--length",
help="length of generated password (default 32)",
default=32)
parsed_args = vars(parser.parse_args(args))
context = EFPWContext()
try:
context.env = parsed_args["env"]
except ValueError as e:
ef_utils.fail("Error in env: {}".format(e))
context.service = parsed_args["service"]
context.decrypt = parsed_args["decrypt"]
context.length = parsed_args["length"]
context.plaintext = parsed_args["plaintext"]
context.secret_file = parsed_args["secret_file"]
context.match = parsed_args["match"]
if context.match or context.secret_file:
if not context.match or not context.secret_file:
raise ValueError("Must have both --match and --secret_file flag")
return context
def cmd_rollback(context):
"""
Roll back by finding the most recent "stable" tagged version, and putting it again, so that
it's the new "current" version.
Args:
context: a populated EFVersionContext object
"""
last_stable = get_versions(context, return_stable=True)
if len(last_stable) != 1:
fail(
"Didn't find a version marked stable for key: {} in env/service: {}/{}"
.format(context.key, context.env, context.service_name))
context.value = last_stable[0].value
context.stable = True
cmd_set(context)
def main():
# Fetch args and load context
context = handle_args_and_set_context(sys.argv[1:])
# Refresh from repo if necessary and possible (gets don't need service registry, sets do)
if (context.rollback or context.value) and not (context.devel or getenv(
"JENKINS_URL", False)):
print("Refreshing repo")
try:
pull_repo()
except RuntimeError as error:
fail("Error checking or pulling repo", error)
# Sign on to AWS and create clients
if context.whereami in ["ec2"]:
# Always use instance credentials in EC2. One day we'll have "lambda" in there too, so use "in" w/ list
aws_session_alias = None
else:
# Otherwise use local user credential matching the account alias
aws_session_alias = context.account_alias
# Make AWS clients
try:
context.set_aws_clients(
create_aws_clients(EFConfig.DEFAULT_REGION, aws_session_alias,
"ec2", "s3", "sts"))
except RuntimeError:
fail(
"Exception creating AWS client in region {} with aws account alias {} (None=instance credentials)"
.format(EFConfig.DEFAULT_REGION, aws_session_alias))
# Instantiate a versionresolver - we'll use some of its methods
context._versionresolver = EFVersionResolver(context.aws_client())
# Carry out the requested action
if context.get:
cmd_get(context)
elif context.history:
cmd_history(context)
elif context.rollback:
cmd_rollback(context)
elif context.rollback_to:
cmd_rollback_to(context)
elif context.show:
cmd_show(context)
elif context.value:
cmd_set(context)
def test_fail_with_message(self, mock_stderr):
"""
Tests fail() with a regular string message and checks if the message in stderr and exit code matches
Args:
mock_stderr: StringIO, captures the string sent to sys.stderr
Returns:
None
Raises:
AssertionError if any of the assert checks fail
"""
with self.assertRaises(SystemExit) as exception:
ef_utils.fail("Error Message")
error_message = mock_stderr.getvalue().strip()
self.assertEquals(error_message, "Error Message")
self.assertEquals(exception.exception.code, 1)
def test_fail_with_empty_string(self, mock_stderr):
"""
Test fail() with a an empty string
Args:
mock_stderr: StringIO, captures the string sent to sys.stderr
Returns:
None
Raises:
AssertionError if any of the assert checks fail
"""
with self.assertRaises(SystemExit) as exception:
ef_utils.fail("")
error_message = mock_stderr.getvalue().strip()
self.assertEquals(error_message, "")
self.assertEquals(exception.exception.code, 1)
def generate_secret_file(file_path, pattern, service, environment, clients):
"""
Generate a parameter files with it's secrets encrypted in KMS
Args:
file_path (string): Path to the parameter file to be encrypted
pattern (string): Pattern to do fuzzy string matching
service (string): Service to use KMS key to encrypt file
environment (string): Environment to encrypt values
clients (dict): KMS AWS client that has been instantiated
Returns:
None
Raises:
IOError: If the file does not exist
"""
changed = False
with open(file_path) as json_file:
data = json.load(json_file, object_pairs_hook=OrderedDict)
try:
for key, value in data["params"][environment].items():
if pattern in key:
if "aws:kms:decrypt" in value:
print(
"Found match, key {} but value is encrypted already; skipping..."
.format(key))
else:
print("Found match, encrypting key {}".format(key))
encrypted_password = ef_utils.kms_encrypt(
clients['kms'], service, environment, value)
data["params"][environment][key] = format_secret(
encrypted_password)
changed = True
except KeyError:
ef_utils.fail(
"Error env: {} does not exist in parameters file".format(
environment))
if changed:
with open(file_path, "w") as encrypted_file:
json.dump(data, encrypted_file, indent=2, separators=(',', ': '))
# Writing new line here so it conforms to WG14 N1256 5.1.1.1 (so github doesn't complain)
encrypted_file.write("\n")
def test_fail_with_message_and_exception_data(self, mock_stderr,
mock_stdout):
"""
Test fail() with a regular string message and a python object as the exception data
Args:
mock_stderr: StringIO, captures the string sent to sys.stderr
mock_stdout: StringIO, captures the string sent to sys.stdout
Returns:
None
Raises:
AssertionError if any of the assert checks fail
"""
with self.assertRaises(SystemExit) as exception:
ef_utils.fail("Error Message", {"ErrorCode": 22})
error_message = mock_stderr.getvalue().strip()
self.assertEquals(error_message, "Error Message")
self.assertEquals(exception.exception.code, 1)
output_message = mock_stdout.getvalue().strip()
self.assertEquals(output_message, "{'ErrorCode': 22}")
def cmd_rollback(context):
"""
Roll back by finding the most recent "stable" tagged version, and putting it again, so that
it's the new "current" version.
Args:
context: a populated EFVersionContext object
"""
stable_versions = _get_stable_versions(context)
latest_version = _get_latest_version(context)
for version in stable_versions:
if latest_version and (version.value != latest_version.value):
context.value = version.value
context.commit_hash = version.commit_hash
context.build_number = version.build_number
context.location = version.location
context.stable = True
cmd_set(context)
return
fail(
"Didn't find a version marked stable for key: {} in env/service: {}/{}"
.format(context.key, context.env, context.service_name))
def validate_context(context):
"""
Set the key for the current context.
Args:
context: a populated EFVersionContext object
"""
# Service must exist in service registry
if not context.service_registry.service_record(context.service_name):
fail("service: {} not found in service registry: {}".format(
context.service_name, context.service_registry.filespec))
service_type = context.service_registry.service_record(
context.service_name)["type"]
# Key must be valid
if not EFConfig.VERSION_KEYS.has_key(context.key):
fail(
"invalid key: {}; see VERSION_KEYS in ef_config for supported keys"
.format(context.key))
# Lookup allowed key for service type
if EFConfig.VERSION_KEYS[context.key].has_key(
"allowed_types") and service_type not in EFConfig.VERSION_KEYS[
context.key]["allowed_types"]:
fail(
"service_type: {} is not allowed for key {}; see VERSION_KEYS[KEY]['allowed_types'] in ef_config and validate service registry entry"
.format(service_type, context.key))
return True
def validate_context(context):
"""
Validate the context. Fails the process on an invalid context
Args:
context: a populated EFVersionContext object
"""
# Key must be valid
key_data = EFConfig.VERSION_KEYS.get(context.key)
if not key_data:
fail(
"invalid key: {}; see VERSION_KEYS in ef_config for supported keys"
.format(context.key))
registry = context.service_registry
service = registry.service_record(context.service_name)
# Service must exist in service registry
if not service:
fail("service: {} not found in service registry: {}".format(
context.service_name, registry.filespec))
# Lookup allowed key for service type
service_type = service["type"]
allowed_types = key_data.get("allowed_types", [])
if service_type not in allowed_types:
fail(
"service_type: {} is not allowed for key {}; see VERSION_KEYS[KEY]['allowed_types']"
"in ef_config and validate service registry entry".format(
service_type, context.key))
return True
def conditionally_attach_aws_managed_policies(role_name, sr_entry):
"""
If 'aws_managed_policies' key lists the names of AWS managed policies to bind to the role,
attach them to the role
Args:
role_name: name of the role to attach the policies to
sr_entry: service registry entry
"""
service_type = sr_entry['type']
if not (service_type in SERVICE_TYPE_ROLE and "aws_managed_policies" in sr_entry):
print_if_verbose("not eligible for policies; service_type: {} is not valid for policies "
"or no 'aws_managed_policies' key in service registry for this role".format(service_type))
return
for policy_name in sr_entry['aws_managed_policies']:
print_if_verbose("loading policy: {} for role: {}".format(policy_name, role_name))
if CONTEXT.commit:
try:
CLIENTS["iam"].attach_role_policy(RoleName=role_name, PolicyArn='arn:aws:iam::aws:policy/' + policy_name)
except:
fail("Exception putting policy: {} onto role: {}".format(policy_name, role_name), sys.exc_info())
def conditionally_create_security_groups(env, service_name, service_type):
"""
Create security groups as needed; name and number created depend on service_type
Args:
env: the environment the SG will be created in
service_name: name of the service in service registry
service_type: service registry service type: 'aws_ec2', 'aws_lambda', 'aws_security_group', or 'http_service'
"""
if service_type not in SG_SERVICE_TYPES:
print_if_verbose(
"not eligible for security group(s); service type: {}".format(
service_type))
return
target_name = "{}-{}".format(env, service_name)
if service_type == "aws_ec2":
sg_names = ["{}-ec2".format(target_name)]
elif service_type == "aws_lambda":
sg_names = ["{}-lambda".format(target_name)]
elif service_type == "http_service":
sg_names = ["{}-ec2".format(target_name), "{}-elb".format(target_name)]
elif service_type == "aws_security_group":
sg_names = [target_name]
else:
fail(
"Unexpected service_type: {} when creating security group for: {}".
format(service_type, target_name))
for sg_name in sg_names:
if not AWS_RESOLVER.ec2_security_group_security_group_id(sg_name):
vpc_name = "vpc-{}".format(env)
print("Create security group: {} in vpc: {}".format(
sg_name, vpc_name))
vpc = AWS_RESOLVER.ec2_vpc_vpc_id(vpc_name)
if not vpc:
fail("Error: could not get VPC by name: {}".format(vpc_name))
# create security group
if CONTEXT.commit:
try:
new_sg = CLIENTS["ec2"].create_security_group(
GroupName=sg_name, VpcId=vpc, Description=sg_name)
except:
fail(
"Exception creating security group named: {} in VpcId: {}"
.format(sg_name, vpc_name), sys.exc_info())
print(new_sg["GroupId"])
else:
print_if_verbose(
"security group already exists: {}".format(sg_name))
class TestEFVersionResolver(unittest.TestCase):
"""Tests for 'ef_version_resolver.py'"""
# initialize based on where running
where = whereami()
if where == "local":
session = boto3.Session(profile_name=get_account_alias("proto0"), region_name=EFConfig.DEFAULT_REGION)
elif where == "ec2":
region = http_get_metadata("placement/availability-zone/")
region = region[:-1]
session = boto3.Session(region_name=region)
else:
fail("Can't test in environment: " + where)
clients = {
"ec2": session.client("ec2")
}
def test_ami_id(self):
"""Does ami-id,data-api resolve to an AMI id"""
test_string = "ami-id,data-api"
resolver = EFVersionResolver(TestEFVersionResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^ami-[a-f0-9]{8}$")
def merge_files(context):
"""
Given a context containing path to template, env, and service:
merge config into template and output the result to stdout
Args:
context: a populated context object
"""
resolver = EFTemplateResolver(profile=context.profile,
region=context.region,
env=context.env,
service=context.service)
try:
with open(context.template_path, 'r') as f:
template_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading template file: {} {}".format(
context.template_path, repr(error)))
if context.no_params is False:
try:
with open(context.param_path, 'r') as f:
param_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading param file: {} {}".format(
context.param_path, repr(error)))
dest = yaml.safe_load(param_body)["dest"]
# if 'dest' for the current object contains an 'environments' list, check it
if "environments" in dest:
if not resolver.resolved["ENV_SHORT"] in dest["environments"]:
print("Environment: {} not enabled for {}".format(
resolver.resolved["ENV_SHORT"], context.template_path))
return
# Process the template_body - apply context + parameters
resolver.load(template_body, param_body)
else:
resolver.load(template_body)
rendered_body = resolver.render()
if not resolver.resolved_ok():
raise RuntimeError(
"Couldn't resolve all symbols; template has leftover {{ or }}: {}".
format(resolver.unresolved_symbols()))
if context.lint:
if context.template_path.endswith(".json"):
try:
json.loads(rendered_body, strict=False)
print("JSON passed linting process.")
except ValueError as e:
fail("JSON failed linting process.", e)
elif context.template_path.endswith((".yml", ".yaml")):
conf = yamllint_config.YamlLintConfig(content='extends: relaxed')
lint_output = yamllinter.run(rendered_body, conf)
lint_level = 'error'
lint_errors = [
issue for issue in lint_output if issue.level == lint_level
]
if lint_errors:
split_body = rendered_body.splitlines()
for error in lint_errors:
print(error)
# printing line - 1 because lists start at 0, but files at 1
print("\t", split_body[error.line - 1])
fail("YAML failed linting process.")
if context.verbose:
print(context)
if context.no_params:
print('no_params flag set to true!')
print(
'Inline template resolution based on external symbol lookup only and no destination for file write.\n'
)
else:
dir_path = normpath(dirname(dest["path"]))
print("make directories: {} {}".format(dir_path, dest["dir_perm"]))
print("chmod file to: " + dest["file_perm"])
user, group = dest["user_group"].split(":")
print("chown last directory in path to user: {}, group: {}".format(
user, group))
print("chown file to user: {}, group: {}\n".format(user, group))
print("template body:\n{}\nrendered body:\n{}\n".format(
template_body, rendered_body))
elif context.silent:
print("Config template rendered successfully.")
else:
print(rendered_body)
class TestEFAwsResolver(unittest.TestCase):
"""Tests for `ef_aws_resolver.py`."""
# initialize based on where running
where = whereami()
if where == "local":
session = boto3.Session(profile_name=context.account_alias,
region_name=EFConfig.DEFAULT_REGION)
elif where == "ec2":
region = http_get_metadata("placement/availability-zone/")
region = region[:-1]
session = boto3.Session(region_name=region)
else:
fail("Can't test in environment: " + where)
clients = {
"cloudformation": session.client("cloudformation"),
"cloudfront": session.client("cloudfront"),
"ec2": session.client("ec2"),
"iam": session.client("iam"),
"route53": session.client("route53"),
"waf": session.client("waf")
}
## Test coverage of ec2:eni/eni-id is disabled because the we are not presently creating
## ENI fixtures and this test does not at present generate an ENI for testing this lookup function
## Why are these retained here? The lookup function is still valid, and useful. We just can't test it at the moment
# def test_ec2_eni_eni_id(self):
# """Does ec2:eni/eni-id,eni-proto3-dnsproxy-1a resolve to an ENI ID"""
# test_string = "ec2:eni/eni-id,eni-proto3-dnsproxy-1a"
# resolver = EFAwsResolver(TestEFAwsResolver.clients)
# self.assertRegexpMatches(resolver.lookup(test_string), "^eni-[a-f0-9]{8}$")
# def test_ec2_eni_eni_id_none(self):
# """Does ec2:eni/eni-id,cant_possibly_match return None"""
# test_string = "ec2:eni/eni-id,cant_possibly_match"
# resolver = EFAwsResolver(TestEFAwsResolver.clients)
# self.assertIsNone(resolver.lookup(test_string))
# def test_ec2_eni_eni_id_default(self):
# """Does ec2:eni/eni-id,cant_possibly_match,DEFAULT return default value"""
# test_string = "ec2:eni/eni-id,cant_possibly_match,DEFAULT"
# resolver = EFAwsResolver(TestEFAwsResolver.clients)
# self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_elasticip_elasticip_id(self):
"""Does ec2:elasticip/elasticip-id,ElasticIpMgmtCingest1 resolve to elastic IP allocation ID"""
test_string = "ec2:elasticip/elasticip-id,ElasticIpMgmtCingest1"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^eipalloc-[a-f0-9]{8}$")
def test_ec2_elasticip_elasticip_id_none(self):
"""Does ec2:elasticip/elasticip-id,cant_possibly_match return None"""
test_string = "ec2:elasticip/elasticip-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_elasticip_elasticip_id_default(self):
"""Does ec2:elasticip/elasticip-id,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:elasticip/elasticip-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_elasticip_elasticip_ipaddress(self):
"""Does ec2:elasticip/elasticip-ipaddress,ElasticIpMgmtCingest1 resolve to elastic IP address"""
test_string = "ec2:elasticip/elasticip-ipaddress,ElasticIpMgmtCingest1"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(
resolver.lookup(test_string),
"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$")
def test_ec2_elasticip_elasticip_ipaddress_none(self):
"""Does ec2:elasticip/elasticip-ipaddress,cant_possibly_match return None"""
test_string = "ec2:elasticip/elasticip-ipaddress,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_elasticip_elasticip_ipaddress_default(self):
"""Does ec2:elasticip/elasticip-ipaddress,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:elasticip/elasticip-ipaddress,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_route_table_main_route_table_id(self):
"""Does ec2:route-table/main-route-table-id,vpc-<env> resolve to route table ID"""
test_string = "ec2:route-table/main-route-table-id,vpc-" + context.env
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^rtb-[a-f0-9]{8}$")
def test_ec2_route_table_main_route_table_id_none(self):
"""Does ec2:route-table/main-route-table-id,cant_possibly_match return None"""
test_string = "ec2:route-table/main-route-table-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_route_table_main_route_table_id_default(self):
"""Does ec2:route-table/main-route-table-id,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:route-table/main-route-table-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_security_group_security_group_id(self):
"""Does ec2:security-group/security-group-id,staging-core-ec2 resolve to a security group id"""
test_string = "ec2:security-group/security-group-id,staging-core-ec2"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^sg-[a-f0-9]{8}$")
def test_ec2_security_group_security_group_id_none(self):
"""Does ec2:security-group/security-group-id,cant_possibly_match return None"""
test_string = "ec2:security-group/security-group-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_security_group_security_group_id_default(self):
"""Does ec2:security-group/security-group-id,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:security-group/security-group-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_subnet_subnet_id(self):
"""Does ec2:subnet/subnet-id,subnet-staging-a resolve to a subnet ID"""
test_string = "ec2:subnet/subnet-id,subnet-staging-a"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^subnet-[a-f0-9]{8}$")
def test_ec2_subnet_subnet_id_none(self):
"""Does ec2:subnet/subnet-id,cant_possibly_match return None"""
test_string = "ec2:subnet/subnet-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_subnet_subnet_id_default(self):
"""Does ec2:subnet/subnet-id,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:subnet/subnet-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_vpc_availabilityzones(self):
"""Does ec2:vpc/availabilityzones,vpc-staging resolve to correctly-delimited string of AZ(s)"""
test_string = "ec2:vpc/availabilityzones,vpc-staging"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(
resolver.lookup(test_string),
"^us-west-2(a|b)(\", \"us-west-2(a|b)){0,1}$")
def test_ec2_vpc_availabilityzones_none(self):
"""Does ec2:vpc/availabilityzones,cant_possibly_match return None"""
test_string = "ec2:vpc/availabilityzones,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_vpc_availabilityzones_default(self):
"""Does ec2:vpc/availabilityzones,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:vpc/availabilityzones,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_vpc_cidrblock(self):
"""Does ec2:vpc/cidrblock,vpc-staging resolve to a CIDR block"""
test_string = "ec2:vpc/cidrblock,vpc-staging"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(
resolver.lookup(test_string),
"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\/\d{2}$")
def test_ec2_vpc_cidrblock_none(self):
"""Does ec2:vpc/cidrblock,cant_possibly_match return None"""
test_string = "ec2:vpc/cidrblock,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_vpc_cidrblock_default(self):
"""Does ec2:vpc/cidrblock,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:vpc/cidrblock,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_vpc_subnets(self):
"""Does ec2:vpc/subnets,vpc-staging resolve to correctly-delimited string of AZ(s)"""
test_string = "ec2:vpc/subnets,vpc-staging"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(
resolver.lookup(test_string),
"^subnet-[a-f0-9]{8}(\", \"subnet-[a-f0-9]{8}){0,1}$")
def test_ec2_vpc_subnets_none(self):
"""Does ec2:vpc/subnets,cant_possibly_match return None"""
test_string = "ec2:vpc/subnets,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_vpc_subnets_default(self):
"""Does ec2:vpc/subnets,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:vpc/subnets,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_ec2_vpc_vpc_id(self):
"""Does ec2:vpc/vpc-id,vpc-staging resolve to VPC ID"""
test_string = "ec2:vpc/vpc-id,vpc-staging"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^vpc-[a-f0-9]{8}$")
def test_ec2_vpc_vpc_id_none(self):
"""Does ec2:vpc/vpc-id,cant_possibly_match return None"""
test_string = "ec2:vpc/vpc-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_ec2_vpc_vpc_id_default(self):
"""Does ec2:vpc/vpc-id,cant_possibly_match,DEFAULT return default value"""
test_string = "ec2:vpc/vpc-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_waf_rule_id(self):
"""Does waf:rule-id,global-OfficeCidr resolve to WAF ID"""
test_string = "waf:rule-id,global-OfficeCidr"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(
resolver.lookup(test_string),
"^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")
def test_waf_rule_id_none(self):
"""Does waf:rule-id,cant_possibly_match return None"""
test_string = "waf:rule-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_waf_rule_id_default(self):
"""Does waf:rule-id,cant_possibly_match,DEFAULT return default value"""
test_string = "waf:rule-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_waf_web_acl_id(self):
"""Does waf:web-acl-id,staging-StaticAcl resolve to Web ACL ID"""
test_string = "waf:web-acl-id,staging-StaticAcl"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(
resolver.lookup(test_string),
"^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$")
def test_waf_web_acl_id_none(self):
"""Does waf:web-acl-id,cant_possibly_match return None"""
test_string = "waf:web-acl-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_waf_web_acl_id_default(self):
"""Does waf:web-acl-id,cant_possibly_match,DEFAULT return default value"""
test_string = "waf:web-acl-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_route53_private_hosted_zone_id(self):
"""Does route53:private-hosted-zone-id,cx-proto0.com. resolve to zone ID"""
test_string = "route53:private-hosted-zone-id,cx-proto0.com."
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^[A-Z0-9]{13,14}$")
def test_route53_private_hosted_zone_id_none(self):
"""Does route53:private-hosted-zone-id,cant_possibly_match return None"""
test_string = "route53:private-hosted-zone-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_route53_private_hosted_zone_id_default(self):
"""Does route53:private-hosted-zone-id,cant_possibly_match,DEFAULT return default value"""
test_string = "route53:private-hosted-zone-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_route53_public_hosted_zone_id(self):
"""Does route53:hosted-zone-id,cx-proto0.com. resolve to zone ID"""
test_string = "route53:public-hosted-zone-id,cx-proto0.com."
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^[A-Z0-9]{13,14}$")
def test_route53_public_hosted_zone_id_none(self):
"""Does route53:public-hosted-zone-id,cant_possibly_match return None"""
test_string = "route53:public-hosted-zone-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_route53_public_hosted_zone_id_default(self):
"""Does route53:public-hosted-zone-id,cant_possibly_match,DEFAULT return default value"""
test_string = "route53:public-hosted-zone-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_cloudfront_domain_name(self):
"""Does cloudfront:domain-name,static.cx-proto0.com resolve to a Cloudfront FQDN"""
test_string = "cloudfront:domain-name,static.cx-proto0.com"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^[a-z0-9]{13,14}.cloudfront.net$")
def test_cloudfront_domain_name_none(self):
"""Does cloudfront:domain-name,cant_possibly_match return None"""
test_string = "cloudfront:domain-name,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_cloudfront_domain_name_default(self):
"""Does cloudfront:domain-name,cant_possibly_match,DEFAULT return default value"""
test_string = "cloudfront:domain-name,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_cloudfront_origin_access_identity_oai_id(self):
"""Does cloudfront:origin-access-identity/oai-id,static.cx-proto0.com resolve to oai ID"""
test_string = "cloudfront:origin-access-identity/oai-id,static.cx-proto0.com"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^[A-Z0-9]{13,14}$")
def test_cloudfront_origin_access_identity_oai_id_none(self):
"""Does cloudfront:origin-access-identity/oai-id,cant_possibly_match return None"""
test_string = "cloudfront:origin-access-identity/oai-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_cloudfront_origin_access_identity_oai_id_default(self):
"""Does cloudfront:origin-access-identity/oai-id,cant_possibly_match,DEFAULT return default value"""
test_string = "cloudfront:origin-access-identity/oai-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def test_cloudfront_origin_access_identity_oai_canonical_user_id(self):
"""Does cloudfront:origin-access-identity/oai-canonical-user-id,static.cx-proto0.com resolve to oai ID"""
test_string = "cloudfront:origin-access-identity/oai-canonical-user-id,static.cx-proto0.com"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string),
"^[a-z0-9]{96}$")
def test_cloudfront_origin_access_identity_oai_canonical_user_id_none(
self):
"""Does cloudfront:origin-access-identity/oai-canonical-user-id,cant_possibly_match return None"""
test_string = "cloudfront:origin-access-identity/oai-canonical-user-id,cant_possibly_match"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertIsNone(resolver.lookup(test_string))
def test_cloudfront_origin_access_identity_oai_canonical_user_id_default(
self):
"""Does cloudfront:origin-access-identity/oai-canonical-user-id,cant_possibly_match,DEFAULT return default value"""
test_string = "cloudfront:origin-access-identity/oai-canonical-user-id,cant_possibly_match,DEFAULT"
resolver = EFAwsResolver(TestEFAwsResolver.clients)
self.assertRegexpMatches(resolver.lookup(test_string), "^DEFAULT$")
def load(self, template, parameters=None):
"""
'template'
Loads template text from a 'string' or 'file' type
Template text contains {{TOKEN}} symbols to be replaced
'parameters'
parameters contains environment-specific sections as discussed in the class documentation.
the 'parameters' arg can be None, a 'string', 'file', or 'dictionary'
Whether from a string or file, or already in a dictionary, parameters must follow the
logical format documented in the class docstring.
if 'parameters' is omitted, template resolution will proceed with AWS, credential, and
version lookups.
"""
# load template
if isinstance(template, str):
self.template = template
elif isinstance(template, file):
try:
self.template = template.read()
template.close()
except IOError as error:
fail("Exception loading template from file: ", error)
else:
fail("Unknown type loading template; expected string or file: " + type(template))
# load parameters, if any
if parameters:
if isinstance(parameters, str):
try:
self.parameters = yaml.safe_load(parameters)
except ValueError as error:
fail("Exception loading parameters from string: ", error)
elif isinstance(parameters, file):
try:
self.parameters = yaml.safe_load(parameters)
parameters.close()
except ValueError as error:
fail("Exception loading parameters from file: {}".format(error), sys.exc_info())
elif isinstance(parameters, dict):
self.parameters = parameters
else:
fail("Unknown type loading parameters; expected string, file, or dict: " + type(parameters))
# sanity check the loaded parameters
if "params" not in self.parameters:
fail("'params' field not found in parameters")
# just the params, please
self.parameters = self.parameters["params"]
# are all the keys valid (must have legal characters)
for k in set().union(*(self.parameters[d].keys() for d in self.parameters.keys())):
invalid_char = re.search(ILLEGAL_PARAMETER_CHARS, k)
if invalid_char:
fail("illegal character: '" + invalid_char.group(0) + "' in parameter key: " + k)
def __init__(self,
profile=None, region=None, # set both for user access mode
lambda_context=None, # set if target is 'self' and this is a lambda
target_other=False, env=None, service=None, # set env & service if target_other=True
verbose=False
):
"""
Depending on how this is called, access mode (how it logs into AWS) and target (what the
various context vars report) will vary
ACCESS MODE - how this logs in to AWS
user: "******"
both 'profile' and 'region' are required
is always "operating on something else" (TARGET is never "self")
role: "running in AWS EC2 or Lambda with a role credential"
do not set profile
TARGET - what context is reported
self: "this ec2 instance or lambda is initializing itself"
assumed for ec2 and lambda, unless target_other=True in the constructor
never an option for "user" access mode
other: "this local user, ec2 instance, or lambda is configuring something else"
always "other" if access mode is "user"
if access mode is "role", set target_other=True in the constructor
Constructor must also set 'env' and 'service'
self:
lambda_context must be provided if this is a lambda; leave it unset for ec2
INSTANCE_ID = instance ID if EC2, else None
FUNCTION_NAME = function name if lambda, else None
ACCOUNT = numeric account this is running in
ACCOUNT_ALIAS = named alias of the account this is running in
ROLE = role this is running as
ENV = the environment this is running in, from role name
ENV_SHORT = derived from ENV
ENV_FULL = fully qualified environment, same as ENV unless env is a global env (mgmt.* or global.*)
SERVICE = the service this is, from role name
REGION = region this is running in
something else:
INSTANCE_ID = None
FUNCTION_NAME = None
ACCOUNT = the numeric account I'm logged into (look up)
ACCOUNT_ALIAS = the alias of the account i'm logged into (look up)
ROLE = None
ENV = the target's environment, passed in from the constructor
ENV_SHORT = derived from ENV
ENV_FULL = ENV, with ".<ACCOUNT_ALIAS>" as appropriate
SERVICE = the service name, passed in from the constructor
REGION = the region I am in (ec2, lambda) or explicitly set (region= in constructor)
Collects instance's environment for use in templates:
{{ACCOUNT}} - AWS account number
CloudFormation can use this or the AWS::AccountID pseudo param
{{ACCOUNT_ALIAS}} - AWS account alias
{{ENV}} - environment: mgmt, prod, staging, proto<N>, etc.
{{ENV_SHORT}} - env with <N> or account trimmed: mgmt, prod, staging, proto, etc.
{{ENV_FULL}} - env fully qualified: prod, staging, proto<N>, mgmt.<account_alias>, etc.
{{FUNCTION_NAME}} - only for lambdas
{{INSTANCE_ID}} - only for ec2
{{REGION}} - the region currently being worked in
CloudFormation can use this or the AWS::Region pseudo param
{{ROLE}} - the role bound to the ec2 instance or lambda; only for ec2 and lambda
CloudFormation: compose role name in template by joining other strings
"""
# instance vars
self.verbose = False # print noisy status if True
# resolved tokens - only look up symbols once per session. Protect internal names by declaring
self.resolved = {
"ACCOUNT": None,
"ACCOUNT_ALIAS": None,
"ENV": None,
"ENV_SHORT": None,
"ENV_FULL": None,
"FUNCTION_NAME": None,
"INSTANCE_ID": None,
"REGION": None,
"ROLE": None
}
# template and parameters are populated by the load() method as each template is processed
self.template = None
# parameters that accompany this template, if any
self.parameters = {}
# Sets of symbols found in the current template (only)
# read back with self.symbols() and self.unresolved_symbols()
self.symbols = set()
# capture verbosity pref from constructor
self.verbose = verbose
# determine ACCESS MODE
if profile: # accessing as a user
target_other = True
if not region:
fail("'region' is required with 'profile' for user-mode access")
where = whereami()
# require env and service params init() when target is 'other'
if (target_other or where == "virtualbox-kvm") and (env is None or service is None):
fail("'env' and 'service' must be set when target is 'other' or running in " + where)
if target_other or profile:
self.resolved["REGION"] = region
# lambda initializing self
elif lambda_context:
self.resolved["REGION"] = lambda_context.invoked_function_arn.split(":")[3]
# ec2 initializing self
else:
self.resolved["REGION"] = get_metadata_or_fail("placement/availability-zone/")[:-1]
# Create clients - if accessing by role, profile should be None
clients = [
"cloudformation",
"cloudfront",
"cognito-identity",
"cognito-idp",
"ec2",
"elbv2",
"iam",
"kms",
"lambda",
"route53",
"s3",
"sts",
"waf"
]
try:
EFTemplateResolver.__CLIENTS = create_aws_clients(self.resolved["REGION"], profile, *clients)
except RuntimeError as error:
fail("Exception logging in with Session()", error)
# Create EFAwsResolver object for interactive lookups
EFTemplateResolver.__AWSR = EFAwsResolver(EFTemplateResolver.__CLIENTS)
# Create EFConfigResolver object for ef tooling config lookups
EFTemplateResolver.__EFCR = EFConfigResolver()
# Create EFVersionResolver object for version lookups
EFTemplateResolver.__VR = EFVersionResolver(EFTemplateResolver.__CLIENTS)
# Set the internal parameter values for aws
# self-configuring lambda
if (not target_other) and lambda_context:
arn_split = lambda_context.invoked_function_arn.split(":")
self.resolved["ACCOUNT"] = arn_split[4]
self.resolved["FUNCTION_NAME"] = arn_split[6]
try:
lambda_desc = EFTemplateResolver.__CLIENTS["lambda"].get_function()
except:
fail("Exception in get_function: ", sys.exc_info())
self.resolved["ROLE"] = lambda_desc["Configuration"]["Role"]
env = re.search("^({})-".format(EFConfig.VALID_ENV_REGEX), self.resolved["ROLE"])
if not env:
fail("Did not find environment in lambda function name.")
self.resolved["ENV"] = env.group(1)
parsed_service = re.search(self.resolved["ENV"] + "-(.*?)-lambda", self.resolved["ROLE"])
if parsed_service:
self.resolved["SERVICE"] = parsed_service.group(1)
# self-configuring EC2
elif (not target_other) and (not lambda_context):
self.resolved["INSTANCE_ID"] = get_metadata_or_fail('instance-id')
try:
instance_desc = EFTemplateResolver.__CLIENTS["ec2"].describe_instances(InstanceIds=[self.resolved["INSTANCE_ID"]])
except:
fail("Exception in describe_instances: ", sys.exc_info())
self.resolved["ACCOUNT"] = instance_desc["Reservations"][0]["OwnerId"]
arn = instance_desc["Reservations"][0]["Instances"][0]["IamInstanceProfile"]["Arn"]
self.resolved["ROLE"] = arn.split(":")[5].split("/")[1]
env = re.search("^({})-".format(EFConfig.VALID_ENV_REGEX), self.resolved["ROLE"])
if not env:
fail("Did not find environment in role name")
self.resolved["ENV"] = env.group(1)
self.resolved["SERVICE"] = "-".join(self.resolved["ROLE"].split("-")[1:])
# target is "other"
else:
try:
if whereami() == "ec2":
self.resolved["ACCOUNT"] = str(json.loads(http_get_metadata('iam/info'))["InstanceProfileArn"].split(":")[4])
else:
self.resolved["ACCOUNT"] = get_account_id(EFTemplateResolver.__CLIENTS["sts"])
except botocore.exceptions.ClientError as error:
fail("Exception in get_user()", error)
self.resolved["ENV"] = env
self.resolved["SERVICE"] = service
# ACCOUNT_ALIAS is resolved consistently for access modes and targets other than virtualbox
try:
self.resolved["ACCOUNT_ALIAS"] = EFTemplateResolver.__CLIENTS["iam"].list_account_aliases()["AccountAliases"][0]
except botocore.exceptions.ClientError as error:
fail("Exception in list_account_aliases", error)
# ENV_SHORT is resolved the same way for all access modes and targets
self.resolved["ENV_SHORT"] = self.resolved["ENV"].strip(".0123456789")
# ENV_FULL is resolved the same way for all access modes and targets, depending on previously-resolved values
if self.resolved["ENV"] in EFConfig.ACCOUNT_SCOPED_ENVS:
self.resolved["ENV_FULL"] = "{}.{}".format(self.resolved["ENV"], self.resolved["ACCOUNT_ALIAS"])
else:
self.resolved["ENV_FULL"] = self.resolved["ENV"]
if self.verbose:
print(repr(self.resolved), file=sys.stderr)
def cmd_set(context):
"""
Set the new "current" value for a key.
If the existing current version and the new version have identical /value/ and /status,
then nothing is written, to avoid stacking up redundant entreis in the version table.
Args:
context: a populated EFVersionContext object
"""
# If key value is a special symbol, see if this env allows it
if context.value in EFConfig.SPECIAL_VERSIONS and context.env_short not in EFConfig.SPECIAL_VERSION_ENVS:
fail("special version: {} not allowed in env: {}".format(context.value, context.env_short))
# If key value is a special symbol, the record cannot be marked "stable"
if context.value in EFConfig.SPECIAL_VERSIONS and context.stable:
fail("special versions such as: {} cannot be marked 'stable'".format(context.value))
# Resolve any references
if context.value == "=prod":
context.value = context.versionresolver.lookup("{},{}/{}".format(context.key, "prod", context.service_name))
elif context.value == "=staging":
context.value = context.versionresolver.lookup("{},{}/{}".format(context.key, "staging", context.service_name))
elif context.value == "=latest":
if not EFConfig.VERSION_KEYS[context.key]["allow_latest"]:
fail("=latest cannot be used with key: {}".format(context.key))
func_name = "_getlatest_" + context.key.replace("-", "_")
if func_name in globals() and isfunction(globals()[func_name]):
context.value = globals()[func_name](context)
else:
raise RuntimeError("{} version for {}/{} is '=latest' but can't look up because method not found: {}".format(
context.key, context.env, context.service_name, func_name))
# precheck to confirm coherent world state before attempting set - whatever that means for the current key type
try:
precheck(context)
except Exception as e:
fail("Precheck failed: {}".format(e.message))
s3_key = "{}/{}/{}".format(context.service_name, context.env, context.key)
s3_version_status = EFConfig.S3_VERSION_STATUS_STABLE if context.stable else EFConfig.S3_VERSION_STATUS_UNDEFINED
# If the set would put a value and status that are the same as the existing 'current' value/status, don't do it
context.limit = 1
current_version = get_versions(context)
# If there is no 'current version' it's ok, just means the set will write the first entry
if len(current_version) == 1 and current_version[0].status == s3_version_status and \
current_version[0].value == context.value:
print("Version not written because current version and new version have identical value and status: {} {}"
.format(current_version[0].value, current_version[0].status))
return
if not context.commit:
print("=== DRY RUN ===\nUse --commit to set value\n=== DRY RUN ===")
print("would set key: {} with value: {} {} {} {} {}".format(
s3_key, context.value, context.build_number, context.commit_hash, context.location, s3_version_status))
else:
context.aws_client("s3").put_object(
ACL='bucket-owner-full-control',
Body=context.value,
Bucket=EFConfig.S3_VERSION_BUCKET,
ContentEncoding=EFConfig.S3_VERSION_CONTENT_ENCODING,
Key=s3_key,
Metadata={
EFConfig.S3_VERSION_BUILDNUMBER_KEY: context.build_number,
EFConfig.S3_VERSION_COMMITHASH_KEY: context.commit_hash,
EFConfig.S3_VERSION_LOCATION_KEY: context.location,
EFConfig.S3_VERSION_MODIFIEDBY_KEY: context.aws_client("sts").get_caller_identity()["Arn"],
EFConfig.S3_VERSION_STATUS_KEY: s3_version_status
},
StorageClass='STANDARD'
)
print("set key: {} with value: {} {} {} {} {}".format(
s3_key, context.value, context.build_number, context.commit_hash, context.location, s3_version_status))
def handle_args_and_set_context(args):
"""
Args:
args: the command line args, probably passed from main() as sys.argv[1:]
Returns:
a populated EFVersionContext object
"""
parser = argparse.ArgumentParser()
parser.add_argument("service_name", help="name of the service")
parser.add_argument("key", help="version key to look up for <service_name> such as 'ami-id' (list in EF_Config)")
parser.add_argument("env", help=", ".join(EFConfig.ENV_LIST))
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument("--get", help="get current version", action="store_true")
group.add_argument("--set", help="set current version of <key> to <value> for <service_name>")
group.add_argument("--rollback", help="set current version to most recent 'stable' version in history",
action="store_true")
group.add_argument("--history", help="Show version history for env/service/key", choices=['json', 'text'])
group.add_argument("--show", help="Show keys and values. '*' allowed for <key> and <env>",
action="store_true", default=False)
parser.add_argument("--build",
help="On --set, also set the externally defined build number associated with the version entity",
default="")
parser.add_argument("--commit_hash", help="On --set, also set the commit hash associated with the version entity",
default="")
parser.add_argument("--commit", help="Actually --set or --rollback (dry run if omitted)",
action="store_true", default=False)
parser.add_argument("--devel", help="Allow running from branch; don't refresh from origin", action="store_true",
default=False)
parser.add_argument("--force_env_full", help="Override env with env_full for account-scoped environments",
action="store_true", default=False)
parser.add_argument("--limit", help="Limit 'history', 'rollback', 'show' to first N records (default 100, max 1000)",
type=int, default=100)
parser.add_argument("--location", help="On --set, also mark the url location of the static build's version file to"
"support dist-hash precheck", default="")
if EFConfig.ALLOW_EF_VERSION_SKIP_PRECHECK:
parser.add_argument("--noprecheck", help="--set or --rollback without precheck", action="store_true", default=False)
parser.add_argument("--sr", help="optional /path/to/service_registry_file.json", default=None)
parser.add_argument("--stable", help="On --set, also mark the version 'stable'", action="store_true")
parser.add_argument("--verbose", help="Print additional info", action="store_true", default=False)
# parse
parsed_args = vars(parser.parse_args(args))
context = EFVersionContext()
# marshall the inherited context values
context._build_number = parsed_args["build"]
context._commit_hash = parsed_args["commit_hash"]
context.commit = parsed_args["commit"]
context.devel = parsed_args["devel"]
context._force_env_full = parsed_args["force_env_full"]
try:
context.env = parsed_args["env"]
except ValueError as e:
fail("Error in env: {}".format(e.message))
# marshall this module's additional context values
context._get = parsed_args["get"]
context._history = parsed_args["history"]
context._key = parsed_args["key"]
if EFConfig.ALLOW_EF_VERSION_SKIP_PRECHECK:
context._noprecheck = parsed_args["noprecheck"]
if not 1 <= parsed_args["limit"] <= 1000:
fail("Error in --limit. Valid range: 1..1000")
context._limit = parsed_args["limit"]
context._location = parsed_args["location"]
context._rollback = parsed_args["rollback"]
context._service_name = parsed_args["service_name"]
context._show = parsed_args["show"]
context._stable = parsed_args["stable"]
context._value = parsed_args["set"]
# Set up service registry and policy template path which depends on it
context.service_registry = EFServiceRegistry(parsed_args["sr"])
# VERBOSE is global
global VERBOSE
VERBOSE = parsed_args["verbose"]
validate_context(context)
return context