def resolve_policy_document(policy_name):
policy_filename = "{}{}.json".format(CONTEXT.policy_template_path, policy_name)
print_if_verbose("Load policy: {} from file: {}".format(policy_name, policy_filename))
# retrieve policy template
try:
policy_file = file(policy_filename, 'r')
policy_template = policy_file.read()
policy_file.close()
except:
fail("error opening policy file: {}".format(policy_filename))
print_if_verbose("pre-resolution policy template:\n{}".format(policy_template))
# If running in EC2, do not set profile and set target_other=True
if CONTEXT.whereami == "ec2":
resolver = EFTemplateResolver(target_other=True, env=CONTEXT.env, region=EFConfig.DEFAULT_REGION,
service=CONTEXT.service, verbose=CONTEXT.verbose)
else:
resolver = EFTemplateResolver(profile=CONTEXT.account_alias, env=CONTEXT.env, region=EFConfig.DEFAULT_REGION,
service=CONTEXT.service, verbose=CONTEXT.verbose)
resolver.load(policy_template)
policy_document = resolver.render()
print_if_verbose("resolved policy document:\n{}".format(policy_document))
if not resolver.resolved_ok():
fail("policy template {} has unresolved symbols or extra {{ or }}: {}".format(
policy_filename, resolver.unresolved_symbols()))
return policy_document
def merge_files(context):
"""
Given a context containing path to template, env, and service:
merge config into template and output the result to stdout
Args:
context: a populated context object
"""
resolver = EFTemplateResolver(profile=context.profile,
region=context.region,
env=context.env,
service=context.service)
try:
with open(context.template_path, 'r') as f:
template_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading template file: {} {}".format(
context.template_path, repr(error)))
try:
with open(context.param_path, 'r') as f:
param_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading param file: {} {}".format(
context.param_path, repr(error)))
dest = json.loads(param_body)["dest"]
# if 'dest' for the current object contains an 'environments' list, check it
if dest.has_key("environments"):
if not resolver.resolved["ENV_SHORT"] in dest["environments"]:
print("Environment: {} not enabled for {}".format(
resolver.resolved["ENV_SHORT"], context.template_path))
return
# Process the template_body - apply context + parameters
resolver.load(template_body, param_body)
rendered_body = resolver.render()
if not resolver.resolved_ok():
raise RuntimeError(
"Couldn't resolve all symbols; template has leftover {{ or }}: {}".
format(resolver.unresolved_symbols()))
if context.verbose:
print(context)
dir_path = normpath(dirname(dest["path"]))
print("make directories: {} {}".format(dir_path, dest["dir_perm"]))
print("chmod file to: " + dest["file_perm"])
user, group = dest["user_group"].split(":")
print("chown last directory in path to user: {}, group: {}".format(
user, group))
print("chown file to user: {}, group: {}\n".format(user, group))
print("template body:\n{}\nrendered body:\n{}\n".format(
template_body, rendered_body))
else:
print(rendered_body)
def resolve_template(template, profile, env, region, service, verbose):
# resolve {{SYMBOLS}} in the passed template file
isfile(template) or fail("Not a file: {}".format(template))
resolver = EFTemplateResolver(profile=profile,
target_other=True,
env=env,
region=region,
service=service,
verbose=verbose)
with open(template) as template_file:
resolver.load(template_file)
resolver.render()
if verbose:
print(resolver.template)
dangling_left, dangling_right = resolver.count_braces()
if resolver.unresolved_symbols():
fail("Unable to resolve symbols: " +
",".join(["{{" + s + "}}"
for s in resolver.unresolved_symbols()]))
elif dangling_left > 0 or dangling_right > 0:
fail("Some {{ or }} were not resolved. left{{: {}, right}}: {}".format(
dangling_left, dangling_right))
else:
return resolver.template
def test_context_vars_protected(self):
"""Context vars like {{ENV}} are not overridden even if present in template"""
test_string = "{{ENV}}"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), TEST_ENV)
def test_embedded_symbols(self):
"""Does a symbol built from other symbols resolve correctly"""
test_string = "{{{{o}}{{ne}}}}"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "testenv one")
def test_hierarchical_overlays(self):
"""Is the hierarchy of default..env applied correctly"""
test_string = "{{one}}|{{two}}|{{my-thing}}"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(),
"testenv one|testenv two|my-hyphen-thing")
def test_newline_literal(self, mock_create_aws):
"""Do newline literals get converted to newlines"""
mock_create_aws.return_value = self._clients
test_string = "foo\nbar"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "foo\nbar")
def test_unresolved_symbols(self):
"""Are unresolved symbols stored and reported, and non-symbols ignored"""
test_string = "{{cannot_resolve}}{{not a symbo}}{{notasymbol?}}{{cannot_resolve}}"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.unresolved_symbols(),
set(["cannot_resolve"]))
def test_newline_literal_against_raw(self, mock_create_aws):
"""Another check to make sure newline literals are not mistakenly written as r'\n'"""
mock_create_aws.return_value = self._clients
test_string = "foo\nbar"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertNotEqual(resolver.render(), r'foo\nbar')
def test_leading_dot_context(self, mock_create_aws):
"""Do context symbols with a leading dot render correctly"""
mock_create_aws.return_value = self._clients
test_string = "{{.ENV}}"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), TEST_ENV)
def test_resolution(self):
"""Do context symbols resolve correctly"""
test_string = "{{one}}|{{two}}|{{/_-.}}|{{ENV}}"
resolver = EFTemplateResolver(profile=TEST_PROFILE,
env=TEST_ENV,
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(
resolver.render(),
"testenv one|testenv two|slashunderscoredashdot|proto0")
def test_load_yaml_file(self, mock_create_aws):
"""Does {{one}} resolve correctly from yaml parameters file"""
mock_create_aws.return_value = self._clients
test_string = "{{one}}"
resolver = EFTemplateResolver(profile=get_account_alias("alpha0"),
env="alpha0",
region=TEST_REGION,
service=TEST_SERVICE)
with open(self.test_params_yaml) as yaml_file:
resolver.load(test_string, yaml_file)
self.assertEqual(resolver.render(), "alpha one")
def test_render_multiline_string_from_list(self, mock_create_aws):
"""Does {{multi}} resolve correctly as a multiline string from yaml parameters file"""
mock_create_aws.return_value = self._clients
test_string = "{{multi2}}"
resolver = EFTemplateResolver(profile=get_account_alias("test"),
env="test",
region=TEST_REGION,
service=TEST_SERVICE)
with open(self.test_params_json) as json_file:
resolver.load(test_string, json_file)
self.assertEqual(resolver.render(), "one\ntwo\nthree")
def test_render_multiline_string(self, mock_create_aws):
"""Does {{multi}} resolve correctly as a multiline string from yaml parameters file"""
mock_create_aws.return_value = self._clients
test_string = "{{multi}}"
resolver = EFTemplateResolver(profile=get_account_alias("test"),
env="test",
region=TEST_REGION,
service=TEST_SERVICE)
with open(self.test_params_yaml) as yaml_file:
resolver.load(test_string, yaml_file)
self.assertEqual(
resolver.render(),
"thisisareallylongstringthatcoversmultiple\nlinesfortestingmultilinestrings"
)
def test_fully_qualified_env(self):
"""Does {{ENV_FULL}} resolve correctly"""
# proto0
test_string = "{{ENV_FULL}}"
resolver = EFTemplateResolver(profile=get_account_alias("proto0"),
env="proto0",
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "proto0")
# prod
resolver = EFTemplateResolver(profile=get_account_alias("prod"),
env="prod",
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "prod")
# mgmt.ellationeng
resolver = EFTemplateResolver(
profile=get_account_alias("mgmt.ellationeng"),
env="mgmt.ellationeng",
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "mgmt.ellationeng")
def test_fully_qualified_env(self, mock_create_aws):
"""Does {{ENV_FULL}} resolve correctly"""
mock_create_aws.return_value = self._clients
# alpha0
test_string = "{{ENV_FULL}}"
resolver = EFTemplateResolver(profile=get_account_alias("alpha0"),
env="alpha0",
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "alpha0")
# prod
resolver = EFTemplateResolver(profile=get_account_alias("test"),
env="test",
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "test")
# mgmt.testaccount
resolver = EFTemplateResolver(
profile=get_account_alias("mgmt.testaccount"),
env="mgmt.testaccount",
region=TEST_REGION,
service=TEST_SERVICE)
resolver.load(test_string, PARAMS)
self.assertEqual(resolver.render(), "mgmt.testaccount")
def merge_files(service, skip_on_user_group_error=False):
"""
Given a prefix, find all templates below; merge with parameters; write to "dest"
Args:
service: "<service>", "all", or "ssh"
skip_on_user_group_error: True or False
For S3, full path becomes:
s3://ellation-cx-global-configs/<service>/templates/<filename>
s3://ellation-cx-global-configs/<service>/parameters/<filename>.parameters.<yaml|yml|json>
For filesystem, full path becomes:
/vagrant/configs/<service>/templates/<filename>
/vagrant/configs/<service>/parameters/<filename>.parameters.<yaml|yml|json>
"""
if WHERE == "ec2":
config_reader = EFInstanceinitConfigReader("s3", service, log_info,
RESOURCES["s3"])
resolver = EFTemplateResolver()
elif WHERE == "virtualbox-kvm":
config_path = "{}/{}".format(VIRTUALBOX_CONFIG_ROOT, service)
config_reader = EFInstanceinitConfigReader("file", config_path,
log_info)
environment = EFConfig.VAGRANT_ENV
resolver = EFTemplateResolver(env=environment,
profile=get_account_alias(environment),
region=EFConfig.DEFAULT_REGION,
service=service)
while config_reader.next():
log_info("checking: {}".format(config_reader.current_key))
# if 'dest' for the current object contains an 'environments' list, check it
dest = config_reader.dest
if "environments" in dest:
if not resolver.resolved["ENV_SHORT"] in dest["environments"]:
log_info("Environment: {} not enabled for {}".format(
resolver.resolved["ENV_SHORT"], config_reader.current_key))
continue
# If 'dest' for the current object contains a user_group that hasn't been created in the environment yet and the
# flag is set to True to skip, log the error and move onto the next config file without blowing up.
if skip_on_user_group_error:
user, group = get_user_group(dest)
try:
getpwnam(user).pw_uid
except KeyError:
log_info(
"File specifies user {} that doesn't exist in environment. Skipping config file."
.format(user))
continue
try:
getgrnam(group).gr_gid
except KeyError:
log_info(
"File specifies group {} that doesn't exist in environment. Skipping config file."
.format(group))
continue
# Process the template_body - apply context + parameters
log_info("Resolving template")
resolver.load(config_reader.template, config_reader.parameters)
rendered_body = resolver.render()
if not resolver.resolved_ok():
critical(
"Couldn't resolve all symbols; template has leftover {{ or }}: {}"
.format(resolver.unresolved_symbols()))
# Write the rendered file
dir_path = normpath(dirname(dest["path"]))
# Resolved OK. try to write the template
log_info("make directories: {} {}".format(dir_path, dest["dir_perm"]))
try:
makedirs(dir_path, int(dest["dir_perm"], 8))
except OSError as error:
if error.errno != 17:
critical("Error making directories {}".format(repr(error)))
log_info("open: " + dest["path"] + ",w+")
try:
outfile = open(dest["path"], 'w+')
log_info("write")
outfile.write(rendered_body)
log_info("close")
outfile.close()
log_info("chmod file to: " + dest["file_perm"])
chmod(dest["path"], int(dest["file_perm"], 8))
user, group = get_user_group(dest)
uid = getpwnam(user).pw_uid
gid = getgrnam(group).gr_gid
log_info("chown last directory in path to: " + dest["user_group"])
chown(dir_path, uid, gid)
log_info("chown file to: " + dest["user_group"])
chown(dest["path"], uid, gid)
except Exception as error:
critical("Error writing file: " + dest["path"] + ": " +
repr(error))
EFConfig resolver, accountaliasofenv,prod: {{efconfig:accountaliasofenv,staging}}\n\
AMI lookup: {{version:ami-id,proto0/test-instance}}\n\
Latest AMI for test-instance: {{version:ami-id,proto0/test-instance}}\n\
Custom Data: \"{{efconfig:customdata,office_ips}}\"\
"
GLOBAL_ENV_TEST_STRING = "fully-qualified environment:{{ENV_FULL}}\n"
# Test with proto0
if LOCAL:
resolver = EFTemplateResolver(profile=get_account_alias("proto0"), env="proto0", region="us-west-2",
service="mine", verbose=True)
else:
resolver = EFTemplateResolver(verbose=True)
resolver.load(TEST_STRING, PARAMS)
resolver.render()
print(resolver.template)
print("unresolved symbol count: "+str(len(resolver.unresolved_symbols())))
print("unresolved symbols: "+repr(resolver.unresolved_symbols()))
print("all template symbols: "+repr(resolver.symbols))
print("all EFTemplateResolver symbols: "+repr(resolver.resolved))
# Demo with the global env 'mgmt.ellationeng' (local only)
if LOCAL:
resolver = EFTemplateResolver(profile=get_account_alias("mgmt.ellationeng"), env="mgmt.ellationeng", region="us-west-2",
service="mine", verbose=True)
resolver.load(GLOBAL_ENV_TEST_STRING, PARAMS)
def merge_files(context):
"""
Given a context containing path to template, env, and service:
merge config into template and output the result to stdout
Args:
context: a populated context object
"""
resolver = EFTemplateResolver(profile=context.profile,
region=context.region,
env=context.env,
service=context.service)
try:
with open(context.template_path, 'r') as f:
template_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading template file: {} {}".format(
context.template_path, repr(error)))
if context.no_params is False:
try:
with open(context.param_path, 'r') as f:
param_body = f.read()
f.close()
except IOError as error:
raise IOError("Error loading param file: {} {}".format(
context.param_path, repr(error)))
dest = yaml.safe_load(param_body)["dest"]
# if 'dest' for the current object contains an 'environments' list, check it
if "environments" in dest:
if not resolver.resolved["ENV_SHORT"] in dest["environments"]:
print("Environment: {} not enabled for {}".format(
resolver.resolved["ENV_SHORT"], context.template_path))
return
# Process the template_body - apply context + parameters
resolver.load(template_body, param_body)
else:
resolver.load(template_body)
rendered_body = resolver.render()
if not resolver.resolved_ok():
raise RuntimeError(
"Couldn't resolve all symbols; template has leftover {{ or }}: {}".
format(resolver.unresolved_symbols()))
if context.lint:
if context.template_path.endswith(".json"):
try:
json.loads(rendered_body, strict=False)
print("JSON passed linting process.")
except ValueError as e:
fail("JSON failed linting process.", e)
elif context.template_path.endswith((".yml", ".yaml")):
conf = yamllint_config.YamlLintConfig(content='extends: relaxed')
lint_output = yamllinter.run(rendered_body, conf)
lint_level = 'error'
lint_errors = [
issue for issue in lint_output if issue.level == lint_level
]
if lint_errors:
split_body = rendered_body.splitlines()
for error in lint_errors:
print(error)
# printing line - 1 because lists start at 0, but files at 1
print("\t", split_body[error.line - 1])
fail("YAML failed linting process.")
if context.verbose:
print(context)
if context.no_params:
print('no_params flag set to true!')
print(
'Inline template resolution based on external symbol lookup only and no destination for file write.\n'
)
else:
dir_path = normpath(dirname(dest["path"]))
print("make directories: {} {}".format(dir_path, dest["dir_perm"]))
print("chmod file to: " + dest["file_perm"])
user, group = dest["user_group"].split(":")
print("chown last directory in path to user: {}, group: {}".format(
user, group))
print("chown file to user: {}, group: {}\n".format(user, group))
print("template body:\n{}\nrendered body:\n{}\n".format(
template_body, rendered_body))
elif context.silent:
print("Config template rendered successfully.")
else:
print(rendered_body)
def merge_files(service):
"""
Given a prefix, find all templates below; merge with parameters; write to "dest"
Args:
service: "<service>" or "all"
For S3, full path becomes:
s3://ellation-cx-global-configs/<service>/templates/<filename>
s3://ellation-cx-global-configs/<service>/parameters/<filename>.parameters.json
For filesystem, full path becomes:
/vagrant/configs/<service>/templates/<filename>
/vagrant/configs/<service>/parameters/<filename>.parameters.json
"""
if WHERE == "ec2":
config_reader = EFInstanceinitConfigReader("s3", service, log_info,
RESOURCES["s3"])
elif WHERE == "virtualbox-kvm":
config_path = "{}/{}".format(VIRTUALBOX_CONFIG_ROOT, service)
config_reader = EFInstanceinitConfigReader("file", config_path,
log_info)
while config_reader.next():
# Make a new TemplateResolver for every file::parameters pair, so cached keys don't carry over
if WHERE == "ec2":
resolver = EFTemplateResolver()
elif WHERE == "virtualbox-kvm":
resolver = EFTemplateResolver(env="localvm", service=service)
log_info("checking: {}".format(config_reader.current_key))
# if 'dest' for the current object contains an 'environments' list, check it
dest = config_reader.dest
if dest.has_key("environments"):
if not resolver.resolved["ENV_SHORT"] in dest["environments"]:
log_info("Environment: {} not enabled for {}".format(
resolver.resolved["ENV_SHORT"], config_reader.current_key))
continue
# Process the template_body - apply context + parameters
log_info("Resolving template")
resolver.load(config_reader.template, config_reader.parameters)
rendered_body = resolver.render()
if not resolver.resolved_ok():
critical(
"Couldn't resolve all symbols; template has leftover {{ or }}: {}"
.format(resolver.unresolved_symbols()))
# Write the rendered file
dir_path = normpath(dirname(dest["path"]))
# Resolved OK. try to write the template
log_info("make directories: {} {}".format(dir_path, dest["dir_perm"]))
try:
makedirs(dir_path, int(dest["dir_perm"], 8))
except OSError as error:
if error.errno != 17:
critical("Error making directories {}".format(repr(error)))
log_info("open: " + dest["path"] + ",w+")
try:
outfile = open(dest["path"], 'w+')
log_info("write")
outfile.write(rendered_body)
log_info("close")
outfile.close()
log_info("chmod file to: " + dest["file_perm"])
chmod(dest["path"], int(dest["file_perm"], 8))
user, group = dest["user_group"].split(":")
uid = getpwnam(user).pw_uid
gid = getgrnam(group).gr_gid
log_info("chown last directory in path to: " + dest["user_group"])
chown(dir_path, uid, gid)
log_info("chown file to: " + dest["user_group"])
chown(dest["path"], uid, gid)
except Exception as error:
critical("Error writing file: " + dest["path"] + ": " +
repr(error))