def test_save_yaml_file():
"""Test we can save yaml data"""
# save and read back data
tempdir = tempfile.mkdtemp()
data = {"avalue": "a"}
filename = os.path.join(tempdir, "foo/foo.yaml")
util.save_yaml_file(filename, data)
loaded = util.read_yaml_file(filename)
assert loaded["avalue"] == data["avalue"]
# save and read back data
comment = "# HELLO\n"
tempdir = tempfile.mkdtemp()
filename = os.path.join(tempdir, "bar/bar.yaml")
util.save_yaml_file(filename, data, comment)
with open(filename, "r") as f:
file_lines = f.readlines()
assert file_lines[0] == comment
loaded = util.read_yaml_file(filename)
assert loaded["avalue"] == data["avalue"]
shutil.rmtree(tempdir)
def test_substitute_placeholders_from_file_to_file_env():
"""Test end-to-end template processing and output file generation with env"""
# copy testcase to tempdir
tempdir = tempfile.mkdtemp()
relative_file = "teststack/0010-foo/test.kubectl.yaml"
source_filename = os.path.join(root_dir, relative_file)
input_filename = os.path.join(tempdir, relative_file)
input_dir = os.path.dirname(input_filename)
pathlib.Path(input_dir).mkdir(parents=True, exist_ok=True)
shutil.copy(source_filename, input_filename)
env_name = "dev"
data = {"parent_value": "override", constants.DATABAG_ENV_KEY: env_name}
processed_filename = util.get_processed_filename(tempdir, relative_file,
env_name)
util.substitute_placeholders_from_file_to_file(tempdir, relative_file, "#",
constants.UP_VERB, data)
# target exists and is valid yaml
yaml_data = util.read_yaml_file(processed_filename)
assert yaml_data["parent_value"] == data["parent_value"]
shutil.rmtree(tempdir)
def test_read_yaml_file():
"""Test we can read a yaml file and raise if its missing"""
filename = os.path.join(root_dir, ".env/databag.yaml")
yaml_data = util.read_yaml_file(filename)
assert yaml_data["common_value"] == "common"
# raises on missing file
with pytest.raises(FileNotFoundError):
_ = util.read_yaml_file("nothere.yaml")
# raises on empty file
_, temp_file = tempfile.mkstemp()
with pytest.raises(RuntimeError):
_ = util.read_yaml_file(temp_file)
os.unlink(temp_file)
def get_env_connections():
"""read `connections.yaml` for this environment"""
if not env_dir:
raise RuntimeError("env_dir not set yet, load databags first")
connections_yaml_filename = os.path.join(env_dir,
constants.CONNECTIONS_YAML)
return util.read_yaml_file(connections_yaml_filename)
def setup_connection(connection_settings):
global profile_data
profile = util.get_connection_profile(connection_settings, "snowflake")
snowflake_config_file = os.path.expanduser(SNOWFLAKE_CONFIG_FILE)
if os.path.exists(snowflake_config_file):
config = util.read_yaml_file(snowflake_config_file)
profile_data = config.get(profile)
else:
raise RuntimeError(
f"snowflake settings not found at: {snowflake_config_file}")
def do_secrets_manager(working_dir, filename, verb, data):
logger.info(f"secretsmanager: {filename}")
processed_file = util.substitute_placeholders_from_file_to_file(
working_dir, filename, "#", verb, data)
config = util.read_yaml_file(processed_file)
for secret in config.get("secrets", []):
ensure_secret(data, verb, secret)
# munged secrets files must not be left on disk
logger.debug(f"delete file that may contain secrets: {processed_file}")
os.unlink(processed_file)
def do_cloudflare(working_dir, filename, verb, data=None):
logger.info(f"cloudflare: {filename}")
try:
processed_file = util.substitute_placeholders_from_file_to_file(
working_dir, filename, "#", verb, data)
yaml_data = util.read_yaml_file(processed_file)
logger.debug(f"cloudflare file processed OK")
prefix = yaml_data.get("aws").get("secrets_manager_prefix") or ""
logger.debug(f"aws secretsmanager prefix: {prefix}")
# callback to run when a certificate has been created
# for now, always creates an AWS secret
def cb(_verb, hostname, certificate_data, private_key_data):
secret_string = json.dumps({
"tls.crt": certificate_data,
"tls.key": private_key_data,
})
secret = {
"name": f"{prefix}tls-{hostname.replace('.', '-')}",
"value": secret_string,
}
if (_verb == constants.UP_VERB and certificate_data and private_key_data) \
or _verb == constants.DOWN_VERB:
aws.ensure_secret(data, _verb, secret)
else:
logger.info(
f"[cloudflare-aws-callback] no update required for: {hostname}"
)
# lookup zone id now as needed everywhere
cf = get_cf()
zone_id = get_zone_id(cf, yaml_data["zone_name"])
origin_ca_certs(cf, zone_id, verb, yaml_data, cb)
edge_certs(cf, zone_id, verb, yaml_data)
zone_settings(cf, zone_id, verb, yaml_data)
# except RuntimeError as e:
# if verb == constants.DOWN_VERB:
# logger.warning(f"kubectl error - moving on: {e}")
# else:
# raise e
except KeyError as e:
if verb == constants.DOWN_VERB:
logger.warning(f"missing key - moving on: {e}")
else:
raise e
def do_remote_cloudformation(working_dir, filename, verb, data):
# local file contains a link to the S3 hosted cloudformation
config = util.read_yaml_file(filename)
remote = config["remote"]
local_file = os.path.join(constants.PROCESSED_DIR,
os.path.dirname(filename), config["local_file"])
# download the remote file so that:
# a) we can look at it
# b) we have a record of what was deployed
# c) we can grab the parameter list
# download a fresh copy every time since cloudformation will...
logger.debug(f"download {remote} to {local_file}")
util.download(remote, local_file)
stack_name = filename_to_stack_name(local_file)
cloudformation(stack_name, local_file, verb, data, template_url=remote)
def do_eksctl(working_dir, filename, verb, data):
logger.info(f"eksctl: ${filename}")
processed_filename = util.substitute_placeholders_from_file_to_file(
working_dir, filename, "#", verb, data)
config = util.read_yaml_file(processed_filename)
# eksctl needs cluster name - grab this well-known fields from
# processed yaml file to avoid needing well-known keys in databag
try:
cluster_name = config["metadata"]["name"]
except KeyError as e:
raise RuntimeError(
f"eksctl file:{filename} missing required value {e}")
try:
util.run_cmd(get_eksctl_cmd() + ["get", "cluster", "-n", cluster_name],
data)
exists = True
except RuntimeError as e:
logger.debug(f"cluster probably doesnt exist: {e}")
exists = False
if verb == constants.UP_VERB and exists:
logger.info(constants.MSG_UP_TO_DATE)
elif verb == constants.UP_VERB and not exists:
util.run_cmd(
get_eksctl_cmd() + ["create", "cluster", "-f", processed_filename],
data)
elif verb == constants.DOWN_VERB and exists:
util.run_cmd(
get_eksctl_cmd() + ["delete", "cluster", "--name", cluster_name],
data)
elif verb == constants.DOWN_VERB and not exists:
logger.info(constants.MSG_UP_TO_DATE)
else:
raise RuntimeError(f"eksctl - invalid verb: {verb}")
if verb == constants.UP_VERB:
eks_cluster_info(cluster_name, data)
def do_helm(working_dir, filename, verb, data=None):
logger.info(f"helm: {filename}")
# settings for helm...
processed_filename = util.substitute_placeholders_from_file_to_file(
working_dir, filename, "#", verb, data)
config = util.read_yaml_file(processed_filename)
# if there is an adjacent `values.yaml` file, process it for substitutions and use it
values_yaml = os.path.join(os.path.dirname(filename), "values.yaml")
if os.path.exists(values_yaml):
processed_values_file = util.substitute_placeholders_from_file_to_file(
working_dir, values_yaml, "#", verb, data)
logger.debug(f"helm - values: {processed_values_file}")
else:
processed_values_file = False
if verb == constants.UP_VERB:
helm_command = "install"
elif verb == constants.DOWN_VERB:
helm_command = "uninstall"
else:
raise RuntimeError(f"helm - invalid verb: {verb}")
base_cmd = get_helm_cmd()
if data.get("debug"):
base_cmd.append("--debug")
if config.get("namespace"):
namespace = config['namespace']
logger.debug(f"using namespace {namespace}")
base_cmd.append("-n")
base_cmd.append(namespace)
try:
# helm repos...
if verb == constants.UP_VERB:
helm_repos(base_cmd, config, filename)
# helm deployments
helm_list = util.run_cmd_json(base_cmd + ["list", "--output", "json"])
helm_deployments = list(map(lambda x: x["name"], helm_list))
logger.debug(f"helm deployments installed: {helm_deployments}")
exists = config["name"] in helm_deployments
if (verb == constants.UP_VERB
and exists) or (verb == constants.DOWN_VERB and not exists):
logger.info(f"helm - already installed:{config['name']}")
elif (verb == constants.UP_VERB
and not exists) or (verb == constants.DOWN_VERB and exists):
logger.info(f"helm - {helm_command}:{config['name']}")
cmd = base_cmd + [helm_command, config["name"]]
if verb == constants.UP_VERB:
cmd.append(config["install"])
if processed_values_file:
cmd.append("--values")
cmd.append(processed_values_file)
else:
for setting in config.get("set", []):
cmd.append("--set")
cmd.append(setting)
cmd += config.get("options", [])
version = config.get("version")
if version:
cmd += ["--version", version]
else:
logger.warning(
f"recommending versioning helm chart in {filename}")
cmd += config.get("options", [])
util.run_cmd(["helm", "repo", "update"])
util.run_cmd(cmd)
else:
raise RuntimeError(f"helm - invalid verb {verb}")
except KeyError as e:
raise RuntimeError(f"helm - {filename} missing key: {e}")
except RuntimeError as e:
if verb == constants.DOWN_VERB:
logger.error(f"helm - error running helm, moving on: {e}")
else:
raise e
