def get_master_key(env: str) -> None:
"""
Get the master key for SealedSecrets for the given env.
:param str env: The environment
"""
settings = load_env_settings(env)
ensure_context(settings.KUBE_CONTEXT)
label(logger.info, f"Getting master key for {env}")
# Based on:
# https://github.com/bitnami-labs/sealed-secrets#how-can-i-do-a-backup-of-my-sealedsecrets
result = run([
"kubectl",
"get",
"secret",
"-n",
"kube-system",
"-l",
"sealedsecrets.bitnami.com/sealed-secrets-key",
"-o",
"yaml",
])
content = result.stdout.decode(encoding="utf-8")
output_file = master_key_path(env=env)
logger.info(f"Saving master key to {output_file}")
output_file.write_text(content, encoding="utf-8")
def kubeval(keep_configs=False):
"""
Check that all Kubernetes configs look valid with kubeval
"""
label(logger.info, "Checking Kubernetes configs")
def _should_ignore(path):
if TMP in path.parents:
return True
return False
merge_tmp = TMP / f"kubeval-{generate_random_id()}"
kube_yamls = [
str(get_merged_kube_file(path, merge_tmp))
for path in Path(".").glob("**/kube/*.yaml")
if not _should_ignore(path)
]
skip_kinds = ",".join(KUBEVAL_SKIP_KINDS)
run(["kubeval", "--strict", "--skip-kinds", skip_kinds] + kube_yamls)
if not keep_configs and merge_tmp.exists():
logger.info(f"Removing temporary kube merges from {merge_tmp}")
rmtree(merge_tmp)
if keep_configs and merge_tmp.exists():
logger.info(f"Keeping temporary kube merges in {merge_tmp}")
def _patch_generic(self, doc: dict):
logger.info("Applying generic patches")
meta = doc["metadata"]
if self.namespace:
logger.info(f"Updating namespace to {self.namespace}")
meta["namespace"] = self.namespace
def _release_kube_config(self, ctx: Context, config: Path, dry_run: bool):
if dry_run:
logger.info(f"[DRY RUN] Applying {config}")
return
logger.info(f"Applying {config}")
run(["kubectl", "apply", "-f", config])
def _delete_kube_config(self, ctx: Context, config: Path, dry_run: bool):
if dry_run:
logger.info(f"[DRY RUN] Deleting {config}")
return
logger.info(f"Deleting {config}")
run(["kubectl", "delete", "-f", config])
def unseal_secrets(env: str) -> None:
"""
Decrypts the secrets for the desired env and base64 decodes them to make
them easy to edit.
:param str env: The environment.
"""
# Validate env
load_env_settings(env)
master_key = get_master_key(env=env)
secrets_pem = secrets_pem_path(env=env)
sealed_secret_files = [
secret_file
for secret_file in (Path("envs") / env / "secrets").glob("*.yaml")
if not secret_file.name.endswith(UNSEALED_SECRETS_EXTENSION)
]
label(logger.info, f"Unsealing secrets for {env}")
for input_file in sealed_secret_files:
output_file = input_file.with_name(input_file.stem +
UNSEALED_SECRETS_EXTENSION)
logger.info(f"Unsealing {input_file} to {output_file}")
content = input_file.read_text(encoding="utf-8")
content = kube_unseal(content, master_key, cert=secrets_pem)
content = base64_decode_secrets(content)
output_file.write_text(content, encoding="utf-8")
def seal_secrets(env: str) -> None:
"""
Base64 encodes and seals the secrets for the desired env.
:param str env: The environment.
"""
# Validate env
load_env_settings(env)
secrets_pem = secrets_pem_path(env=env)
unsealed_secret_files = (Path("envs") / env /
"secrets").glob(f"*{UNSEALED_SECRETS_EXTENSION}")
label(logger.info, f"Sealing secrets for {env}")
for input_file in unsealed_secret_files:
output_file_name = input_file.name[:-len(UNSEALED_SECRETS_EXTENSION
)] + ".yaml"
output_file = input_file.with_name(output_file_name)
logger.info(f"Sealing {input_file} as {output_file}")
content = input_file.read_text(encoding="utf-8")
content = base64_encode_secrets(content)
content = kube_seal(content, cert=secrets_pem)
output_file.write_text(content, encoding="utf-8")
def render_template_kind(self, kind, env, settings):
plural_kind = f"{kind}s"
if kind not in TEMPLATE_KINDS:
raise Exception(f"Unsupported kind of template: {kind}")
output_path = Path("envs") / env / plural_kind / self.path.as_posix() / "kube"
# Remove all old rendered files of this kind, but leave any manually created ones
logger.info(f"Cleaning up old {kind} files for {self.name} for env {env}")
old_files = output_path.glob("*.yaml")
for old_file in old_files:
template_path = old_file.relative_to(Path("envs") / env / plural_kind)
template_path = (
template_path.parent / f"{kind}-templates" / template_path.name
)
with old_file.open(mode="r", encoding="utf-8") as f:
content = f.read()
if content.startswith(TEMPLATE_HEADER.format(file=template_path)):
old_file.unlink()
logger.debug(f"Deleted rendered file {old_file}")
else:
logger.debug(
f"Keeping {kind} file {old_file}, it does not appear to have been rendered from a template"
)
jinja_context = getattr(settings, "TEMPLATE_VARIABLES", {})
rendered_files = []
if not self.kube_templates[kind]:
return rendered_files
logger.info(f"Creating {kind} files for {self.name} for env {env}")
if not output_path.is_dir():
output_path.mkdir(mode=0o700, parents=True)
for name, template_path in self.kube_templates[kind].items():
with template_path.open(mode="r", encoding="utf-8") as f:
content = f.read()
template = jinja2.Template(content, undefined=jinja2.StrictUndefined)
try:
content = TEMPLATE_HEADER.format(file=template_path)
content += template.render(jinja_context)
content += "\n"
except jinja2.exceptions.UndefinedError as ex:
raise Exception(
f"Failed to render template {template_path} for env {env}, "
f"reason: {ex.message}"
)
output_file = output_path / name
with output_file.open(mode="w", encoding="utf-8") as f:
f.write(content)
rendered_files.append(output_file)
logger.debug(f"Rendered {kind} file {output_file}")
return rendered_files
def patch_from_env(self, env):
env_path = Path("envs") / env / "overrides" / self.path.as_posix()
for match in (env_path / "kube").glob("*.yaml"):
logger.info(f"Found kube override {match.name} for {self.name} in {env}")
self.kube_configs[match.name] = match
merge_path = Path("envs") / env / "merges" / self.path.as_posix()
for match in (merge_path / "kube").glob("*.yaml"):
logger.info(f"Found kube merges {match.name} for {self.name} in {env}")
self.kube_merges[match.name] = match
def load_env_settings(env: str) -> Settings:
module = f"envs.{env}.settings"
logger.info(f"Loading settings from {module}")
settings = importlib.import_module(module)
# Set some defaults for optional values
settings.IMAGE_PULL_SECRETS = getattr(settings, "IMAGE_PULL_SECRETS", {})
settings.REPLICAS = getattr(settings, "REPLICAS", {})
return settings
def _get_kube_configs(self, path=None):
if path is None:
path = self.path
config = {}
for match in (path / "kube").glob("*.yaml"):
logger.info(f"Found kube config {match.name} for {self.name}")
config[match.name] = match
return config
def _get_kube_templates(self, path=None):
if path is None:
path = self.path
templates = defaultdict(dict)
for kind in TEMPLATE_KINDS:
for match in (path / "kube" / f"{kind}-templates").glob("*.yaml"):
logger.info(f"Found {kind}-template {match.name} for {self.name}")
templates[kind][match.name] = match
return templates
def validate_release_configs(ctx):
envs = list_envs()
for env in envs:
logger.info("Validating configs for {} environment".format(env))
settings = load_env_settings(env)
components = settings.COMPONENTS
for path in components:
component = Component(path)
component.validate(ctx)
component.patch_from_env(env)
component.validate(ctx)
def release_env(ctx: Context, env, dry_run=False):
env_path = Path("envs") / env
secrets = sorted([
secret_file for secret_file in (env_path / "secrets").glob("*.yaml")
if not secret_file.name.endswith(UNSEALED_SECRETS_EXTENSION)
])
for secret in secrets:
# Sealed Secrets can't be validated like this
# ctx.run(f"kubeval {secret}")
if dry_run:
logger.info(f"[DRY RUN] Applying {secret}")
continue
logger.info(f"Applying {secret}")
run(["kubectl", "apply", "-f", secret])
old_secrets = (env_path / "secrets" / "obsolete").glob("*.yaml")
for secret in sorted(old_secrets, reverse=True):
if dry_run:
logger.info(f"[DRY RUN] Deleting {secret}")
continue
logger.info(f"Deleting {secret}")
run(["kubectl", "delete", "-f", secret])
def _get_obsolete_kube_configs(self, path=None):
if path is None:
path = self.path
obs_path = path / "kube" / "obsolete"
configs = {}
if not obs_path.exists():
return configs
for match in obs_path.glob("*.yaml"):
logger.info(f"Found obsoleted kube config {match.name} for {self.name}")
configs[match.name] = match
return configs
def _restart_resource(
self, ctx: Context, resource: str, dry_run: bool, no_rollout_wait: bool
):
if dry_run:
logger.info(f"[DRY RUN] Restarting resource {resource}")
return
logger.info(f"Restarting resource {resource}")
run(["kubectl", "-n", self.namespace, "rollout", "restart", resource])
if not no_rollout_wait:
run(
["kubectl", "-n", self.namespace, "rollout", "status", resource],
timeout=ROLLOUT_TIMEOUT,
)
def run(args,
cwd=None,
check=True,
env=None,
stream=False,
timeout=None,
input=None) -> subprocess.CompletedProcess:
"""
Run a command
:param List[str] args:
:param str cwd:
:param bool check:
:param dict env:
:param bool stream: If the output should be streamed instead of captured
:param float timeout: Seconds to wait before failing
:param bytes input: Data to be sent to the child process via stdin
:raises subprocess.CalledProcessError:
:raises subprocess.TimeoutExpired:
:return subprocess.CompletedProcess:
"""
# Convert Paths to strings
for index, value in enumerate(args):
args[index] = str(value)
logger.info(" " + " ".join(args))
kwargs = {"cwd": cwd, "check": check, "env": env, "input": input}
if not stream:
kwargs["stdout"] = subprocess.PIPE
kwargs["stderr"] = subprocess.PIPE
if timeout:
kwargs["timeout"] = timeout
start = time()
try:
res = subprocess.run(args, **kwargs) # nosec
except subprocess.CalledProcessError as e:
logger.error("Failed to run " + " ".join(args))
log_subprocess_output(e, logger.error)
logger.error(f" ✘ ... failed in {time() - start:.3f}s")
raise
else:
log_subprocess_output(res, logger.debug)
logger.info(f" ✔ ... done in {time() - start:.3f}s")
return res
def _try_post_release(
self, ctx: Context, resource: str, selector: str, dry_run: bool
):
if dry_run:
logger.info(f"[DRY RUN] Running post-release.sh for {resource}")
return
result = run(
[
"kubectl",
"-n",
self.namespace,
"get",
"pods",
"-l",
selector,
"-o",
"json",
]
)
pods = []
image = self._get_full_docker_name()
for pod in json.loads(result.stdout)["items"]:
for container in pod["spec"]["containers"]:
if container["image"] == image:
pods.append(pod["metadata"]["name"])
if not pods:
raise Exception(f"No running pods with correct image found for {resource}")
pod = random.choice(pods) # nosec
run(
[
"kubectl",
"-n",
self.namespace,
"exec",
"-it",
pod,
"sh",
"post-release.sh",
],
check=False,
)
def _patch_image_pull_secrets(self, doc: dict):
spec = doc["spec"]
containers = spec["template"]["spec"]["containers"]
image = ""
if self.image:
image = self.image
else:
for container in containers:
image, _ = container["image"].split(":")
break
if "/" in image:
host, _ = image.split("/", maxsplit=1)
if host in self.image_pull_secrets:
secret = self.image_pull_secrets[host]
logger.info(f"Patching imagePullSecrets to {secret}")
tpl_spec = spec["template"]["spec"]
tpl_spec["imagePullSecrets"] = [{"name": secret}]
def seal_secrets(env: str, only_changed=False) -> None:
"""
Base64 encodes and seals the secrets for the desired env.
:param str env: The environment.
:param bool only_changed: Reseal only changed secrets.
"""
# Validate env
load_env_settings(env)
secrets_pem = secrets_pem_path(env=env)
unsealed_secret_files = (Path("envs") / env /
"secrets").glob(f"*{UNSEALED_SECRETS_EXTENSION}")
label(logger.info, f"Sealing secrets for {env}")
for input_file in unsealed_secret_files:
output_file_name = input_file.name[:-len(UNSEALED_SECRETS_EXTENSION
)] + ".yaml"
output_file = input_file.with_name(output_file_name)
logger.info(f"Sealing {input_file} as {output_file}")
content = input_file.read_text(encoding="utf-8")
content = base64_encode_secrets(content)
sealed_content = kube_seal(content, cert=secrets_pem)
if only_changed and output_file.exists():
master_key = get_master_key(env=env)
sealed_original_content = output_file.read_text(encoding="utf-8")
original_content = kube_unseal(sealed_original_content,
master_key,
cert=secrets_pem)
sealed_content = _revert_unchanged_secrets(
content, sealed_content, original_content,
sealed_original_content)
else:
# Load and dump yaml to ensure consistent formatting with above
sealed_content = yaml.safe_dump(yaml.safe_load(sealed_content))
output_file.write_text(sealed_content, encoding="utf-8")
def _patch_yaml_docs(self, config: List[dict]):
processed = []
for doc in config:
kind = doc["kind"]
if kind in SKIP_PATCH_KUBE_KINDS:
logger.info(f"Skipping {kind} patching")
continue
self._patch_generic(doc)
if kind == "Deployment":
self._patch_deployment(doc)
elif kind == "DaemonSet":
self._patch_daemon_set(doc)
elif kind == "StatefulSet":
self._patch_stateful_set(doc)
processed.append(doc)
return processed
def get_master_key(env: str, use_existing=True) -> Path:
"""
Get the master key for SealedSecrets for the given env.
:param str env: The environment
:param bool use_existing: If set to True, tries to use existing key from filesystem
instead of fetching a new one from the cluster.
:return Path: The path to the master key
"""
settings = load_env_settings(env)
master_key_file = master_key_path(env=env)
if use_existing and master_key_file.exists():
return master_key_file
ensure_context(settings.KUBE_CONTEXT)
label(logger.info, f"Getting master key for {env}")
# Based on:
# https://github.com/bitnami-labs/sealed-secrets#how-can-i-do-a-backup-of-my-sealedsecrets
result = run([
"kubectl",
"get",
"secret",
"-n",
"kube-system",
"-l",
"sealedsecrets.bitnami.com/sealed-secrets-key",
"-o",
"yaml",
])
content = result.stdout.decode(encoding="utf-8")
logger.info(f"Saving master key to {master_key_file}")
master_key_file.write_text(content, encoding="utf-8")
return master_key_file
def _patch_containers(self, doc: dict):
logger.info("Patching containers")
containers = doc["spec"]["template"]["spec"]["containers"]
for container in containers:
image, tag = container["image"].split(":")
if self.image:
logger.info(f"Patching image from {image} to {self.image}")
image = self.image
if self.tag:
logger.info(f"Patching tag from {tag} to {self.tag}")
tag = self.tag
container["image"] = f"{image}:{tag}"
def build(self, ctx: Context, dry_run=False):
label(logger.info, f"Building {self.path}")
dockerfile = self.path / "Dockerfile"
if not dockerfile.exists():
logger.info(f"No Dockerfile for {self.name} component")
return
if dry_run:
logger.info(f"[DRY RUN] Building {self.name} Docker image")
else:
logger.info(f"Building {self.name} Docker image")
tag = self._get_full_docker_name()
run(["docker", "build", self.path, "-t", tag], stream=True)
def release_env(ctx: Context, env, dry_run=False):
env_path = Path("envs") / env
secrets = (env_path / "secrets").glob("*.yaml")
for secret in sorted(secrets):
# Sealed Secrets can't be validated like this
# ctx.run(f"kubeval {secret}")
if dry_run:
logger.info(f"[DRY RUN] Applying {secret}")
continue
logger.info(f"Applying {secret}")
run(["kubectl", "apply", "-f", secret])
old_secrets = (env_path / "secrets" / "obsolete").glob("*.yaml")
for secret in sorted(old_secrets, reverse=True):
if dry_run:
logger.info(f"[DRY RUN] Deleting {secret}")
continue
logger.info(f"Deleting {secret}")
run(["kubectl", "delete", "-f", secret])
def _patch_yaml_docs(self, config: List[dict]):
processed = []
for doc in config:
kind = doc["kind"]
if kind in SKIP_PATCH_KUBE_KINDS:
logger.info(f"Skipping {kind} patching")
continue
self._patch_generic(doc)
if kind in ("Deployment", "DaemonSet", "StatefulSet"):
logger.info(f"Patching found {kind}")
self._patch_extra(doc)
elif kind == "CronJob":
logger.info(f"Patching found {kind}")
self._patch_cronjob(doc)
processed.append(doc)
return processed
def build(self, ctx: Context, dry_run=False, docker_args=None):
label(logger.info, f"Building {self.path}")
dockerfile = self.path / "Dockerfile"
if not dockerfile.exists():
logger.info(f"No Dockerfile for {self.name} component")
return
build_args = []
if docker_args:
# Insert --build-arg before each item from docker_args.
for docker_arg in docker_args:
build_args.extend(["--build-arg", docker_arg])
if dry_run:
logger.info(f"[DRY RUN] Building {self.name} Docker image")
else:
logger.info(f"Building {self.name} Docker image")
tag = self._get_full_docker_name()
run(["docker", "build", *build_args, self.path, "-t", tag],
stream=True)
def _prepare_configs(self, dst: Path):
dst = dst / self.path
kube_dst = dst / "kube"
kube_dst.mkdir(mode=0o700, parents=True)
logger.info(f"Writing configs to {dst}")
dockerfile = self.path / "Dockerfile"
if dockerfile.exists():
logger.info("Copying Dockerfile")
copy(dockerfile, dst / "Dockerfile")
for config in self.kube_configs:
config_file = self.path / "kube" / config
src = self.kube_configs[config] # Incl. env patch
logger.info(f"Patching {config_file}")
with src.open("r") as f:
docs = list(yaml.load_all(f, Loader))
self._patch_yaml_docs(docs)
if config in self.kube_merges:
# Use the Loader to get the values with the actual types.
with self.kube_merges[config].open("r") as f:
overrides = list(yaml.load_all(f, Loader))
# Use the BaseLoader to get literal values, such as tilde (~).
with self.kube_merges[config].open("r") as f:
base_overrides = list(yaml.load_all(f, BaseLoader))
docs = merge_docs(docs, overrides, base_overrides)
dst_path = kube_dst / config
with dst_path.open("w") as config_dst:
yaml.dump_all(docs, stream=config_dst, Dumper=Dumper)
self.kube_configs[config] = dst_path
self.path = dst
def release(
ctx,
env,
component=None,
image=None,
tag=None,
replicas=None,
dry_run=False,
keep_configs=False,
no_rollout_wait=False,
):
tags: dict = {}
images: dict = {}
replica_counts: dict = {}
components: List[str] = []
if image:
for i in image:
path, value = i.split("=")
images[path] = value
if tag:
for t in tag:
path, value = t.split("=")
tags[path] = value
if replicas:
for r in replicas:
path, value = r.split("=")
replica_counts[path] = value
rel_id = generate_release_id()
big_label(logger.info, f"Release {rel_id} to {env} environment starting")
settings = load_env_settings(env)
if component:
components = component
else:
components = settings.COMPONENTS
# Override env settings for replicas
if replica_counts:
for path in replica_counts:
settings.REPLICAS[path] = replica_counts[path]
rel_path = RELEASE_TMP / rel_id
logger.info("")
logger.info("Releasing components:")
for component in components:
logger.info(f" - {component}")
logger.info("")
logger.info("Setting images and tags:")
for path in components:
tag = "(default)"
image = "(default)"
if path in tags:
tag = tags[path]
if path in images:
image = images[path]
logger.info(f" - {path} = {image}:{tag}")
logger.info("")
ensure_context(settings.KUBE_CONTEXT)
ensure_namespace(settings.KUBE_NAMESPACE)
release_env(ctx, env, dry_run)
for path in components:
logger.info("")
label(logger.info, f"Releasing component {path}")
component = Component(path)
if path in images:
component.image = images[path]
images.pop(path)
if path in tags:
component.tag = tags[path]
tags.pop(path)
if path in settings.REPLICAS:
component.replicas = settings.REPLICAS[path]
replica_counts.pop(path, None)
component.namespace = settings.KUBE_NAMESPACE
component.context = settings.KUBE_CONTEXT
component.image_pull_secrets = settings.IMAGE_PULL_SECRETS
component.patch_from_env(env)
component.validate(ctx)
component.release(ctx, rel_path, dry_run, no_rollout_wait)
if images:
logger.error("Unprocessed image configurations:")
for path in images:
logger.error(f" - {path}={images[path]}")
if tags:
logger.error("Unprocessed tag configurations:")
for path in tags:
logger.error(f" - {path}={tags[path]}")
if replica_counts:
logger.error("Unprocessed replica configurations:")
for path in replica_counts:
logger.error(f" - {path}={replica_counts[path]}")
if not keep_configs:
logger.info(f"Removing temporary configurations from {rel_path}")
if rel_path.exists():
rmtree(rel_path)
def init_kubernetes(ctx, env):
"""
Initialize Kubernetes cluster
:param Context ctx:
:param str env:
:return:
"""
label(logger.info, f"Initializing Kubernetes for {env}")
settings = load_env_settings(env)
devops.tasks.ensure_context(settings.KUBE_CONTEXT)
devops.tasks.ensure_namespace(settings.KUBE_NAMESPACE)
def _get_kube_files(kube_context):
kube_files = {f.name: f for f in Path("kube").glob("*.yaml")}
overrides = (Path("kube") / kube_context / "overrides").glob("*.yaml")
for f in overrides:
kube_files[f.name] = f
# Convert to sorted list
kube_files = [kube_files[name] for name in sorted(kube_files.keys())]
return kube_files
def _apply(config, **kwargs):
run(["kubectl", "apply", "-f", config], **kwargs)
secrets = Path("envs") / env / "secrets.pem"
if env == LOCAL_ENV:
# Make sure local Sealed Secrets master key is applied first
master_key = Path("envs") / env / "secrets.key"
if master_key.exists():
logger.info(
f"Applying Sealed Secrets master key from {master_key}")
_apply(master_key, check=False)
for c in _get_kube_files(settings.KUBE_CONTEXT):
_apply(c)
# Wait for Sealed Secrets -controller to start up
run([
"kubectl",
"rollout",
"status",
"--namespace",
"kube-system",
"deploy/sealed-secrets-controller",
])
# And try to dump the signing cert
logger.info("Trying to fetch Sealed Secrets signing cert")
attempts = 5
while True:
try:
res = run(["kubeseal", "--fetch-cert"])
except CalledProcessError:
attempts -= 1
if attempts <= 0:
raise Exception("Failed to fetch Sealed Secrets cert")
sleep(2)
continue
with secrets.open("w") as dst:
dst.write(res.stdout.decode("utf-8"))
break
if env == LOCAL_ENV:
# Store master key if needed
master_key = Path("envs") / env / "secrets.key"
if not master_key.exists():
logger.info("Trying to store Sealed Secrets master key")
res = run([
"kubectl",
"get",
"secret",
"--namespace",
"kube-system",
"-o",
"custom-columns=name:metadata.name",
])
secrets = []
for line in res.stdout.decode("utf-8").splitlines():
if line.startswith("sealed-secrets-key"):
secrets.append(line)
with master_key.open("w") as dst:
first = True
for secret in secrets:
if not first:
dst.write("---\n")
first = False
res = run([
"kubectl",
"get",
"secret",
"--namespace",
"kube-system",
secret,
"-o",
"yaml",
])
print(res.stdout)
dst.write(res.stdout.decode("utf-8") + "\n")
