def unseal_secrets(env: str) -> None: """ Decrypts the secrets for the desired env and base64 decodes them to make them easy to edit. :param str env: The environment. """ # Validate env load_env_settings(env) master_key = get_master_key(env=env) secrets_pem = secrets_pem_path(env=env) sealed_secret_files = [ secret_file for secret_file in (Path("envs") / env / "secrets").glob("*.yaml") if not secret_file.name.endswith(UNSEALED_SECRETS_EXTENSION) ] label(logger.info, f"Unsealing secrets for {env}") for input_file in sealed_secret_files: output_file = input_file.with_name(input_file.stem + UNSEALED_SECRETS_EXTENSION) logger.info(f"Unsealing {input_file} to {output_file}") content = input_file.read_text(encoding="utf-8") content = kube_unseal(content, master_key, cert=secrets_pem) content = base64_decode_secrets(content) output_file.write_text(content, encoding="utf-8")
def seal_secrets(env: str) -> None: """ Base64 encodes and seals the secrets for the desired env. :param str env: The environment. """ # Validate env load_env_settings(env) secrets_pem = secrets_pem_path(env=env) unsealed_secret_files = (Path("envs") / env / "secrets").glob(f"*{UNSEALED_SECRETS_EXTENSION}") label(logger.info, f"Sealing secrets for {env}") for input_file in unsealed_secret_files: output_file_name = input_file.name[:-len(UNSEALED_SECRETS_EXTENSION )] + ".yaml" output_file = input_file.with_name(output_file_name) logger.info(f"Sealing {input_file} as {output_file}") content = input_file.read_text(encoding="utf-8") content = base64_encode_secrets(content) content = kube_seal(content, cert=secrets_pem) output_file.write_text(content, encoding="utf-8")
def get_master_key(env: str) -> None: """ Get the master key for SealedSecrets for the given env. :param str env: The environment """ settings = load_env_settings(env) ensure_context(settings.KUBE_CONTEXT) label(logger.info, f"Getting master key for {env}") # Based on: # https://github.com/bitnami-labs/sealed-secrets#how-can-i-do-a-backup-of-my-sealedsecrets result = run([ "kubectl", "get", "secret", "-n", "kube-system", "-l", "sealedsecrets.bitnami.com/sealed-secrets-key", "-o", "yaml", ]) content = result.stdout.decode(encoding="utf-8") output_file = master_key_path(env=env) logger.info(f"Saving master key to {output_file}") output_file.write_text(content, encoding="utf-8")
def seal_secrets(env: str, only_changed=False) -> None: """ Base64 encodes and seals the secrets for the desired env. :param str env: The environment. :param bool only_changed: Reseal only changed secrets. """ # Validate env load_env_settings(env) secrets_pem = secrets_pem_path(env=env) unsealed_secret_files = (Path("envs") / env / "secrets").glob(f"*{UNSEALED_SECRETS_EXTENSION}") label(logger.info, f"Sealing secrets for {env}") for input_file in unsealed_secret_files: output_file_name = input_file.name[:-len(UNSEALED_SECRETS_EXTENSION )] + ".yaml" output_file = input_file.with_name(output_file_name) logger.info(f"Sealing {input_file} as {output_file}") content = input_file.read_text(encoding="utf-8") content = base64_encode_secrets(content) sealed_content = kube_seal(content, cert=secrets_pem) if only_changed and output_file.exists(): master_key = get_master_key(env=env) sealed_original_content = output_file.read_text(encoding="utf-8") original_content = kube_unseal(sealed_original_content, master_key, cert=secrets_pem) sealed_content = _revert_unchanged_secrets( content, sealed_content, original_content, sealed_original_content) else: # Load and dump yaml to ensure consistent formatting with above sealed_content = yaml.safe_dump(yaml.safe_load(sealed_content)) output_file.write_text(sealed_content, encoding="utf-8")
def test_load_env_settings(clean_test_settings): envs = list_envs() settings = load_env_settings(envs[0]) getattr(settings, "IMAGE_PULL_SECRETS") getattr(settings, "KUBE_CONTEXT") getattr(settings, "KUBE_NAMESPACE") getattr(settings, "COMPONENTS") getattr(settings, "REPLICAS") TEST_ENV_PATH.mkdir(parents=True) TEST_ENV_SETTINGS.write_text(TEST_SETTINGS) settings = load_env_settings(TEST_ENV) assert len(settings.COMPONENTS) == 1 assert "service/TEST_COMPONENT_LOL" in settings.COMPONENTS assert settings.KUBE_CONTEXT == "TEST_CONTEXT_LOL" assert settings.KUBE_NAMESPACE == "TEST_NAMESPACE_LOL" assert len(settings.IMAGE_PULL_SECRETS) == 0 assert len(settings.REPLICAS) == 0
def validate_release_configs(ctx): envs = list_envs() for env in envs: logger.info("Validating configs for {} environment".format(env)) settings = load_env_settings(env) components = settings.COMPONENTS for path in components: component = Component(path) component.validate(ctx) component.patch_from_env(env) component.validate(ctx)
def update_from_templates(ctx): envs = list_envs() rendered_files = [] for env in envs: settings = load_env_settings(env) components = settings.COMPONENTS for path in components: component = Component(path) rendered_files.extend(component.render_templates(env, settings)) return rendered_files
def update_from_templates(): envs = list_envs() rendered_files = [] for env in envs: settings = load_env_settings(env) enabled_components = set(settings.COMPONENTS) components_in_filesystem = { p.parent.relative_to("envs", env, "merges").as_posix() for p in Path("envs", env, "merges").glob("**/kube") } components_in_filesystem |= { p.parent.relative_to("envs", env, "overrides").as_posix() for p in Path("envs", env, "overrides").glob("**/kube") } for path in enabled_components | components_in_filesystem: component = Component(path) rendered_files.extend(component.render_templates(env, settings)) return rendered_files
def get_master_key(env: str, use_existing=True) -> Path: """ Get the master key for SealedSecrets for the given env. :param str env: The environment :param bool use_existing: If set to True, tries to use existing key from filesystem instead of fetching a new one from the cluster. :return Path: The path to the master key """ settings = load_env_settings(env) master_key_file = master_key_path(env=env) if use_existing and master_key_file.exists(): return master_key_file ensure_context(settings.KUBE_CONTEXT) label(logger.info, f"Getting master key for {env}") # Based on: # https://github.com/bitnami-labs/sealed-secrets#how-can-i-do-a-backup-of-my-sealedsecrets result = run([ "kubectl", "get", "secret", "-n", "kube-system", "-l", "sealedsecrets.bitnami.com/sealed-secrets-key", "-o", "yaml", ]) content = result.stdout.decode(encoding="utf-8") logger.info(f"Saving master key to {master_key_file}") master_key_file.write_text(content, encoding="utf-8") return master_key_file
def release( ctx, env, component=None, image=None, tag=None, replicas=None, dry_run=False, keep_configs=False, no_rollout_wait=False, ): tags: dict = {} images: dict = {} replica_counts: dict = {} components: List[str] = [] if image: for i in image: path, value = i.split("=") images[path] = value if tag: for t in tag: path, value = t.split("=") tags[path] = value if replicas: for r in replicas: path, value = r.split("=") replica_counts[path] = value rel_id = generate_release_id() big_label(logger.info, f"Release {rel_id} to {env} environment starting") settings = load_env_settings(env) if component: components = component else: components = settings.COMPONENTS # Override env settings for replicas if replica_counts: for path in replica_counts: settings.REPLICAS[path] = replica_counts[path] rel_path = RELEASE_TMP / rel_id logger.info("") logger.info("Releasing components:") for component in components: logger.info(f" - {component}") logger.info("") logger.info("Setting images and tags:") for path in components: tag = "(default)" image = "(default)" if path in tags: tag = tags[path] if path in images: image = images[path] logger.info(f" - {path} = {image}:{tag}") logger.info("") ensure_context(settings.KUBE_CONTEXT) ensure_namespace(settings.KUBE_NAMESPACE) release_env(ctx, env, dry_run) for path in components: logger.info("") label(logger.info, f"Releasing component {path}") component = Component(path) if path in images: component.image = images[path] images.pop(path) if path in tags: component.tag = tags[path] tags.pop(path) if path in settings.REPLICAS: component.replicas = settings.REPLICAS[path] replica_counts.pop(path, None) component.namespace = settings.KUBE_NAMESPACE component.context = settings.KUBE_CONTEXT component.image_pull_secrets = settings.IMAGE_PULL_SECRETS component.patch_from_env(env) component.validate(ctx) component.release(ctx, rel_path, dry_run, no_rollout_wait) if images: logger.error("Unprocessed image configurations:") for path in images: logger.error(f" - {path}={images[path]}") if tags: logger.error("Unprocessed tag configurations:") for path in tags: logger.error(f" - {path}={tags[path]}") if replica_counts: logger.error("Unprocessed replica configurations:") for path in replica_counts: logger.error(f" - {path}={replica_counts[path]}") if not keep_configs: logger.info(f"Removing temporary configurations from {rel_path}") if rel_path.exists(): rmtree(rel_path)
def init_kubernetes(ctx, env): """ Initialize Kubernetes cluster :param Context ctx: :param str env: :return: """ label(logger.info, f"Initializing Kubernetes for {env}") settings = load_env_settings(env) devops.tasks.ensure_context(settings.KUBE_CONTEXT) devops.tasks.ensure_namespace(settings.KUBE_NAMESPACE) def _get_kube_files(kube_context): kube_files = {f.name: f for f in Path("kube").glob("*.yaml")} overrides = (Path("kube") / kube_context / "overrides").glob("*.yaml") for f in overrides: kube_files[f.name] = f # Convert to sorted list kube_files = [kube_files[name] for name in sorted(kube_files.keys())] return kube_files def _apply(config, **kwargs): run(["kubectl", "apply", "-f", config], **kwargs) secrets = Path("envs") / env / "secrets.pem" if env == LOCAL_ENV: # Make sure local Sealed Secrets master key is applied first master_key = Path("envs") / env / "secrets.key" if master_key.exists(): logger.info( f"Applying Sealed Secrets master key from {master_key}") _apply(master_key, check=False) for c in _get_kube_files(settings.KUBE_CONTEXT): _apply(c) # Wait for Sealed Secrets -controller to start up run([ "kubectl", "rollout", "status", "--namespace", "kube-system", "deploy/sealed-secrets-controller", ]) # And try to dump the signing cert logger.info("Trying to fetch Sealed Secrets signing cert") attempts = 5 while True: try: res = run(["kubeseal", "--fetch-cert"]) except CalledProcessError: attempts -= 1 if attempts <= 0: raise Exception("Failed to fetch Sealed Secrets cert") sleep(2) continue with secrets.open("w") as dst: dst.write(res.stdout.decode("utf-8")) break if env == LOCAL_ENV: # Store master key if needed master_key = Path("envs") / env / "secrets.key" if not master_key.exists(): logger.info("Trying to store Sealed Secrets master key") res = run([ "kubectl", "get", "secret", "--namespace", "kube-system", "-o", "custom-columns=name:metadata.name", ]) secrets = [] for line in res.stdout.decode("utf-8").splitlines(): if line.startswith("sealed-secrets-key"): secrets.append(line) with master_key.open("w") as dst: first = True for secret in secrets: if not first: dst.write("---\n") first = False res = run([ "kubectl", "get", "secret", "--namespace", "kube-system", secret, "-o", "yaml", ]) print(res.stdout) dst.write(res.stdout.decode("utf-8") + "\n")