def test_reveal_maybe_tag_no_reveal_flag(self): """ creates ?{base64:some_value} and runs reveal_maybe jinja2 filters without --reveal flag """ with tempfile.NamedTemporaryFile() as f: f.write("{{ my_ref_tag_var|reveal_maybe }}".encode("UTF-8")) f.seek(0) # new argparse namespace with --reveal and --refs-path values namespace = namedtuple("Namespace", []) namespace.reveal = False namespace.refs_path = tempfile.mkdtemp() # reveal_maybe uses cached, so inject namespace cached.args["compile"] = namespace cached.ref_controller_obj = RefController( cached.args["compile"].refs_path) cached.revealer_obj = Revealer(cached.ref_controller_obj) ref_tag = "?{base64:some_value}" ref_value = b"sitar_rock!" cached.ref_controller_obj[ref_tag] = Base64Ref(ref_value) context = {"my_ref_tag_var": ref_tag} self.assertEqual(render_jinja2_file(f.name, context), "?{base64:some_value}")
def trigger_compile(args): search_paths = [os.path.abspath(path) for path in args.search_paths] if not args.ignore_version_check: check_version() ref_controller = RefController(args.refs_path, embed_refs=args.embed_refs) # cache controller for use in reveal_maybe jinja2 filter cached.ref_controller_obj = ref_controller cached.revealer_obj = Revealer(ref_controller) compile_targets( args.inventory_path, search_paths, args.output_path, args.parallelism, args.targets, args.labels, ref_controller, prune=(args.prune), indent=args.indent, reveal=args.reveal, cache=args.cache, cache_paths=args.cache_paths, fetch_inventories=args.fetch, fetch_dependencies=args.fetch, force_fetch=args.force, validate=args.validate, schemas_path=args.schemas_path, jinja2_filters=args.jinja2_filters, verbose=hasattr(args, "verbose") and args.verbose, )
def test_reveal_maybe_b64encode_tag(self): """ creates ?{base64:some_value} and runs reveal_maybe|b64encode jinja2 filters """ with tempfile.NamedTemporaryFile() as f: f.write( "{{ my_ref_tag_var|reveal_maybe|b64encode }}".encode("UTF-8")) f.seek(0) # new argparse namespace with --reveal and --refs-path values namespace = namedtuple('Namespace', []) namespace.reveal = True namespace.refs_path = tempfile.mkdtemp() # reveal_maybe uses cached, so inject namespace cached.args['compile'] = namespace cached.ref_controller_obj = RefController( cached.args['compile'].refs_path) cached.revealer_obj = Revealer(cached.ref_controller_obj) ref_tag = '?{base64:some_value}' ref_value = b'sitar_rock!' cached.ref_controller_obj[ref_tag] = Base64Ref(ref_value) context = {"my_ref_tag_var": ref_tag} ref_value_b64 = base64.b64encode(ref_value).decode() self.assertEqual(render_jinja2_file(f.name, context), ref_value_b64)
def handle_refs_command(args): ref_controller = RefController(args.refs_path) if args.write is not None: ref_write(args, ref_controller) elif args.reveal: ref_reveal(args, ref_controller) elif args.update: secret_update(args, ref_controller) elif args.update_targets or args.validate_targets: secret_update_validate(args, ref_controller)
def kapitan_compile( config: Config, target="cluster", output_dir="./", search_paths=None, fake_refs=False, fetch_dependencies=True, reveal=False, ): if not search_paths: search_paths = [] search_paths = search_paths + [ "./", __install_dir__, ] reset_reclass_cache() refController = RefController("./catalog/refs") if fake_refs: refController.register_backend(FakeVaultBackend()) click.secho("Compiling catalog...", bold=True) cached.args["compile"] = ArgumentCache(inventory_path="./inventory") targets.compile_targets( inventory_path="./inventory", search_paths=search_paths, output_path=output_dir, targets=[target], parallel=4, labels=None, ref_controller=refController, verbose=config.trace, prune=False, indent=2, reveal=reveal, cache=False, cache_paths=None, fetch_dependencies=fetch_dependencies, validate=False, schemas_path="./schemas", jinja2_filters=defaults.DEFAULT_JINJA2_FILTERS_PATH, )
def test_reveal_maybe_no_tag(self): """ runs reveal_maybe jinja2 filter on data without ref tags """ with tempfile.NamedTemporaryFile() as f: f.write("{{ my_var|reveal_maybe }}".encode("UTF-8")) f.seek(0) # new argparse namespace with --reveal and --refs-path values namespace = namedtuple("Namespace", []) namespace.reveal = True namespace.refs_path = tempfile.mkdtemp() # reveal_maybe uses cached, so inject namespace cached.args["compile"] = namespace cached.ref_controller_obj = RefController(cached.args["compile"].refs_path) cached.revealer_obj = Revealer(cached.ref_controller_obj) var_value = "heavy_rock!" context = {"my_var": var_value} self.assertEqual(render_jinja2_file(f.name, context), var_value)
GNUPGHOME = tempfile.mkdtemp() os.environ["GNUPGHOME"] = GNUPGHOME gpg_obj(gnupghome=GNUPGHOME) KEY = cached.gpg_obj.gen_key(cached.gpg_obj.gen_key_input(key_type="RSA", key_length=2048, passphrase="testphrase")) KEY2 = cached.gpg_obj.gen_key(cached.gpg_obj.gen_key_input(key_type="RSA", key_length=2048, passphrase="testphrase")) GPG_TARGET_FINGERPRINTS["KEY"] = KEY.fingerprint GPG_KWARGS["passphrase"] = "testphrase" REFS_HOME = tempfile.mkdtemp() REF_CONTROLLER = RefController(REFS_HOME) REVEALER = Revealer(REF_CONTROLLER) class GPGSecretsTest(unittest.TestCase): "Test GPG secrets" def test_gpg_write_reveal(self): "write secret, confirm secret file exists, reveal and compare content" tag = '?{gpg:secret/sauce}' REF_CONTROLLER[tag] = GPGSecret("super secret value", [{'fingerprint': KEY.fingerprint}]) self.assertTrue(os.path.isfile(os.path.join(REFS_HOME, 'secret/sauce'))) file_with_secret_tags = tempfile.mktemp() with open(file_with_secret_tags, 'w') as fp: fp.write('I am a file with a ?{gpg:secret/sauce}')
def main(): """main function for command line usage""" parser = argparse.ArgumentParser(prog=PROJECT_NAME, description=DESCRIPTION) parser.add_argument("--version", action="version", version=VERSION) subparser = parser.add_subparsers(help="commands") eval_parser = subparser.add_parser("eval", help="evaluate jsonnet file") eval_parser.add_argument("jsonnet_file", type=str) eval_parser.add_argument( "--output", type=str, choices=("yaml", "json"), default=from_dot_kapitan("eval", "output", "yaml"), help='set output format, default is "yaml"', ) eval_parser.add_argument( "--vars", type=str, default=from_dot_kapitan("eval", "vars", []), nargs="*", metavar="VAR", help="set variables", ) eval_parser.add_argument( "--search-paths", "-J", type=str, nargs="+", default=from_dot_kapitan("eval", "search-paths", ["."]), metavar="JPATH", help='set search paths, default is ["."]', ) compile_parser = subparser.add_parser("compile", help="compile targets") compile_parser.add_argument( "--search-paths", "-J", type=str, nargs="+", default=from_dot_kapitan("compile", "search-paths", [".", "lib"]), metavar="JPATH", help='set search paths, default is ["."]', ) compile_parser.add_argument( "--jinja2-filters", "-J2F", type=str, default=from_dot_kapitan("compile", "jinja2-filters", defaults.DEFAULT_JINJA2_FILTERS_PATH), metavar="FPATH", help="load custom jinja2 filters from any file, default is to put\ them inside lib/jinja2_filters.py", ) compile_parser.add_argument( "--verbose", "-v", help="set verbose mode", action="store_true", default=from_dot_kapitan("compile", "verbose", False), ) compile_parser.add_argument( "--prune", help="prune jsonnet output", action="store_true", default=from_dot_kapitan("compile", "prune", False), ) compile_parser.add_argument( "--quiet", help="set quiet mode, only critical output", action="store_true", default=from_dot_kapitan("compile", "quiet", False), ) compile_parser.add_argument( "--output-path", type=str, default=from_dot_kapitan("compile", "output-path", "."), metavar="PATH", help='set output path, default is "."', ) compile_parser.add_argument( "--fetch", help="fetches external dependencies", action="store_true", default=from_dot_kapitan("compile", "fetch", False), ) compile_parser.add_argument( "--validate", help= "validate compile output against schemas as specified in inventory", action="store_true", default=from_dot_kapitan("compile", "validate", False), ) compile_parser.add_argument( "--parallelism", "-p", type=int, default=from_dot_kapitan("compile", "parallelism", 4), metavar="INT", help="Number of concurrent compile processes, default is 4", ) compile_parser.add_argument( "--indent", "-i", type=int, default=from_dot_kapitan("compile", "indent", 2), metavar="INT", help="Indentation spaces for YAML/JSON, default is 2", ) compile_parser.add_argument( "--refs-path", help='set refs path, default is "./refs"', default=from_dot_kapitan("compile", "refs-path", "./refs"), ) compile_parser.add_argument( "--reveal", help= "reveal refs (warning: this will potentially write sensitive data)", action="store_true", default=from_dot_kapitan("compile", "reveal", False), ) compile_parser.add_argument( "--inventory-path", default=from_dot_kapitan("compile", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) compile_parser.add_argument( "--cache", "-c", help="enable compilation caching to .kapitan_cache, default is False", action="store_true", default=from_dot_kapitan("compile", "cache", False), ) compile_parser.add_argument( "--cache-paths", type=str, nargs="+", default=from_dot_kapitan("compile", "cache-paths", []), metavar="PATH", help="cache additional paths to .kapitan_cache, default is []", ) compile_parser.add_argument( "--ignore-version-check", help="ignore the version from .kapitan", action="store_true", default=from_dot_kapitan("compile", "ignore-version-check", False), ) compile_parser.add_argument( "--schemas-path", default=from_dot_kapitan("validate", "schemas-path", "./schemas"), help='set schema cache path, default is "./schemas"', ) compile_selector_parser = compile_parser.add_mutually_exclusive_group() compile_selector_parser.add_argument( "--targets", "-t", help="targets to compile, default is all", type=str, nargs="+", default=from_dot_kapitan("compile", "targets", []), metavar="TARGET", ) compile_selector_parser.add_argument( "--labels", "-l", help="compile targets matching the labels, default is all", type=str, nargs="*", default=from_dot_kapitan("compile", "labels", []), metavar="key=value", ) inventory_parser = subparser.add_parser("inventory", help="show inventory") inventory_parser.add_argument( "--target-name", "-t", default=from_dot_kapitan("inventory", "target-name", ""), help="set target name, default is all targets", ) inventory_parser.add_argument( "--inventory-path", default=from_dot_kapitan("inventory", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) inventory_parser.add_argument( "--flat", "-F", help="flatten nested inventory variables", action="store_true", default=from_dot_kapitan("inventory", "flat", False), ) inventory_parser.add_argument( "--pattern", "-p", default=from_dot_kapitan("inventory", "pattern", ""), help= "filter pattern (e.g. parameters.mysql.storage_class, or storage_class," + ' or storage_*), default is ""', ) inventory_parser.add_argument( "--verbose", "-v", help="set verbose mode", action="store_true", default=from_dot_kapitan("inventory", "verbose", False), ) searchvar_parser = subparser.add_parser( "searchvar", help="show all inventory files where var is declared") searchvar_parser.add_argument( "searchvar", type=str, metavar="VARNAME", help= "e.g. parameters.mysql.storage_class, or storage_class, or storage_*", ) searchvar_parser.add_argument( "--inventory-path", default=from_dot_kapitan("searchvar", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) searchvar_parser.add_argument( "--verbose", "-v", help="set verbose mode", action="store_true", default=from_dot_kapitan("searchvar", "verbose", False), ) searchvar_parser.add_argument( "--pretty-print", "-p", help="Pretty print content of var", action="store_true", default=from_dot_kapitan("searchvar", "pretty-print", False), ) subparser.add_parser("secrets", help="(DEPRECATED) please use refs") refs_parser = subparser.add_parser("refs", help="manage refs") refs_parser.add_argument( "--write", "-w", help="write ref token", metavar="TOKENNAME", ) refs_parser.add_argument( "--update", help="update GPG recipients for ref token", metavar="TOKENNAME", ) refs_parser.add_argument( "--update-targets", action="store_true", default=from_dot_kapitan("refs", "update-targets", False), help="update target secret refs", ) refs_parser.add_argument( "--validate-targets", action="store_true", default=from_dot_kapitan("refs", "validate-targets", False), help="validate target secret refs", ) refs_parser.add_argument( "--base64", "-b64", help="base64 encode file content", action="store_true", default=from_dot_kapitan("refs", "base64", False), ) refs_parser.add_argument( "--reveal", "-r", help="reveal refs", action="store_true", default=from_dot_kapitan("refs", "reveal", False), ) refs_parser.add_argument("--file", "-f", help='read file or directory, set "-" for stdin', metavar="FILENAME") refs_parser.add_argument("--target-name", "-t", help="grab recipients from target name") refs_parser.add_argument( "--inventory-path", default=from_dot_kapitan("refs", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) refs_parser.add_argument( "--recipients", "-R", help="set GPG recipients", type=str, nargs="+", default=from_dot_kapitan("refs", "recipients", []), metavar="RECIPIENT", ) refs_parser.add_argument("--key", "-K", help="set KMS key", default=from_dot_kapitan("refs", "key", ""), metavar="KEY") refs_parser.add_argument( "--vault-auth", help="set authentication type for vaultkv secrets", default=from_dot_kapitan("refs", "vault-auth", ""), metavar="AUTH", ) refs_parser.add_argument( "--refs-path", help='set refs path, default is "./refs"', default=from_dot_kapitan("refs", "refs-path", "./refs"), ) refs_parser.add_argument( "--verbose", "-v", help= "set verbose mode (warning: this will potentially show sensitive data)", action="store_true", default=from_dot_kapitan("refs", "verbose", False), ) lint_parser = subparser.add_parser("lint", help="linter for inventory and refs") lint_parser.add_argument( "--fail-on-warning", default=from_dot_kapitan("lint", "fail-on-warning", False), action="store_true", help="exit with failure code if warnings exist, default is False", ) lint_parser.add_argument( "--skip-class-checks", action="store_true", help="skip checking for unused classes, default is False", default=from_dot_kapitan("lint", "skip-class-checks", False), ) lint_parser.add_argument( "--skip-yamllint", action="store_true", help="skip running yamllint on inventory, default is False", default=from_dot_kapitan("lint", "skip-yamllint", False), ) lint_parser.add_argument( "--search-secrets", default=from_dot_kapitan("lint", "search-secrets", False), action="store_true", help="searches for plaintext secrets in inventory, default is False", ) lint_parser.add_argument( "--refs-path", help='set refs path, default is "./refs"', default=from_dot_kapitan("lint", "refs-path", "./refs"), ) lint_parser.add_argument( "--compiled-path", default=from_dot_kapitan("lint", "compiled-path", "./compiled"), help='set compiled path, default is "./compiled"', ) lint_parser.add_argument( "--inventory-path", default=from_dot_kapitan("lint", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) init_parser = subparser.add_parser( "init", help= "initialize a directory with the recommended kapitan project skeleton." ) init_parser.add_argument( "--directory", default=from_dot_kapitan("init", "directory", "."), help="set path, in which to generate the project skeleton," 'assumes directory already exists. default is "./"', ) validate_parser = subparser.add_parser( "validate", help= "validates the compile output against schemas as specified in inventory" ) validate_parser.add_argument( "--compiled-path", default=from_dot_kapitan("compile", "compiled-path", "./compiled"), help='set compiled path, default is "./compiled', ) validate_parser.add_argument( "--inventory-path", default=from_dot_kapitan("compile", "inventory-path", "./inventory"), help='set inventory path, default is "./inventory"', ) validate_parser.add_argument( "--targets", "-t", help="targets to validate, default is all", type=str, nargs="+", default=from_dot_kapitan("compile", "targets", []), metavar="TARGET", ), validate_parser.add_argument( "--schemas-path", default=from_dot_kapitan("validate", "schemas-path", "./schemas"), help='set schema cache path, default is "./schemas"', ) validate_parser.add_argument( "--parallelism", "-p", type=int, default=from_dot_kapitan("validate", "parallelism", 4), metavar="INT", help="Number of concurrent validate processes, default is 4", ) args = parser.parse_args() logger.debug("Running with args: %s", args) try: cmd = sys.argv[1] except IndexError: parser.print_help() sys.exit(1) # cache args where key is subcommand cached.args[sys.argv[1]] = args if hasattr(args, "verbose") and args.verbose: logging.basicConfig( level=logging.DEBUG, format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s") elif hasattr(args, "quiet") and args.quiet: logging.basicConfig(level=logging.CRITICAL, format="%(message)s") else: logging.basicConfig(level=logging.INFO, format="%(message)s") if cmd == "eval": file_path = args.jsonnet_file search_paths = [os.path.abspath(path) for path in args.search_paths] ext_vars = {} if args.vars: ext_vars = dict(var.split("=") for var in args.vars) json_output = None def _search_imports(cwd, imp): return search_imports(cwd, imp, search_paths) json_output = jsonnet_file( file_path, import_callback=_search_imports, native_callbacks=resource_callbacks(search_paths), ext_vars=ext_vars, ) if args.output == "yaml": json_obj = json.loads(json_output) yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False) elif json_output: print(json_output) elif cmd == "compile": search_paths = [os.path.abspath(path) for path in args.search_paths] if not args.ignore_version_check: check_version() ref_controller = RefController(args.refs_path) # cache controller for use in reveal_maybe jinja2 filter cached.ref_controller_obj = ref_controller cached.revealer_obj = Revealer(ref_controller) compile_targets( args.inventory_path, search_paths, args.output_path, args.parallelism, args.targets, args.labels, ref_controller, prune=(args.prune), indent=args.indent, reveal=args.reveal, cache=args.cache, cache_paths=args.cache_paths, fetch_dependencies=args.fetch, validate=args.validate, schemas_path=args.schemas_path, jinja2_filters=args.jinja2_filters, ) elif cmd == "inventory": if args.pattern and args.target_name == "": parser.error("--pattern requires --target_name") try: inv = inventory_reclass(args.inventory_path) if args.target_name != "": inv = inv["nodes"][args.target_name] if args.pattern != "": pattern = args.pattern.split(".") inv = deep_get(inv, pattern) if args.flat: inv = flatten_dict(inv) yaml.dump(inv, sys.stdout, width=10000, default_flow_style=False) else: yaml.dump(inv, sys.stdout, Dumper=PrettyDumper, default_flow_style=False) except Exception as e: if not isinstance(e, KapitanError): logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") sys.exit(1) elif cmd == "searchvar": searchvar(args.searchvar, args.inventory_path, args.pretty_print) elif cmd == "lint": start_lint( args.fail_on_warning, args.skip_class_checks, args.skip_yamllint, args.inventory_path, args.search_secrets, args.refs_path, args.compiled_path, ) elif cmd == "init": initialise_skeleton(args.directory) elif cmd == "secrets": logger.error( "Secrets have been renamed to refs, please refer to: '$ kapitan refs --help'" ) sys.exit(1) elif cmd == "refs": ref_controller = RefController(args.refs_path) if args.write is not None: ref_write(args, ref_controller) elif args.reveal: ref_reveal(args, ref_controller) elif args.update: secret_update(args, ref_controller) elif args.update_targets or args.validate_targets: secret_update_validate(args, ref_controller) elif cmd == "validate": schema_validate_compiled( args.targets, inventory_path=args.inventory_path, compiled_path=args.compiled_path, schema_cache_path=args.schemas_path, parallel=args.parallelism, )
def main(): """main function for command line usage""" parser = argparse.ArgumentParser(prog=PROJECT_NAME, description=DESCRIPTION) parser.add_argument('--version', action='version', version=VERSION) subparser = parser.add_subparsers(help="commands") eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file') eval_parser.add_argument('jsonnet_file', type=str) eval_parser.add_argument('--output', type=str, choices=('yaml', 'json'), default=from_dot_kapitan('eval', 'output', 'yaml'), help='set output format, default is "yaml"') eval_parser.add_argument('--vars', type=str, default=from_dot_kapitan('eval', 'vars', []), nargs='*', metavar='VAR', help='set variables') eval_parser.add_argument('--search-paths', '-J', type=str, nargs='+', default=from_dot_kapitan('eval', 'search-paths', ['.']), metavar='JPATH', help='set search paths, default is ["."]') compile_parser = subparser.add_parser('compile', help='compile targets') compile_parser.add_argument('--search-paths', '-J', type=str, nargs='+', default=from_dot_kapitan( 'compile', 'search-paths', ['.', 'lib']), metavar='JPATH', help='set search paths, default is ["."]') compile_parser.add_argument( '--jinja2-filters', '-J2F', type=str, default=from_dot_kapitan('compile', 'jinja2-filters', default_jinja2_filters_path), metavar='FPATH', help='load custom jinja2 filters from any file, default is to put\ them inside lib/jinja2_filters.py') compile_parser.add_argument('--verbose', '-v', help='set verbose mode', action='store_true', default=from_dot_kapitan( 'compile', 'verbose', False)) compile_parser.add_argument('--prune', help='prune jsonnet output', action='store_true', default=from_dot_kapitan( 'compile', 'prune', False)) compile_parser.add_argument('--quiet', help='set quiet mode, only critical output', action='store_true', default=from_dot_kapitan( 'compile', 'quiet', False)) compile_parser.add_argument('--output-path', type=str, default=from_dot_kapitan( 'compile', 'output-path', '.'), metavar='PATH', help='set output path, default is "."') compile_parser.add_argument('--fetch', help='fetches external dependencies', action='store_true', default=from_dot_kapitan( 'compile', 'fetch', False)) compile_parser.add_argument( '--validate', help= 'validate compile output against schemas as specified in inventory', action='store_true', default=from_dot_kapitan('compile', 'validate', False)) compile_parser.add_argument('--targets', '-t', help='targets to compile, default is all', type=str, nargs='+', default=from_dot_kapitan( 'compile', 'targets', []), metavar='TARGET') compile_parser.add_argument( '--parallelism', '-p', type=int, default=from_dot_kapitan('compile', 'parallelism', 4), metavar='INT', help='Number of concurrent compile processes, default is 4') compile_parser.add_argument( '--indent', '-i', type=int, default=from_dot_kapitan('compile', 'indent', 2), metavar='INT', help='Indentation spaces for YAML/JSON, default is 2') compile_parser.add_argument( '--secrets-path', help='set secrets path, default is "./secrets"', default=from_dot_kapitan('compile', 'secrets-path', './secrets')) compile_parser.add_argument( '--reveal', help='reveal secrets (warning: this will write sensitive data)', action='store_true', default=from_dot_kapitan('compile', 'reveal', False)) compile_parser.add_argument( '--inventory-path', default=from_dot_kapitan('compile', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') compile_parser.add_argument( '--cache', '-c', help='enable compilation caching to .kapitan_cache, default is False', action='store_true', default=from_dot_kapitan('compile', 'cache', False)) compile_parser.add_argument( '--cache-paths', type=str, nargs='+', default=from_dot_kapitan('compile', 'cache-paths', []), metavar='PATH', help='cache additional paths to .kapitan_cache, default is []') compile_parser.add_argument('--ignore-version-check', help='ignore the version from .kapitan', action='store_true', default=from_dot_kapitan( 'compile', 'ignore-version-check', False)) compile_parser.add_argument( '--schemas-path', default=from_dot_kapitan('validate', 'schemas-path', './schemas'), help='set schema cache path, default is "./schemas"') inventory_parser = subparser.add_parser('inventory', help='show inventory') inventory_parser.add_argument( '--target-name', '-t', default=from_dot_kapitan('inventory', 'target-name', ''), help='set target name, default is all targets') inventory_parser.add_argument( '--inventory-path', default=from_dot_kapitan('inventory', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') inventory_parser.add_argument('--flat', '-F', help='flatten nested inventory variables', action='store_true', default=from_dot_kapitan( 'inventory', 'flat', False)) inventory_parser.add_argument( '--pattern', '-p', default=from_dot_kapitan('inventory', 'pattern', ''), help= 'filter pattern (e.g. parameters.mysql.storage_class, or storage_class,' + ' or storage_*), default is ""') inventory_parser.add_argument('--verbose', '-v', help='set verbose mode', action='store_true', default=from_dot_kapitan( 'inventory', 'verbose', False)) searchvar_parser = subparser.add_parser( 'searchvar', help='show all inventory files where var is declared') searchvar_parser.add_argument( 'searchvar', type=str, metavar='VARNAME', help= 'e.g. parameters.mysql.storage_class, or storage_class, or storage_*') searchvar_parser.add_argument( '--inventory-path', default=from_dot_kapitan('searchvar', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') searchvar_parser.add_argument('--verbose', '-v', help='set verbose mode', action='store_true', default=from_dot_kapitan( 'searchvar', 'verbose', False)) searchvar_parser.add_argument('--pretty-print', '-p', help='Pretty print content of var', action='store_true', default=from_dot_kapitan( 'searchvar', 'pretty-print', False)) secrets_parser = subparser.add_parser('secrets', help='manage secrets') secrets_parser.add_argument( '--write', '-w', help='write secret token', metavar='TOKENNAME', ) secrets_parser.add_argument( '--update', help='update recipients for secret token', metavar='TOKENNAME', ) secrets_parser.add_argument('--update-targets', action='store_true', default=from_dot_kapitan( 'secrets', 'update-targets', False), help='update target secrets') secrets_parser.add_argument('--validate-targets', action='store_true', default=from_dot_kapitan( 'secrets', 'validate-targets', False), help='validate target secrets') secrets_parser.add_argument('--base64', '-b64', help='base64 encode file content', action='store_true', default=from_dot_kapitan( 'secrets', 'base64', False)) secrets_parser.add_argument('--reveal', '-r', help='reveal secrets', action='store_true', default=from_dot_kapitan( 'secrets', 'reveal', False)) secrets_parser.add_argument( '--file', '-f', help='read file or directory, set "-" for stdin', metavar='FILENAME') secrets_parser.add_argument('--target-name', '-t', help='grab recipients from target name') secrets_parser.add_argument( '--inventory-path', default=from_dot_kapitan('secrets', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') secrets_parser.add_argument('--recipients', '-R', help='set GPG recipients', type=str, nargs='+', default=from_dot_kapitan( 'secrets', 'recipients', []), metavar='RECIPIENT') secrets_parser.add_argument('--key', '-K', help='set KMS key', default=from_dot_kapitan('secrets', 'key', ''), metavar='KEY') secrets_parser.add_argument( '--secrets-path', help='set secrets path, default is "./secrets"', default=from_dot_kapitan('secrets', 'secrets-path', './secrets')) secrets_parser.add_argument( '--verbose', '-v', help='set verbose mode (warning: this will show sensitive data)', action='store_true', default=from_dot_kapitan('secrets', 'verbose', False)) lint_parser = subparser.add_parser('lint', help='linter for inventory and secrets') lint_parser.add_argument( '--fail-on-warning', default=from_dot_kapitan('lint', 'fail-on-warning', False), action='store_true', help='exit with failure code if warnings exist, default is False') lint_parser.add_argument( '--skip-class-checks', action='store_true', help='skip checking for unused classes, default is False', default=from_dot_kapitan('lint', 'skip-class-checks', False)) lint_parser.add_argument( '--skip-yamllint', action='store_true', help='skip running yamllint on inventory, default is False', default=from_dot_kapitan('lint', 'skip-yamllint', False)) lint_parser.add_argument( '--search-secrets', default=from_dot_kapitan('lint', 'search-secrets', False), action='store_true', help='searches for plaintext secrets in inventory, default is False') lint_parser.add_argument('--secrets-path', help='set secrets path, default is "./secrets"', default=from_dot_kapitan('lint', 'secrets-path', './secrets')) lint_parser.add_argument('--compiled-path', default=from_dot_kapitan('lint', 'compiled-path', './compiled'), help='set compiled path, default is "./compiled"') lint_parser.add_argument( '--inventory-path', default=from_dot_kapitan('lint', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') init_parser = subparser.add_parser( 'init', help= 'initialize a directory with the recommended kapitan project skeleton.' ) init_parser.add_argument( '--directory', default=from_dot_kapitan('init', 'directory', '.'), help= 'set path, in which to generate the project skeleton, assumes directory already exists. default is "./"' ) validate_parser = subparser.add_parser( 'validate', help= 'validates the compile output against schemas as specified in inventory' ) validate_parser.add_argument( '--compiled-path', default=from_dot_kapitan('compile', 'compiled-path', './compiled'), help='set compiled path, default is "./compiled') validate_parser.add_argument( '--inventory-path', default=from_dot_kapitan('compile', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') validate_parser.add_argument('--targets', '-t', help='targets to validate, default is all', type=str, nargs='+', default=from_dot_kapitan( 'compile', 'targets', []), metavar='TARGET'), validate_parser.add_argument( '--schemas-path', default=from_dot_kapitan('validate', 'schemas-path', './schemas'), help='set schema cache path, default is "./schemas"') validate_parser.add_argument( '--parallelism', '-p', type=int, default=from_dot_kapitan('validate', 'parallelism', 4), metavar='INT', help='Number of concurrent validate processes, default is 4') args = parser.parse_args() logger.debug('Running with args: %s', args) try: cmd = sys.argv[1] except IndexError: parser.print_help() sys.exit(1) if hasattr(args, 'verbose') and args.verbose: logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') elif hasattr(args, 'quiet') and args.quiet: logging.basicConfig(level=logging.CRITICAL, format="%(message)s") else: logging.basicConfig(level=logging.INFO, format="%(message)s") if cmd == 'eval': file_path = args.jsonnet_file search_paths = [os.path.abspath(path) for path in args.search_paths] ext_vars = {} if args.vars: ext_vars = dict(var.split('=') for var in args.vars) json_output = None def _search_imports(cwd, imp): return search_imports(cwd, imp, search_paths) json_output = jsonnet_file( file_path, import_callback=_search_imports, native_callbacks=resource_callbacks(search_paths), ext_vars=ext_vars) if args.output == 'yaml': json_obj = json.loads(json_output) yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False) elif json_output: print(json_output) elif cmd == 'compile': search_paths = [os.path.abspath(path) for path in args.search_paths] if not args.ignore_version_check: check_version() ref_controller = RefController(args.secrets_path) compile_targets(args.inventory_path, search_paths, args.output_path, args.parallelism, args.targets, ref_controller, prune=(args.prune), indent=args.indent, reveal=args.reveal, cache=args.cache, cache_paths=args.cache_paths, fetch_dependencies=args.fetch, validate=args.validate, schemas_path=args.schemas_path, jinja2_filters=args.jinja2_filters) elif cmd == 'inventory': if args.pattern and args.target_name == '': parser.error("--pattern requires --target_name") try: inv = inventory_reclass(args.inventory_path) if args.target_name != '': inv = inv['nodes'][args.target_name] if args.pattern != '': pattern = args.pattern.split(".") inv = deep_get(inv, pattern) if args.flat: inv = flatten_dict(inv) yaml.dump(inv, sys.stdout, width=10000, default_flow_style=False) else: yaml.dump(inv, sys.stdout, Dumper=PrettyDumper, default_flow_style=False) except Exception as e: if not isinstance(e, KapitanError): logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") sys.exit(1) elif cmd == 'searchvar': searchvar(args.searchvar, args.inventory_path, args.pretty_print) elif cmd == 'lint': start_lint(args.fail_on_warning, args.skip_class_checks, args.skip_yamllint, args.inventory_path, args.search_secrets, args.secrets_path, args.compiled_path) elif cmd == 'init': initialise_skeleton(args.directory) elif cmd == 'secrets': ref_controller = RefController(args.secrets_path) if args.write is not None: secret_write(args, ref_controller) elif args.reveal: secret_reveal(args, ref_controller) elif args.update: secret_update(args, ref_controller) elif args.update_targets or args.validate_targets: secret_update_validate(args, ref_controller) elif cmd == 'validate': schema_validate_compiled(args.targets, inventory_path=args.inventory_path, compiled_path=args.compiled_path, schema_cache_path=args.schemas_path, parallel=args.parallelism)
from kapitan.refs.base import RefController, Revealer REF_CONTROLLER = RefController("/tmp", embed_refs=True) REVEALER = Revealer(REF_CONTROLLER)
"Azure secrets test" import os import tempfile import unittest import io import sys import contextlib from kapitan import cached from kapitan.cli import main from kapitan.refs.base import RefController, RefParams, Revealer from kapitan.refs.secrets.azkms import AzureKMSSecret, AzureKMSError REFS_HOME = tempfile.mkdtemp() REF_CONTROLLER = RefController(REFS_HOME) REVEALER = Revealer(REF_CONTROLLER) REF_CONTROLLER_EMBEDDED = RefController(REFS_HOME, embed_refs=True) REVEALER_EMBEDDED = Revealer(REF_CONTROLLER_EMBEDDED) class AzureKMSTest(unittest.TestCase): "Test Azure key vault secrets" def test_azkms_write_reveal(self): """ Write secret, confirm secret file exists, reveal and compare content """ tag = "?{azkms:secret/test}" REF_CONTROLLER[tag] = AzureKMSSecret("mock", "mock") self.assertTrue(os.path.isfile(os.path.join(REFS_HOME, "secret/test")))
def main(): """main function for command line usage""" parser = argparse.ArgumentParser(prog=PROJECT_NAME, description=DESCRIPTION) parser.add_argument('--version', action='version', version=VERSION) subparser = parser.add_subparsers(help="commands") eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file') eval_parser.add_argument('jsonnet_file', type=str) eval_parser.add_argument('--output', type=str, choices=('yaml', 'json', 'str'), default=from_dot_kapitan('eval', 'output', 'yaml'), help='set output format, default is "yaml"') eval_parser.add_argument('--vars', type=str, default=from_dot_kapitan('eval', 'vars', []), nargs='*', metavar='VAR', help='set variables') eval_parser.add_argument('--search-paths', '-J', type=str, nargs='+', default=from_dot_kapitan('eval', 'search-paths', ['.']), metavar='JPATH', help='set search paths, default is ["."]') compile_parser = subparser.add_parser('compile', help='compile targets') compile_parser.add_argument('--search-paths', '-J', type=str, nargs='+', default=from_dot_kapitan( 'compile', 'search-paths', ['.']), metavar='JPATH', help='set search paths, default is ["."]') compile_parser.add_argument('--verbose', '-v', help='set verbose mode', action='store_true', default=from_dot_kapitan( 'compile', 'verbose', False)) compile_parser.add_argument('--prune', help='prune jsonnet output', action='store_true', default=from_dot_kapitan( 'compile', 'prune', False)) compile_parser.add_argument('--quiet', help='set quiet mode, only critical output', action='store_true', default=from_dot_kapitan( 'compile', 'quiet', False)) compile_parser.add_argument('--output-path', type=str, default=from_dot_kapitan( 'compile', 'output-path', '.'), metavar='PATH', help='set output path, default is "."') compile_parser.add_argument('--targets', '-t', help='targets to compile, default is all', type=str, nargs='+', default=from_dot_kapitan( 'compile', 'targets', []), metavar='TARGET') compile_parser.add_argument( '--parallelism', '-p', type=int, default=from_dot_kapitan('compile', 'parallelism', 4), metavar='INT', help='Number of concurrent compile processes, default is 4') compile_parser.add_argument( '--indent', '-i', type=int, default=from_dot_kapitan('compile', 'indent', 2), metavar='INT', help='Indentation spaces for YAML/JSON, default is 2') compile_parser.add_argument( '--secrets-path', help='set secrets path, default is "./secrets"', default=from_dot_kapitan('compile', 'secrets-path', './secrets')) compile_parser.add_argument( '--reveal', help='reveal secrets (warning: this will write sensitive data)', action='store_true', default=from_dot_kapitan('compile', 'reveal', False)) compile_parser.add_argument( '--inventory-path', default=from_dot_kapitan('compile', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') compile_parser.add_argument( '--cache', '-c', help='enable compilation caching to .kapitan_cache, default is False', action='store_true', default=from_dot_kapitan('compile', 'cache', False)) compile_parser.add_argument( '--cache-paths', type=str, nargs='+', default=from_dot_kapitan('compile', 'cache-paths', []), metavar='PATH', help='cache additional paths to .kapitan_cache, default is []') compile_parser.add_argument('--ignore-version-check', help='ignore the version from .kapitan', action='store_true', default=from_dot_kapitan( 'compile', 'ignore-version-check', False)) inventory_parser = subparser.add_parser('inventory', help='show inventory') inventory_parser.add_argument( '--target-name', '-t', default=from_dot_kapitan('inventory', 'target-name', ''), help='set target name, default is all targets') inventory_parser.add_argument( '--inventory-path', default=from_dot_kapitan('inventory', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') inventory_parser.add_argument('--flat', '-F', help='flatten nested inventory variables', action='store_true', default=from_dot_kapitan( 'inventory', 'flat', False)) inventory_parser.add_argument( '--pattern', '-p', default=from_dot_kapitan('inventory', 'pattern', ''), help= 'filter pattern (e.g. parameters.mysql.storage_class, or storage_class,' + ' or storage_*), default is ""') inventory_parser.add_argument('--verbose', '-v', help='set verbose mode', action='store_true', default=from_dot_kapitan( 'inventory', 'verbose', False)) searchvar_parser = subparser.add_parser( 'searchvar', help='show all inventory files where var is declared') searchvar_parser.add_argument( 'searchvar', type=str, metavar='VARNAME', help= 'e.g. parameters.mysql.storage_class, or storage_class, or storage_*') searchvar_parser.add_argument( '--inventory-path', default=from_dot_kapitan('searchvar', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') searchvar_parser.add_argument('--verbose', '-v', help='set verbose mode', action='store_true', default=from_dot_kapitan( 'searchvar', 'verbose', False)) searchvar_parser.add_argument('--pretty-print', '-p', help='Pretty print content of var', action='store_true', default=from_dot_kapitan( 'searchvar', 'pretty-print', False)) secrets_parser = subparser.add_parser('secrets', help='manage secrets') secrets_parser.add_argument( '--write', '-w', help='write secret token', metavar='TOKENNAME', ) secrets_parser.add_argument( '--update', help='update recipients for secret token', metavar='TOKENNAME', ) secrets_parser.add_argument('--update-targets', action='store_true', default=from_dot_kapitan( 'secrets', 'update-targets', False), help='update target secrets') secrets_parser.add_argument('--validate-targets', action='store_true', default=from_dot_kapitan( 'secrets', 'validate-targets', False), help='validate target secrets') secrets_parser.add_argument('--base64', '-b64', help='base64 encode file content', action='store_true', default=from_dot_kapitan( 'secrets', 'base64', False)) secrets_parser.add_argument('--reveal', '-r', help='reveal secrets', action='store_true', default=from_dot_kapitan( 'secrets', 'reveal', False)) secrets_parser.add_argument( '--file', '-f', help='read file or directory, set "-" for stdin', metavar='FILENAME') secrets_parser.add_argument('--target-name', '-t', help='grab recipients from target name') secrets_parser.add_argument( '--inventory-path', default=from_dot_kapitan('secrets', 'inventory-path', './inventory'), help='set inventory path, default is "./inventory"') secrets_parser.add_argument('--recipients', '-R', help='set recipients', type=str, nargs='+', default=from_dot_kapitan( 'secrets', 'recipients', []), metavar='RECIPIENT') secrets_parser.add_argument( '--secrets-path', help='set secrets path, default is "./secrets"', default=from_dot_kapitan('secrets', 'secrets-path', './secrets')) secrets_parser.add_argument('--backend', help='set secrets backend, default is "gpg"', type=str, choices=('gpg', ), default=from_dot_kapitan( 'secrets', 'backend', 'gpg')) secrets_parser.add_argument( '--verbose', '-v', help='set verbose mode (warning: this will show sensitive data)', action='store_true', default=from_dot_kapitan('secrets', 'verbose', False)) args = parser.parse_args() logger.debug('Running with args: %s', args) try: cmd = sys.argv[1] except IndexError: parser.print_help() sys.exit(1) if cmd == 'eval': file_path = args.jsonnet_file search_paths = [os.path.abspath(path) for path in args.search_paths] ext_vars = {} if args.vars: ext_vars = dict(var.split('=') for var in args.vars) json_output = None def _search_imports(cwd, imp): return search_imports(cwd, imp, search_paths) json_output = jsonnet_file( file_path, import_callback=_search_imports, native_callbacks=resource_callbacks(search_paths), ext_vars=ext_vars) if args.output == 'yaml': json_obj = json.loads(json_output) yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False) elif json_output: print(json_output) elif cmd == 'compile': if args.verbose: logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') elif args.quiet: logging.basicConfig(level=logging.CRITICAL, format="%(message)s") else: logging.basicConfig(level=logging.INFO, format="%(message)s") search_paths = [os.path.abspath(path) for path in args.search_paths] if not args.ignore_version_check: check_version() ref_controller = RefController(args.secrets_path) compile_targets(args.inventory_path, search_paths, args.output_path, args.parallelism, args.targets, ref_controller, prune=(args.prune), indent=args.indent, reveal=args.reveal, cache=args.cache, cache_paths=args.cache_paths) elif cmd == 'inventory': if args.verbose: logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') else: logging.basicConfig(level=logging.INFO, format="%(message)s") if args.pattern and args.target_name == '': parser.error("--pattern requires --target_name") try: inv = inventory_reclass(args.inventory_path) if args.target_name != '': inv = inv['nodes'][args.target_name] if args.pattern != '': pattern = args.pattern.split(".") inv = deep_get(inv, pattern) if args.flat: inv = flatten_dict(inv) yaml.dump(inv, sys.stdout, width=10000) else: yaml.dump(inv, sys.stdout, Dumper=PrettyDumper, default_flow_style=False) except Exception as e: if not isinstance(e, KapitanError): logger.error("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~") traceback.print_exc() sys.exit(1) elif cmd == 'searchvar': if args.verbose: logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') else: logging.basicConfig(level=logging.INFO, format="%(message)s") searchvar(args.searchvar, args.inventory_path, args.pretty_print) elif cmd == 'secrets': if args.verbose: logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s') else: logging.basicConfig(level=logging.INFO, format="%(message)s") ref_controller = RefController(args.secrets_path) if args.write is not None: if args.file is None: parser.error('--file is required with --write') data = None recipients = [dict((("name", name), )) for name in args.recipients] if args.target_name: inv = inventory_reclass(args.inventory_path) # TODO move into kapitan:secrets:gpg:recipients key recipients = inv['nodes'][args.target_name]['parameters'][ 'kapitan']['secrets']['recipients'] if args.file == '-': data = '' for line in sys.stdin: data += line else: with open(args.file) as fp: data = fp.read() # TODO deprecate backend and move to passing ref tags in command line if args.backend == "gpg": secret_obj = GPGSecret(data, recipients, args.base64) ref_controller.backends['gpg'][args.write] = secret_obj elif args.reveal: revealer = Revealer(ref_controller) if args.file is None: parser.error('--file is required with --reveal') if args.file == '-': # TODO deal with RefHashMismatchError or KeyError exceptions out = revealer.reveal_raw_file(None) sys.stdout.write(out) elif args.file: for rev_obj in revealer.reveal_path(args.file): sys.stdout.write(rev_obj.content) elif args.update: # update recipients for secret tag # args.recipients is a list, convert to recipients dict recipients = [ dict([ ("name", name), ]) for name in args.recipients ] if args.target_name: inv = inventory_reclass(args.inventory_path) # TODO move into kapitan:secrets:gpg:recipients key recipients = inv['nodes'][args.target_name]['parameters'][ 'kapitan']['secrets']['recipients'] if args.backend == "gpg": secret_obj = ref_controller.backends['gpg'][args.update] secret_obj.update_recipients(recipients) ref_controller.backends['gpg'][args.update] = secret_obj elif args.update_targets or args.validate_targets: # update recipients for all secrets in secrets_path # use --secrets-path to set scanning path inv = inventory_reclass(args.inventory_path) targets = set(inv['nodes'].keys()) secrets_path = os.path.abspath(args.secrets_path) target_token_paths = search_target_token_paths( secrets_path, targets) ret_code = 0 # override gpg backend for new secrets_path ref_controller.register_backend(GPGBackend(secrets_path)) for target_name, token_paths in target_token_paths.items(): try: recipients = inv['nodes'][target_name]['parameters'][ 'kapitan']['secrets']['recipients'] for token_path in token_paths: secret_obj = ref_controller.backends['gpg'][token_path] target_fingerprints = set( lookup_fingerprints(recipients)) secret_fingerprints = set( lookup_fingerprints(secret_obj.recipients)) if target_fingerprints != secret_fingerprints: if args.validate_targets: logger.info("%s recipient mismatch", token_path) ret_code = 1 else: new_recipients = [ dict([ ("fingerprint", f), ]) for f in target_fingerprints ] secret_obj.update_recipients(new_recipients) ref_controller.backends['gpg'][ token_path] = secret_obj except KeyError: logger.debug( "secret_gpg_update_target: target: %s has no inventory recipients, skipping", target_name) sys.exit(ret_code)
import ssl def setup_logging(level=logging.INFO): logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', level=level, datefmt='%Y-%m-%d %H:%M:%S') logging.getLogger('tesoro').setLevel(level) setup_logging() logger = logging.getLogger('tesoro') ROUTES = web.RouteTableDef() REF_CONTROLLER = RefController('/tmp', embed_refs=True) REVEALER = Revealer(REF_CONTROLLER) TESORO_COUNTER = Counter('tesoro_requests', 'Tesoro requests') TESORO_FAILED_COUNTER = Counter('tesoro_requests_failed', 'Tesoro failed requests') REVEAL_COUNTER = Counter('kapitan_reveal_requests', 'Kapitan reveal requests') REVEAL_FAILED_COUNTER = Counter('kapitan_reveal_requests_failed', 'Kapitan reveal failed requests ') @ROUTES.get('/healthz') async def healthz(request): return web.Response(status=200, text='ok')