def ref_reveal(args, ref_controller):
"Reveal secrets in file_name"
revealer = Revealer(ref_controller)
file_name = args.file
reffile_name = args.ref_file
tag_name = args.tag
if file_name is None and reffile_name is None and tag_name is None:
fatal_error("--file or --ref-file is required with --reveal")
try:
if file_name == "-" or reffile_name == "-":
out = revealer.reveal_raw_file(None)
sys.stdout.write(out)
elif file_name:
for rev_obj in revealer.reveal_path(file_name):
sys.stdout.write(rev_obj.content)
elif reffile_name:
ref = ref_controller.ref_from_ref_file(reffile_name)
sys.stdout.write(ref.reveal())
elif tag_name:
out = revealer.reveal_raw_string(tag_name)
sys.stdout.write(out)
except (RefHashMismatchError, KeyError):
raise KapitanError(
"Reveal failed for file {name}".format(name=file_name))
class CompilingFile(object):
def __init__(self, context, fp, ref_controller, **kwargs):
self.fp = fp
self.ref_controller = ref_controller
self.kwargs = kwargs
self.revealer = Revealer(ref_controller)
def write(self, data):
"""write data into file"""
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.fp.write(self.revealer.reveal_raw(data))
else:
self.fp.write(
self.revealer.compile_raw(data, target_name=target_name))
def write_yaml(self, obj):
"""recursively compile or reveal refs and convert obj to yaml and write to file"""
indent = self.kwargs.get('indent', 2)
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.revealer.reveal_obj(obj)
else:
self.revealer.compile_obj(obj, target_name=target_name)
if isinstance(obj, collections.Mapping):
yaml.dump(obj,
stream=self.fp,
indent=indent,
Dumper=PrettyDumper,
default_flow_style=False)
else:
yaml.dump_all(obj,
stream=self.fp,
indent=indent,
Dumper=PrettyDumper,
default_flow_style=False)
logger.debug("Wrote %s", self.fp.name)
def write_json(self, obj):
"""recursively hash or reveal refs and convert obj to json and write to file"""
indent = self.kwargs.get('indent', 2)
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.revealer.reveal_obj(obj)
else:
self.revealer.compile_obj(obj, target_name=target_name)
json.dump(obj, self.fp, indent=indent)
logger.debug("Wrote %s", self.fp.name)
def secret_reveal(args, ref_controller):
"Reveal secrets in file_name"
revealer = Revealer(ref_controller)
file_name = args.file
if file_name is None:
fatal_error('--file is required with --reveal')
if file_name == '-':
# TODO deal with RefHashMismatchError or KeyError exceptions
out = revealer.reveal_raw_file(None)
sys.stdout.write(out)
elif file_name:
for rev_obj in revealer.reveal_path(file_name):
sys.stdout.write(rev_obj.content)
def ref_reveal(args, ref_controller):
"Reveal secrets in file_name"
revealer = Revealer(ref_controller)
file_name = args.file
if file_name is None:
fatal_error('--file is required with --reveal')
try:
if file_name == '-':
out = revealer.reveal_raw_file(None)
sys.stdout.write(out)
elif file_name:
for rev_obj in revealer.reveal_path(file_name):
sys.stdout.write(rev_obj.content)
except (RefHashMismatchError, KeyError):
raise KapitanError("Reveal failed for file {name}".format(name=file_name))
def trigger_compile(args):
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.refs_path, embed_refs=args.embed_refs)
# cache controller for use in reveal_maybe jinja2 filter
cached.ref_controller_obj = ref_controller
cached.revealer_obj = Revealer(ref_controller)
compile_targets(
args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
args.labels,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths,
fetch_inventories=args.fetch,
fetch_dependencies=args.fetch,
force_fetch=args.force,
validate=args.validate,
schemas_path=args.schemas_path,
jinja2_filters=args.jinja2_filters,
verbose=hasattr(args, "verbose") and args.verbose,
)
def test_reveal_maybe_tag_no_reveal_flag(self):
"""
creates ?{base64:some_value} and runs reveal_maybe jinja2 filters without --reveal flag
"""
with tempfile.NamedTemporaryFile() as f:
f.write("{{ my_ref_tag_var|reveal_maybe }}".encode("UTF-8"))
f.seek(0)
# new argparse namespace with --reveal and --refs-path values
namespace = namedtuple("Namespace", [])
namespace.reveal = False
namespace.refs_path = tempfile.mkdtemp()
# reveal_maybe uses cached, so inject namespace
cached.args["compile"] = namespace
cached.ref_controller_obj = RefController(
cached.args["compile"].refs_path)
cached.revealer_obj = Revealer(cached.ref_controller_obj)
ref_tag = "?{base64:some_value}"
ref_value = b"sitar_rock!"
cached.ref_controller_obj[ref_tag] = Base64Ref(ref_value)
context = {"my_ref_tag_var": ref_tag}
self.assertEqual(render_jinja2_file(f.name, context),
"?{base64:some_value}")
def test_reveal_maybe_b64encode_tag(self):
"""
creates ?{base64:some_value} and runs reveal_maybe|b64encode jinja2 filters
"""
with tempfile.NamedTemporaryFile() as f:
f.write(
"{{ my_ref_tag_var|reveal_maybe|b64encode }}".encode("UTF-8"))
f.seek(0)
# new argparse namespace with --reveal and --refs-path values
namespace = namedtuple('Namespace', [])
namespace.reveal = True
namespace.refs_path = tempfile.mkdtemp()
# reveal_maybe uses cached, so inject namespace
cached.args['compile'] = namespace
cached.ref_controller_obj = RefController(
cached.args['compile'].refs_path)
cached.revealer_obj = Revealer(cached.ref_controller_obj)
ref_tag = '?{base64:some_value}'
ref_value = b'sitar_rock!'
cached.ref_controller_obj[ref_tag] = Base64Ref(ref_value)
context = {"my_ref_tag_var": ref_tag}
ref_value_b64 = base64.b64encode(ref_value).decode()
self.assertEqual(render_jinja2_file(f.name, context),
ref_value_b64)
def test_reveal_maybe_no_tag(self):
"""
runs reveal_maybe jinja2 filter on data without ref tags
"""
with tempfile.NamedTemporaryFile() as f:
f.write("{{ my_var|reveal_maybe }}".encode("UTF-8"))
f.seek(0)
# new argparse namespace with --reveal and --refs-path values
namespace = namedtuple("Namespace", [])
namespace.reveal = True
namespace.refs_path = tempfile.mkdtemp()
# reveal_maybe uses cached, so inject namespace
cached.args["compile"] = namespace
cached.ref_controller_obj = RefController(cached.args["compile"].refs_path)
cached.revealer_obj = Revealer(cached.ref_controller_obj)
var_value = "heavy_rock!"
context = {"my_var": var_value}
self.assertEqual(render_jinja2_file(f.name, context), var_value)
def __init__(self, context, fp, ref_controller, **kwargs):
self.fp = fp
self.ref_controller = ref_controller
self.kwargs = kwargs
self.revealer = Revealer(ref_controller)
os.environ["GNUPGHOME"] = GNUPGHOME
gpg_obj(gnupghome=GNUPGHOME)
KEY = cached.gpg_obj.gen_key(cached.gpg_obj.gen_key_input(key_type="RSA",
key_length=2048,
passphrase="testphrase"))
KEY2 = cached.gpg_obj.gen_key(cached.gpg_obj.gen_key_input(key_type="RSA",
key_length=2048,
passphrase="testphrase"))
GPG_TARGET_FINGERPRINTS["KEY"] = KEY.fingerprint
GPG_KWARGS["passphrase"] = "testphrase"
REFS_HOME = tempfile.mkdtemp()
REF_CONTROLLER = RefController(REFS_HOME)
REVEALER = Revealer(REF_CONTROLLER)
class GPGSecretsTest(unittest.TestCase):
"Test GPG secrets"
def test_gpg_write_reveal(self):
"write secret, confirm secret file exists, reveal and compare content"
tag = '?{gpg:secret/sauce}'
REF_CONTROLLER[tag] = GPGSecret("super secret value", [{'fingerprint': KEY.fingerprint}])
self.assertTrue(os.path.isfile(os.path.join(REFS_HOME, 'secret/sauce')))
file_with_secret_tags = tempfile.mktemp()
with open(file_with_secret_tags, 'w') as fp:
fp.write('I am a file with a ?{gpg:secret/sauce}')
revealed = REVEALER.reveal_raw_file(file_with_secret_tags)
def main():
"""main function for command line usage"""
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument("--version", action="version", version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser("eval", help="evaluate jsonnet file")
eval_parser.add_argument("jsonnet_file", type=str)
eval_parser.add_argument(
"--output",
type=str,
choices=("yaml", "json"),
default=from_dot_kapitan("eval", "output", "yaml"),
help='set output format, default is "yaml"',
)
eval_parser.add_argument(
"--vars",
type=str,
default=from_dot_kapitan("eval", "vars", []),
nargs="*",
metavar="VAR",
help="set variables",
)
eval_parser.add_argument(
"--search-paths",
"-J",
type=str,
nargs="+",
default=from_dot_kapitan("eval", "search-paths", ["."]),
metavar="JPATH",
help='set search paths, default is ["."]',
)
compile_parser = subparser.add_parser("compile", help="compile targets")
compile_parser.add_argument(
"--search-paths",
"-J",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "search-paths", [".", "lib"]),
metavar="JPATH",
help='set search paths, default is ["."]',
)
compile_parser.add_argument(
"--jinja2-filters",
"-J2F",
type=str,
default=from_dot_kapitan("compile", "jinja2-filters",
defaults.DEFAULT_JINJA2_FILTERS_PATH),
metavar="FPATH",
help="load custom jinja2 filters from any file, default is to put\
them inside lib/jinja2_filters.py",
)
compile_parser.add_argument(
"--verbose",
"-v",
help="set verbose mode",
action="store_true",
default=from_dot_kapitan("compile", "verbose", False),
)
compile_parser.add_argument(
"--prune",
help="prune jsonnet output",
action="store_true",
default=from_dot_kapitan("compile", "prune", False),
)
compile_parser.add_argument(
"--quiet",
help="set quiet mode, only critical output",
action="store_true",
default=from_dot_kapitan("compile", "quiet", False),
)
compile_parser.add_argument(
"--output-path",
type=str,
default=from_dot_kapitan("compile", "output-path", "."),
metavar="PATH",
help='set output path, default is "."',
)
compile_parser.add_argument(
"--fetch",
help="fetches external dependencies",
action="store_true",
default=from_dot_kapitan("compile", "fetch", False),
)
compile_parser.add_argument(
"--validate",
help=
"validate compile output against schemas as specified in inventory",
action="store_true",
default=from_dot_kapitan("compile", "validate", False),
)
compile_parser.add_argument(
"--parallelism",
"-p",
type=int,
default=from_dot_kapitan("compile", "parallelism", 4),
metavar="INT",
help="Number of concurrent compile processes, default is 4",
)
compile_parser.add_argument(
"--indent",
"-i",
type=int,
default=from_dot_kapitan("compile", "indent", 2),
metavar="INT",
help="Indentation spaces for YAML/JSON, default is 2",
)
compile_parser.add_argument(
"--refs-path",
help='set refs path, default is "./refs"',
default=from_dot_kapitan("compile", "refs-path", "./refs"),
)
compile_parser.add_argument(
"--reveal",
help=
"reveal refs (warning: this will potentially write sensitive data)",
action="store_true",
default=from_dot_kapitan("compile", "reveal", False),
)
compile_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("compile", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
compile_parser.add_argument(
"--cache",
"-c",
help="enable compilation caching to .kapitan_cache, default is False",
action="store_true",
default=from_dot_kapitan("compile", "cache", False),
)
compile_parser.add_argument(
"--cache-paths",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "cache-paths", []),
metavar="PATH",
help="cache additional paths to .kapitan_cache, default is []",
)
compile_parser.add_argument(
"--ignore-version-check",
help="ignore the version from .kapitan",
action="store_true",
default=from_dot_kapitan("compile", "ignore-version-check", False),
)
compile_parser.add_argument(
"--schemas-path",
default=from_dot_kapitan("validate", "schemas-path", "./schemas"),
help='set schema cache path, default is "./schemas"',
)
compile_selector_parser = compile_parser.add_mutually_exclusive_group()
compile_selector_parser.add_argument(
"--targets",
"-t",
help="targets to compile, default is all",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "targets", []),
metavar="TARGET",
)
compile_selector_parser.add_argument(
"--labels",
"-l",
help="compile targets matching the labels, default is all",
type=str,
nargs="*",
default=from_dot_kapitan("compile", "labels", []),
metavar="key=value",
)
inventory_parser = subparser.add_parser("inventory", help="show inventory")
inventory_parser.add_argument(
"--target-name",
"-t",
default=from_dot_kapitan("inventory", "target-name", ""),
help="set target name, default is all targets",
)
inventory_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("inventory", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
inventory_parser.add_argument(
"--flat",
"-F",
help="flatten nested inventory variables",
action="store_true",
default=from_dot_kapitan("inventory", "flat", False),
)
inventory_parser.add_argument(
"--pattern",
"-p",
default=from_dot_kapitan("inventory", "pattern", ""),
help=
"filter pattern (e.g. parameters.mysql.storage_class, or storage_class,"
+ ' or storage_*), default is ""',
)
inventory_parser.add_argument(
"--verbose",
"-v",
help="set verbose mode",
action="store_true",
default=from_dot_kapitan("inventory", "verbose", False),
)
searchvar_parser = subparser.add_parser(
"searchvar", help="show all inventory files where var is declared")
searchvar_parser.add_argument(
"searchvar",
type=str,
metavar="VARNAME",
help=
"e.g. parameters.mysql.storage_class, or storage_class, or storage_*",
)
searchvar_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("searchvar", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
searchvar_parser.add_argument(
"--verbose",
"-v",
help="set verbose mode",
action="store_true",
default=from_dot_kapitan("searchvar", "verbose", False),
)
searchvar_parser.add_argument(
"--pretty-print",
"-p",
help="Pretty print content of var",
action="store_true",
default=from_dot_kapitan("searchvar", "pretty-print", False),
)
subparser.add_parser("secrets", help="(DEPRECATED) please use refs")
refs_parser = subparser.add_parser("refs", help="manage refs")
refs_parser.add_argument(
"--write",
"-w",
help="write ref token",
metavar="TOKENNAME",
)
refs_parser.add_argument(
"--update",
help="update GPG recipients for ref token",
metavar="TOKENNAME",
)
refs_parser.add_argument(
"--update-targets",
action="store_true",
default=from_dot_kapitan("refs", "update-targets", False),
help="update target secret refs",
)
refs_parser.add_argument(
"--validate-targets",
action="store_true",
default=from_dot_kapitan("refs", "validate-targets", False),
help="validate target secret refs",
)
refs_parser.add_argument(
"--base64",
"-b64",
help="base64 encode file content",
action="store_true",
default=from_dot_kapitan("refs", "base64", False),
)
refs_parser.add_argument(
"--reveal",
"-r",
help="reveal refs",
action="store_true",
default=from_dot_kapitan("refs", "reveal", False),
)
refs_parser.add_argument("--file",
"-f",
help='read file or directory, set "-" for stdin',
metavar="FILENAME")
refs_parser.add_argument("--target-name",
"-t",
help="grab recipients from target name")
refs_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("refs", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
refs_parser.add_argument(
"--recipients",
"-R",
help="set GPG recipients",
type=str,
nargs="+",
default=from_dot_kapitan("refs", "recipients", []),
metavar="RECIPIENT",
)
refs_parser.add_argument("--key",
"-K",
help="set KMS key",
default=from_dot_kapitan("refs", "key", ""),
metavar="KEY")
refs_parser.add_argument(
"--vault-auth",
help="set authentication type for vaultkv secrets",
default=from_dot_kapitan("refs", "vault-auth", ""),
metavar="AUTH",
)
refs_parser.add_argument(
"--refs-path",
help='set refs path, default is "./refs"',
default=from_dot_kapitan("refs", "refs-path", "./refs"),
)
refs_parser.add_argument(
"--verbose",
"-v",
help=
"set verbose mode (warning: this will potentially show sensitive data)",
action="store_true",
default=from_dot_kapitan("refs", "verbose", False),
)
lint_parser = subparser.add_parser("lint",
help="linter for inventory and refs")
lint_parser.add_argument(
"--fail-on-warning",
default=from_dot_kapitan("lint", "fail-on-warning", False),
action="store_true",
help="exit with failure code if warnings exist, default is False",
)
lint_parser.add_argument(
"--skip-class-checks",
action="store_true",
help="skip checking for unused classes, default is False",
default=from_dot_kapitan("lint", "skip-class-checks", False),
)
lint_parser.add_argument(
"--skip-yamllint",
action="store_true",
help="skip running yamllint on inventory, default is False",
default=from_dot_kapitan("lint", "skip-yamllint", False),
)
lint_parser.add_argument(
"--search-secrets",
default=from_dot_kapitan("lint", "search-secrets", False),
action="store_true",
help="searches for plaintext secrets in inventory, default is False",
)
lint_parser.add_argument(
"--refs-path",
help='set refs path, default is "./refs"',
default=from_dot_kapitan("lint", "refs-path", "./refs"),
)
lint_parser.add_argument(
"--compiled-path",
default=from_dot_kapitan("lint", "compiled-path", "./compiled"),
help='set compiled path, default is "./compiled"',
)
lint_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("lint", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
init_parser = subparser.add_parser(
"init",
help=
"initialize a directory with the recommended kapitan project skeleton."
)
init_parser.add_argument(
"--directory",
default=from_dot_kapitan("init", "directory", "."),
help="set path, in which to generate the project skeleton,"
'assumes directory already exists. default is "./"',
)
validate_parser = subparser.add_parser(
"validate",
help=
"validates the compile output against schemas as specified in inventory"
)
validate_parser.add_argument(
"--compiled-path",
default=from_dot_kapitan("compile", "compiled-path", "./compiled"),
help='set compiled path, default is "./compiled',
)
validate_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("compile", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
validate_parser.add_argument(
"--targets",
"-t",
help="targets to validate, default is all",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "targets", []),
metavar="TARGET",
),
validate_parser.add_argument(
"--schemas-path",
default=from_dot_kapitan("validate", "schemas-path", "./schemas"),
help='set schema cache path, default is "./schemas"',
)
validate_parser.add_argument(
"--parallelism",
"-p",
type=int,
default=from_dot_kapitan("validate", "parallelism", 4),
metavar="INT",
help="Number of concurrent validate processes, default is 4",
)
args = parser.parse_args()
logger.debug("Running with args: %s", args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
# cache args where key is subcommand
cached.args[sys.argv[1]] = args
if hasattr(args, "verbose") and args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
elif hasattr(args, "quiet") and args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
if cmd == "eval":
file_path = args.jsonnet_file
search_paths = [os.path.abspath(path) for path in args.search_paths]
ext_vars = {}
if args.vars:
ext_vars = dict(var.split("=") for var in args.vars)
json_output = None
def _search_imports(cwd, imp):
return search_imports(cwd, imp, search_paths)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_paths),
ext_vars=ext_vars,
)
if args.output == "yaml":
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == "compile":
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.refs_path)
# cache controller for use in reveal_maybe jinja2 filter
cached.ref_controller_obj = ref_controller
cached.revealer_obj = Revealer(ref_controller)
compile_targets(
args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
args.labels,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths,
fetch_dependencies=args.fetch,
validate=args.validate,
schemas_path=args.schemas_path,
jinja2_filters=args.jinja2_filters,
)
elif cmd == "inventory":
if args.pattern and args.target_name == "":
parser.error("--pattern requires --target_name")
try:
inv = inventory_reclass(args.inventory_path)
if args.target_name != "":
inv = inv["nodes"][args.target_name]
if args.pattern != "":
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv,
sys.stdout,
width=10000,
default_flow_style=False)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
sys.exit(1)
elif cmd == "searchvar":
searchvar(args.searchvar, args.inventory_path, args.pretty_print)
elif cmd == "lint":
start_lint(
args.fail_on_warning,
args.skip_class_checks,
args.skip_yamllint,
args.inventory_path,
args.search_secrets,
args.refs_path,
args.compiled_path,
)
elif cmd == "init":
initialise_skeleton(args.directory)
elif cmd == "secrets":
logger.error(
"Secrets have been renamed to refs, please refer to: '$ kapitan refs --help'"
)
sys.exit(1)
elif cmd == "refs":
ref_controller = RefController(args.refs_path)
if args.write is not None:
ref_write(args, ref_controller)
elif args.reveal:
ref_reveal(args, ref_controller)
elif args.update:
secret_update(args, ref_controller)
elif args.update_targets or args.validate_targets:
secret_update_validate(args, ref_controller)
elif cmd == "validate":
schema_validate_compiled(
args.targets,
inventory_path=args.inventory_path,
compiled_path=args.compiled_path,
schema_cache_path=args.schemas_path,
parallel=args.parallelism,
)
import os
import tempfile
import unittest
import io
import sys
import contextlib
from kapitan import cached
from kapitan.cli import main
from kapitan.refs.base import RefController, RefParams, Revealer
from kapitan.refs.secrets.azkms import AzureKMSSecret, AzureKMSError
REFS_HOME = tempfile.mkdtemp()
REF_CONTROLLER = RefController(REFS_HOME)
REVEALER = Revealer(REF_CONTROLLER)
REF_CONTROLLER_EMBEDDED = RefController(REFS_HOME, embed_refs=True)
REVEALER_EMBEDDED = Revealer(REF_CONTROLLER_EMBEDDED)
class AzureKMSTest(unittest.TestCase):
"Test Azure key vault secrets"
def test_azkms_write_reveal(self):
"""
Write secret, confirm secret file exists, reveal and compare content
"""
tag = "?{azkms:secret/test}"
REF_CONTROLLER[tag] = AzureKMSSecret("mock", "mock")
self.assertTrue(os.path.isfile(os.path.join(REFS_HOME, "secret/test")))
def main():
"""main function for command line usage"""
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument('--version', action='version', version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file')
eval_parser.add_argument('jsonnet_file', type=str)
eval_parser.add_argument('--output',
type=str,
choices=('yaml', 'json'),
default=from_dot_kapitan('eval', 'output',
'yaml'),
help='set output format, default is "yaml"')
eval_parser.add_argument('--vars',
type=str,
default=from_dot_kapitan('eval', 'vars', []),
nargs='*',
metavar='VAR',
help='set variables')
eval_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan('eval', 'search-paths',
['.']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser = subparser.add_parser('compile', help='compile targets')
compile_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'search-paths', ['.', 'lib']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser.add_argument(
'--jinja2-filters',
'-J2F',
type=str,
default=from_dot_kapitan('compile', 'jinja2-filters',
DEFAULT_JINJA2_FILTERS_PATH),
metavar='FPATH',
help='load custom jinja2 filters from any file, default is to put\
them inside lib/jinja2_filters.py')
compile_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'compile', 'verbose', False))
compile_parser.add_argument('--prune',
help='prune jsonnet output',
action='store_true',
default=from_dot_kapitan(
'compile', 'prune', False))
compile_parser.add_argument('--quiet',
help='set quiet mode, only critical output',
action='store_true',
default=from_dot_kapitan(
'compile', 'quiet', False))
compile_parser.add_argument('--output-path',
type=str,
default=from_dot_kapitan(
'compile', 'output-path', '.'),
metavar='PATH',
help='set output path, default is "."')
compile_parser.add_argument('--fetch',
help='fetches external dependencies',
action='store_true',
default=from_dot_kapitan(
'compile', 'fetch', False))
compile_parser.add_argument(
'--validate',
help=
'validate compile output against schemas as specified in inventory',
action='store_true',
default=from_dot_kapitan('compile', 'validate', False))
compile_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=from_dot_kapitan('compile', 'parallelism', 4),
metavar='INT',
help='Number of concurrent compile processes, default is 4')
compile_parser.add_argument(
'--indent',
'-i',
type=int,
default=from_dot_kapitan('compile', 'indent', 2),
metavar='INT',
help='Indentation spaces for YAML/JSON, default is 2')
compile_parser.add_argument('--refs-path',
help='set refs path, default is "./refs"',
default=from_dot_kapitan(
'compile', 'refs-path', './refs'))
compile_parser.add_argument(
'--reveal',
help=
'reveal refs (warning: this will potentially write sensitive data)',
action='store_true',
default=from_dot_kapitan('compile', 'reveal', False))
compile_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
compile_parser.add_argument(
'--cache',
'-c',
help='enable compilation caching to .kapitan_cache, default is False',
action='store_true',
default=from_dot_kapitan('compile', 'cache', False))
compile_parser.add_argument(
'--cache-paths',
type=str,
nargs='+',
default=from_dot_kapitan('compile', 'cache-paths', []),
metavar='PATH',
help='cache additional paths to .kapitan_cache, default is []')
compile_parser.add_argument('--ignore-version-check',
help='ignore the version from .kapitan',
action='store_true',
default=from_dot_kapitan(
'compile', 'ignore-version-check', False))
compile_parser.add_argument(
'--schemas-path',
default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
help='set schema cache path, default is "./schemas"')
compile_selector_parser = compile_parser.add_mutually_exclusive_group()
compile_selector_parser.add_argument(
'--targets',
'-t',
help='targets to compile, default is all',
type=str,
nargs='+',
default=from_dot_kapitan('compile', 'targets', []),
metavar='TARGET')
compile_selector_parser.add_argument(
'--labels',
'-l',
help='compile targets matching the labels, default is all',
type=str,
nargs='*',
default=from_dot_kapitan('compile', 'labels', []),
metavar='key=value')
inventory_parser = subparser.add_parser('inventory', help='show inventory')
inventory_parser.add_argument(
'--target-name',
'-t',
default=from_dot_kapitan('inventory', 'target-name', ''),
help='set target name, default is all targets')
inventory_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('inventory', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
inventory_parser.add_argument('--flat',
'-F',
help='flatten nested inventory variables',
action='store_true',
default=from_dot_kapitan(
'inventory', 'flat', False))
inventory_parser.add_argument(
'--pattern',
'-p',
default=from_dot_kapitan('inventory', 'pattern', ''),
help=
'filter pattern (e.g. parameters.mysql.storage_class, or storage_class,'
+ ' or storage_*), default is ""')
inventory_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'inventory', 'verbose', False))
searchvar_parser = subparser.add_parser(
'searchvar', help='show all inventory files where var is declared')
searchvar_parser.add_argument(
'searchvar',
type=str,
metavar='VARNAME',
help=
'e.g. parameters.mysql.storage_class, or storage_class, or storage_*')
searchvar_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('searchvar', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
searchvar_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'verbose', False))
searchvar_parser.add_argument('--pretty-print',
'-p',
help='Pretty print content of var',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'pretty-print', False))
subparser.add_parser('secrets', help='(DEPRECATED) please use refs')
refs_parser = subparser.add_parser('refs', help='manage refs')
refs_parser.add_argument(
'--write',
'-w',
help='write ref token',
metavar='TOKENNAME',
)
refs_parser.add_argument(
'--update',
help='update GPG recipients for ref token',
metavar='TOKENNAME',
)
refs_parser.add_argument('--update-targets',
action='store_true',
default=from_dot_kapitan('refs', 'update-targets',
False),
help='update target secret refs')
refs_parser.add_argument('--validate-targets',
action='store_true',
default=from_dot_kapitan('refs',
'validate-targets',
False),
help='validate target secret refs')
refs_parser.add_argument('--base64',
'-b64',
help='base64 encode file content',
action='store_true',
default=from_dot_kapitan('refs', 'base64', False))
refs_parser.add_argument('--reveal',
'-r',
help='reveal refs',
action='store_true',
default=from_dot_kapitan('refs', 'reveal', False))
refs_parser.add_argument('--file',
'-f',
help='read file or directory, set "-" for stdin',
metavar='FILENAME')
refs_parser.add_argument('--target-name',
'-t',
help='grab recipients from target name')
refs_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('refs', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
refs_parser.add_argument('--recipients',
'-R',
help='set GPG recipients',
type=str,
nargs='+',
default=from_dot_kapitan('refs', 'recipients',
[]),
metavar='RECIPIENT')
refs_parser.add_argument('--key',
'-K',
help='set KMS key',
default=from_dot_kapitan('refs', 'key', ''),
metavar='KEY')
refs_parser.add_argument(
'--vault-auth',
help='set authentication type for vaultkv secrets',
default=from_dot_kapitan('refs', 'vault-auth', ''),
metavar='AUTH')
refs_parser.add_argument('--refs-path',
help='set refs path, default is "./refs"',
default=from_dot_kapitan('refs', 'refs-path',
'./refs'))
refs_parser.add_argument(
'--verbose',
'-v',
help=
'set verbose mode (warning: this will potentially show sensitive data)',
action='store_true',
default=from_dot_kapitan('refs', 'verbose', False))
lint_parser = subparser.add_parser('lint',
help='linter for inventory and refs')
lint_parser.add_argument(
'--fail-on-warning',
default=from_dot_kapitan('lint', 'fail-on-warning', False),
action='store_true',
help='exit with failure code if warnings exist, default is False')
lint_parser.add_argument(
'--skip-class-checks',
action='store_true',
help='skip checking for unused classes, default is False',
default=from_dot_kapitan('lint', 'skip-class-checks', False))
lint_parser.add_argument(
'--skip-yamllint',
action='store_true',
help='skip running yamllint on inventory, default is False',
default=from_dot_kapitan('lint', 'skip-yamllint', False))
lint_parser.add_argument(
'--search-secrets',
default=from_dot_kapitan('lint', 'search-secrets', False),
action='store_true',
help='searches for plaintext secrets in inventory, default is False')
lint_parser.add_argument('--refs-path',
help='set refs path, default is "./refs"',
default=from_dot_kapitan('lint', 'refs-path',
'./refs'))
lint_parser.add_argument('--compiled-path',
default=from_dot_kapitan('lint', 'compiled-path',
'./compiled'),
help='set compiled path, default is "./compiled"')
lint_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('lint', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
init_parser = subparser.add_parser(
'init',
help=
'initialize a directory with the recommended kapitan project skeleton.'
)
init_parser.add_argument(
'--directory',
default=from_dot_kapitan('init', 'directory', '.'),
help='set path, in which to generate the project skeleton,'
'assumes directory already exists. default is "./"')
validate_parser = subparser.add_parser(
'validate',
help=
'validates the compile output against schemas as specified in inventory'
)
validate_parser.add_argument(
'--compiled-path',
default=from_dot_kapitan('compile', 'compiled-path', './compiled'),
help='set compiled path, default is "./compiled')
validate_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
validate_parser.add_argument('--targets',
'-t',
help='targets to validate, default is all',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'targets', []),
metavar='TARGET'),
validate_parser.add_argument(
'--schemas-path',
default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
help='set schema cache path, default is "./schemas"')
validate_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=from_dot_kapitan('validate', 'parallelism', 4),
metavar='INT',
help='Number of concurrent validate processes, default is 4')
args = parser.parse_args()
logger.debug('Running with args: %s', args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
# cache args where key is subcommand
cached.args[sys.argv[1]] = args
if hasattr(args, 'verbose') and args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
elif hasattr(args, 'quiet') and args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
if cmd == 'eval':
file_path = args.jsonnet_file
search_paths = [os.path.abspath(path) for path in args.search_paths]
ext_vars = {}
if args.vars:
ext_vars = dict(var.split('=') for var in args.vars)
json_output = None
def _search_imports(cwd, imp):
return search_imports(cwd, imp, search_paths)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_paths),
ext_vars=ext_vars)
if args.output == 'yaml':
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == 'compile':
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.refs_path)
# cache controller for use in reveal_maybe jinja2 filter
cached.ref_controller_obj = ref_controller
cached.revealer_obj = Revealer(ref_controller)
compile_targets(args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
args.labels,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths,
fetch_dependencies=args.fetch,
validate=args.validate,
schemas_path=args.schemas_path,
jinja2_filters=args.jinja2_filters)
elif cmd == 'inventory':
if args.pattern and args.target_name == '':
parser.error("--pattern requires --target_name")
try:
inv = inventory_reclass(args.inventory_path)
if args.target_name != '':
inv = inv['nodes'][args.target_name]
if args.pattern != '':
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv,
sys.stdout,
width=10000,
default_flow_style=False)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
sys.exit(1)
elif cmd == 'searchvar':
searchvar(args.searchvar, args.inventory_path, args.pretty_print)
elif cmd == 'lint':
start_lint(args.fail_on_warning, args.skip_class_checks,
args.skip_yamllint, args.inventory_path,
args.search_secrets, args.refs_path, args.compiled_path)
elif cmd == 'init':
initialise_skeleton(args.directory)
elif cmd == 'secrets':
logger.error(
"Secrets have been renamed to refs, please refer to: '$ kapitan refs --help'"
)
sys.exit(1)
elif cmd == 'refs':
ref_controller = RefController(args.refs_path)
if args.write is not None:
ref_write(args, ref_controller)
elif args.reveal:
ref_reveal(args, ref_controller)
elif args.update:
secret_update(args, ref_controller)
elif args.update_targets or args.validate_targets:
secret_update_validate(args, ref_controller)
elif cmd == 'validate':
schema_validate_compiled(args.targets,
inventory_path=args.inventory_path,
compiled_path=args.compiled_path,
schema_cache_path=args.schemas_path,
parallel=args.parallelism)
class CompilingFile(object):
def __init__(self, context, fp, ref_controller, **kwargs):
self.fp = fp
self.ref_controller = ref_controller
self.kwargs = kwargs
self.revealer = Revealer(ref_controller)
def write(self, data):
"""write data into file"""
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.fp.write(self.revealer.reveal_raw(data))
else:
self.fp.write(
self.revealer.compile_raw(data, target_name=target_name))
def write_yaml(self, obj):
"""recursively compile or reveal refs and convert obj to yaml and write to file"""
indent = self.kwargs.get('indent', 2)
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.revealer.reveal_obj(obj)
else:
self.revealer.compile_obj(obj, target_name=target_name)
yaml.dump(obj,
stream=self.fp,
indent=indent,
Dumper=PrettyDumper,
default_flow_style=False)
def write_json(self, obj):
"""recursively hash or reveal refs and convert obj to json and write to file"""
indent = self.kwargs.get('indent', 2)
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.revealer.reveal_obj(obj)
else:
self.revealer.compile_obj(obj, target_name=target_name)
json.dump(obj, self.fp, indent=indent, escape_forward_slashes=False)
def write_str(self, obj):
"""recursively hash or reveal refs and convert obj to str and write to file"""
reveal = self.kwargs.get('reveal', False)
target_name = self.kwargs.get('target_name', None)
if reveal:
self.revealer.reveal_obj(obj)
else:
self.revealer.compile_obj(obj, target_name=target_name)
#json.dump(obj, self.fp, indent=indent, escape_forward_slashes=False)
self.fp.write(obj)
def main():
"""main function for command line usage"""
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument('--version', action='version', version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file')
eval_parser.add_argument('jsonnet_file', type=str)
eval_parser.add_argument('--output',
type=str,
choices=('yaml', 'json', 'str'),
default=from_dot_kapitan('eval', 'output',
'yaml'),
help='set output format, default is "yaml"')
eval_parser.add_argument('--vars',
type=str,
default=from_dot_kapitan('eval', 'vars', []),
nargs='*',
metavar='VAR',
help='set variables')
eval_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan('eval', 'search-paths',
['.']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser = subparser.add_parser('compile', help='compile targets')
compile_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'search-paths', ['.']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'compile', 'verbose', False))
compile_parser.add_argument('--prune',
help='prune jsonnet output',
action='store_true',
default=from_dot_kapitan(
'compile', 'prune', False))
compile_parser.add_argument('--quiet',
help='set quiet mode, only critical output',
action='store_true',
default=from_dot_kapitan(
'compile', 'quiet', False))
compile_parser.add_argument('--output-path',
type=str,
default=from_dot_kapitan(
'compile', 'output-path', '.'),
metavar='PATH',
help='set output path, default is "."')
compile_parser.add_argument('--targets',
'-t',
help='targets to compile, default is all',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'targets', []),
metavar='TARGET')
compile_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=from_dot_kapitan('compile', 'parallelism', 4),
metavar='INT',
help='Number of concurrent compile processes, default is 4')
compile_parser.add_argument(
'--indent',
'-i',
type=int,
default=from_dot_kapitan('compile', 'indent', 2),
metavar='INT',
help='Indentation spaces for YAML/JSON, default is 2')
compile_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('compile', 'secrets-path', './secrets'))
compile_parser.add_argument(
'--reveal',
help='reveal secrets (warning: this will write sensitive data)',
action='store_true',
default=from_dot_kapitan('compile', 'reveal', False))
compile_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
compile_parser.add_argument(
'--cache',
'-c',
help='enable compilation caching to .kapitan_cache, default is False',
action='store_true',
default=from_dot_kapitan('compile', 'cache', False))
compile_parser.add_argument(
'--cache-paths',
type=str,
nargs='+',
default=from_dot_kapitan('compile', 'cache-paths', []),
metavar='PATH',
help='cache additional paths to .kapitan_cache, default is []')
compile_parser.add_argument('--ignore-version-check',
help='ignore the version from .kapitan',
action='store_true',
default=from_dot_kapitan(
'compile', 'ignore-version-check', False))
inventory_parser = subparser.add_parser('inventory', help='show inventory')
inventory_parser.add_argument(
'--target-name',
'-t',
default=from_dot_kapitan('inventory', 'target-name', ''),
help='set target name, default is all targets')
inventory_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('inventory', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
inventory_parser.add_argument('--flat',
'-F',
help='flatten nested inventory variables',
action='store_true',
default=from_dot_kapitan(
'inventory', 'flat', False))
inventory_parser.add_argument(
'--pattern',
'-p',
default=from_dot_kapitan('inventory', 'pattern', ''),
help=
'filter pattern (e.g. parameters.mysql.storage_class, or storage_class,'
+ ' or storage_*), default is ""')
inventory_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'inventory', 'verbose', False))
searchvar_parser = subparser.add_parser(
'searchvar', help='show all inventory files where var is declared')
searchvar_parser.add_argument(
'searchvar',
type=str,
metavar='VARNAME',
help=
'e.g. parameters.mysql.storage_class, or storage_class, or storage_*')
searchvar_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('searchvar', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
searchvar_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'verbose', False))
searchvar_parser.add_argument('--pretty-print',
'-p',
help='Pretty print content of var',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'pretty-print', False))
secrets_parser = subparser.add_parser('secrets', help='manage secrets')
secrets_parser.add_argument(
'--write',
'-w',
help='write secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument(
'--update',
help='update recipients for secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument('--update-targets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'update-targets', False),
help='update target secrets')
secrets_parser.add_argument('--validate-targets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'validate-targets', False),
help='validate target secrets')
secrets_parser.add_argument('--base64',
'-b64',
help='base64 encode file content',
action='store_true',
default=from_dot_kapitan(
'secrets', 'base64', False))
secrets_parser.add_argument('--reveal',
'-r',
help='reveal secrets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'reveal', False))
secrets_parser.add_argument(
'--file',
'-f',
help='read file or directory, set "-" for stdin',
metavar='FILENAME')
secrets_parser.add_argument('--target-name',
'-t',
help='grab recipients from target name')
secrets_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('secrets', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
secrets_parser.add_argument('--recipients',
'-R',
help='set recipients',
type=str,
nargs='+',
default=from_dot_kapitan(
'secrets', 'recipients', []),
metavar='RECIPIENT')
secrets_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('secrets', 'secrets-path', './secrets'))
secrets_parser.add_argument('--backend',
help='set secrets backend, default is "gpg"',
type=str,
choices=('gpg', ),
default=from_dot_kapitan(
'secrets', 'backend', 'gpg'))
secrets_parser.add_argument(
'--verbose',
'-v',
help='set verbose mode (warning: this will show sensitive data)',
action='store_true',
default=from_dot_kapitan('secrets', 'verbose', False))
args = parser.parse_args()
logger.debug('Running with args: %s', args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
if cmd == 'eval':
file_path = args.jsonnet_file
search_paths = [os.path.abspath(path) for path in args.search_paths]
ext_vars = {}
if args.vars:
ext_vars = dict(var.split('=') for var in args.vars)
json_output = None
def _search_imports(cwd, imp):
return search_imports(cwd, imp, search_paths)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_paths),
ext_vars=ext_vars)
if args.output == 'yaml':
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == 'compile':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
elif args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.secrets_path)
compile_targets(args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths)
elif cmd == 'inventory':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
if args.pattern and args.target_name == '':
parser.error("--pattern requires --target_name")
try:
inv = inventory_reclass(args.inventory_path)
if args.target_name != '':
inv = inv['nodes'][args.target_name]
if args.pattern != '':
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv, sys.stdout, width=10000)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.error("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
traceback.print_exc()
sys.exit(1)
elif cmd == 'searchvar':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
searchvar(args.searchvar, args.inventory_path, args.pretty_print)
elif cmd == 'secrets':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
ref_controller = RefController(args.secrets_path)
if args.write is not None:
if args.file is None:
parser.error('--file is required with --write')
data = None
recipients = [dict((("name", name), )) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
# TODO move into kapitan:secrets:gpg:recipients key
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
if args.file == '-':
data = ''
for line in sys.stdin:
data += line
else:
with open(args.file) as fp:
data = fp.read()
# TODO deprecate backend and move to passing ref tags in command line
if args.backend == "gpg":
secret_obj = GPGSecret(data, recipients, args.base64)
ref_controller.backends['gpg'][args.write] = secret_obj
elif args.reveal:
revealer = Revealer(ref_controller)
if args.file is None:
parser.error('--file is required with --reveal')
if args.file == '-':
# TODO deal with RefHashMismatchError or KeyError exceptions
out = revealer.reveal_raw_file(None)
sys.stdout.write(out)
elif args.file:
for rev_obj in revealer.reveal_path(args.file):
sys.stdout.write(rev_obj.content)
elif args.update:
# update recipients for secret tag
# args.recipients is a list, convert to recipients dict
recipients = [
dict([
("name", name),
]) for name in args.recipients
]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
# TODO move into kapitan:secrets:gpg:recipients key
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
if args.backend == "gpg":
secret_obj = ref_controller.backends['gpg'][args.update]
secret_obj.update_recipients(recipients)
ref_controller.backends['gpg'][args.update] = secret_obj
elif args.update_targets or args.validate_targets:
# update recipients for all secrets in secrets_path
# use --secrets-path to set scanning path
inv = inventory_reclass(args.inventory_path)
targets = set(inv['nodes'].keys())
secrets_path = os.path.abspath(args.secrets_path)
target_token_paths = search_target_token_paths(
secrets_path, targets)
ret_code = 0
# override gpg backend for new secrets_path
ref_controller.register_backend(GPGBackend(secrets_path))
for target_name, token_paths in target_token_paths.items():
try:
recipients = inv['nodes'][target_name]['parameters'][
'kapitan']['secrets']['recipients']
for token_path in token_paths:
secret_obj = ref_controller.backends['gpg'][token_path]
target_fingerprints = set(
lookup_fingerprints(recipients))
secret_fingerprints = set(
lookup_fingerprints(secret_obj.recipients))
if target_fingerprints != secret_fingerprints:
if args.validate_targets:
logger.info("%s recipient mismatch",
token_path)
ret_code = 1
else:
new_recipients = [
dict([
("fingerprint", f),
]) for f in target_fingerprints
]
secret_obj.update_recipients(new_recipients)
ref_controller.backends['gpg'][
token_path] = secret_obj
except KeyError:
logger.debug(
"secret_gpg_update_target: target: %s has no inventory recipients, skipping",
target_name)
sys.exit(ret_code)
