def secret_update(args, ref_controller):
"Update secret gpg recipients/gkms/awskms key"
# TODO --update *might* mean something else for other types
token_name = args.update
if token_name.startswith("gpg:"):
# args.recipients is a list, convert to recipients dict
recipients = [dict([("name", name), ]) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv['nodes'][args.target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
recipients = kap_inv_params['secrets']['gpg']['recipients']
if not recipients:
raise KapitanError("No GPG recipients specified. Use --recipients or specify them in " +
"parameters.kapitan.secrets.gpg.recipients and use --target")
type_name, token_path = token_name.split(":")
tag = '?{{gpg:{}}}'.format(token_path)
secret_obj = ref_controller[tag]
secret_obj.update_recipients(recipients)
ref_controller[tag] = secret_obj
elif token_name.startswith("gkms:"):
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv['nodes'][args.target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
key = kap_inv_params['secrets']['gkms']['key']
if not key:
raise KapitanError("No KMS key specified. Use --key or specify it in parameters.kapitan.secrets.gkms.key and use --target")
type_name, token_path = token_name.split(":")
tag = '?{{gkms:{}}}'.format(token_path)
secret_obj = ref_controller[tag]
secret_obj.update_key(key)
ref_controller[tag] = secret_obj
elif token_name.startswith("awskms:"):
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv['nodes'][args.target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError("parameters.kapitan.secrets not defined in {}".format(args.target_name))
key = kap_inv_params['secrets']['awskms']['key']
if not key:
raise KapitanError("No KMS key specified. Use --key or specify it in parameters.kapitan.secrets.awskms.key and use --target")
type_name, token_path = token_name.split(":")
tag = '?{{awskms:{}}}'.format(token_path)
secret_obj = ref_controller[tag]
secret_obj.update_key(key)
ref_controller[tag] = secret_obj
else:
fatal_error("Invalid token: {name}. Try using gpg/gkms/awskms:{name}".format(name=token_name))
def secret_write(args, ref_controller):
"Write secret to ref_controller based on cli args"
token_name = args.write
file_name = args.file
data = None
if file_name is None:
fatal_error('--file is required with --write')
if file_name == '-':
data = ''
for line in sys.stdin:
data += line
else:
with open(file_name) as fp:
data = fp.read()
if token_name.startswith("gpg:"):
type_name, token_path = token_name.split(":")
recipients = [dict((("name", name), )) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
try:
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['gpg']['recipients']
except KeyError:
# TODO: Keeping gpg recipients backwards-compatible until we make a breaking release
logger.warning(
"WARNING: parameters.kapitan.secrets.recipients is deprecated, "
+
"please move them to parameters.kapitan.secrets.gpg.recipients"
)
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
secret_obj = GPGSecret(data, recipients, encode_base64=args.base64)
tag = '?{{gpg:{}}}'.format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("gkms:"):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
key = inv['nodes'][args.target_name]['parameters']['kapitan'][
'secrets']['gkms']['key']
if not key:
raise KapitanError(
"No KMS key specified. Use --key or specify in parameters.kapitan.secrets.gkms.key and use --target"
)
secret_obj = GoogleKMSSecret(data, key, encode_base64=args.base64)
tag = '?{{gkms:{}}}'.format(token_path)
ref_controller[tag] = secret_obj
else:
fatal_error("Invalid token: {}".format(token_name))
def secret_update(args, ref_controller):
"Update secret recipients"
# TODO --update *might* mean something else for other types
token_name = args.update
if token_name.startswith("gpg:"):
# args.recipients is a list, convert to recipients dict
recipients = [dict([
("name", name),
]) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
try:
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['gpg']['recipients']
except KeyError:
# TODO: Keeping gpg recipients backwards-compatible until we make a breaking release
logger.warning(
"WARNING: parameters.kapitan.secrets.recipients is deprecated, "
+
"please move them to parameters.kapitan.secrets.gpg.recipients"
)
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
type_name, token_path = token_name.split(":")
tag = '?{{gpg:{}}}'.format(token_path)
secret_obj = ref_controller[tag]
secret_obj.update_recipients(recipients)
ref_controller[tag] = secret_obj
# TODO: Implement --update for KMS
else:
fatal_error("Invalid token: {}".format(token_name))
def search_targets(inventory_path, targets, labels):
"""returns a list of targets where the labels match, otherwise just return the original targets"""
if not labels:
return targets
try:
labels_dict = dict(label.split("=") for label in labels)
except ValueError:
raise CompileError(
"Compile error: Failed to parse labels, should be formatted like: kapitan compile -l env=prod app=example"
)
targets_found = []
inv = inventory_reclass(inventory_path)
for target_name in inv["nodes"]:
matched_all_labels = False
for label, value in labels_dict.items():
try:
if inv["nodes"][target_name]["parameters"]["kapitan"]["labels"][label] == value:
matched_all_labels = True
continue
except KeyError:
logger.debug("search_targets: label %s=%s didn't match target %s", label, value, target_name)
matched_all_labels = False
break
if matched_all_labels:
targets_found.append(target_name)
if len(targets_found) == 0:
raise CompileError("No targets found with labels: {}".format(labels))
return targets_found
def load_target_inventory(inventory_path,
targets,
ignore_class_notfound=False):
"""returns a list of target objects from the inventory"""
target_objs = []
inv = inventory_reclass(inventory_path, ignore_class_notfound)
# if '-t' is set on compile, only loop through selected targets
if targets:
targets_list = targets
else:
targets_list = inv["nodes"]
for target_name in targets_list:
try:
inv_target = inv["nodes"][target_name]
target_obj = inv_target["parameters"]["kapitan"]
target_obj["target_full_path"] = inv_target["__reclass__"][
"node"].replace("./", "")
require_compile = not ignore_class_notfound
valid_target_obj(target_obj, require_compile)
validate_matching_target_name(target_name, target_obj,
inventory_path)
logger.debug(
"load_target_inventory: found valid kapitan target %s",
target_name)
target_objs.append(target_obj)
except KeyError:
logger.debug(
"load_target_inventory: target %s has no kapitan compile obj",
target_name)
pass
return target_objs
def load_target_inventory(inventory_path, targets):
"""retuns a list of target objects from the inventory"""
target_objs = []
inv = inventory_reclass(inventory_path)
# if '-t' is set on compile, only loop through selected targets
if targets:
targets_list = targets
else:
targets_list = inv['nodes']
for target_name in targets_list:
try:
target_obj = inv['nodes'][target_name]['parameters']['kapitan']
valid_target_obj(target_obj)
logger.debug(
"load_target_inventory: found valid kapitan target %s",
target_name)
target_objs.append(target_obj)
except KeyError:
logger.debug(
"load_target_inventory: target %s has no kapitan compile obj",
target_name)
pass
return target_objs
def generate_inv_cache_hashes(inventory_path, targets, cache_paths):
"""
generates the hashes for the inventory per target and jsonnet/jinja2 folders for caching purposes
struct: {
inventory:
<target>:
classes: <sha256>
parameters: <sha256>
folder:
components: <sha256>
docs: <sha256>
lib: <sha256>
scripts: <sha256>
...
}
"""
inv = inventory_reclass(inventory_path)
cached.inv_cache = {}
cached.inv_cache['inventory'] = {}
cached.inv_cache['folder'] = {}
if targets:
for target in targets:
try:
cached.inv_cache['inventory'][target] = {}
cached.inv_cache['inventory'][target]['classes'] = dictionary_hash(inv['nodes'][target]['classes'])
cached.inv_cache['inventory'][target]['parameters'] = dictionary_hash(inv['nodes'][target]['parameters'])
except KeyError as e:
logger.error("'%s' target not found", target)
raise
else:
for target in inv['nodes']:
cached.inv_cache['inventory'][target] = {}
cached.inv_cache['inventory'][target]['classes'] = dictionary_hash(inv['nodes'][target]['classes'])
cached.inv_cache['inventory'][target]['parameters'] = dictionary_hash(inv['nodes'][target]['parameters'])
compile_obj = inv['nodes'][target]['parameters']['kapitan']['compile']
for obj in compile_obj:
for input_path in obj['input_paths']:
base_folder = os.path.dirname(input_path).split('/')[0]
if base_folder == '':
base_folder = os.path.basename(input_path).split('/')[0]
if base_folder not in cached.inv_cache['folder'].keys():
if os.path.exists(base_folder) and os.path.isdir(base_folder):
cached.inv_cache['folder'][base_folder] = directory_hash(base_folder)
# Cache additional folders set by --cache-paths
for path in cache_paths:
if path not in cached.inv_cache['folder'].keys():
if os.path.exists(path) and os.path.isdir(path):
cached.inv_cache['folder'][path] = directory_hash(path)
# Most commonly changed but not referenced in input_paths
for common in ('lib', 'vendor', 'secrets'):
if common not in cached.inv_cache['folder'].keys():
if os.path.exists(common) and os.path.isdir(common):
cached.inv_cache['folder'][common] = directory_hash(common)
def compile(config, cluster_id):
if config.local:
catalog_repo = _local_setup(config, cluster_id, TARGET)
else:
clean_working_tree(config)
catalog_repo = _regular_setup(config, cluster_id, TARGET)
# Compile kapitan inventory to extract component versions. Component
# versions are assumed to be defined in the inventory key
# 'parameters.component_versions'
reset_reclass_cache()
kapitan_inventory = inventory_reclass("inventory")["nodes"][TARGET]
versions = kapitan_inventory["parameters"].get("component_versions", None)
if versions and not config.local:
set_component_overrides(config, versions)
update_target(config, TARGET)
# Rebuild reclass inventory to use new version of components
reset_reclass_cache()
kapitan_inventory = inventory_reclass("inventory")["nodes"][TARGET]
jsonnet_libs = (kapitan_inventory["parameters"].get("commodore", {}).get(
"jsonnet_libs", None))
if jsonnet_libs and not config.local:
fetch_jsonnet_libs(config, jsonnet_libs)
if not config.local:
write_jsonnetfile(config)
fetch_jsonnet_libraries()
clean_catalog(catalog_repo)
# Generate Kapitan secret references from refs found in inventory
# parameters
update_refs(config, kapitan_inventory["parameters"])
kapitan_compile(config, search_paths=["./vendor/"])
postprocess_components(config, kapitan_inventory, TARGET,
config.get_components())
update_catalog(config, TARGET, catalog_repo)
click.secho("Catalog compiled! 🎉", bold=True)
def changed_targets(inventory_path, output_path):
"""returns a list of targets that have changed since last compilation"""
targets = []
inv = inventory_reclass(inventory_path)
saved_inv_cache = None
saved_inv_cache_path = os.path.join(output_path, "compiled/.kapitan_cache")
if os.path.exists(saved_inv_cache_path):
with open(saved_inv_cache_path, "r") as f:
try:
saved_inv_cache = yaml.safe_load(f)
except Exception as e:
logger.error("Failed to load kapitan cache: %s",
saved_inv_cache_path)
targets_list = list(inv['nodes'])
# If .kapitan_cache doesn't exist or failed to load, recompile all targets
if not saved_inv_cache:
return targets_list
else:
for key, hash in cached.inv_cache['folder'].items():
try:
if hash != saved_inv_cache['folder'][key]:
logger.debug(
"%s folder hash changed, recompiling all targets", key)
return targets_list
except KeyError as e:
# Errors usually occur when saved_inv_cache doesn't contain a new folder
# Recompile anyway to be safe
return targets_list
for target in targets_list:
try:
if cached.inv_cache['inventory'][target][
'classes'] != saved_inv_cache['inventory'][target][
'classes']:
logger.debug("classes hash changed in %s, recompiling",
target)
targets.append(target)
elif cached.inv_cache['inventory'][target][
'parameters'] != saved_inv_cache['inventory'][target][
'parameters']:
logger.debug("parameters hash changed in %s, recompiling",
target)
targets.append(target)
except KeyError as e:
# Errors usually occur when saved_inv_cache doesn't contain a new target
# Recompile anyway to be safe
targets.append(target)
return targets
def compile(config, cluster_id):
if config.local:
cluster, target_name, catalog_repo = _local_setup(config, cluster_id)
else:
clean_working_tree(config)
cluster, target_name, catalog_repo = _regular_setup(config, cluster_id)
# Compile kapitan inventory to extract component versions. Component
# versions are assumed to be defined in the inventory key
# 'parameters.component_versions'
kapitan_inventory = inventory_reclass('inventory')['nodes'][target_name]
versions = kapitan_inventory['parameters'].get('component_versions', None)
if versions and not config.local:
set_component_overrides(config, versions)
update_target(config, cluster)
# Rebuild reclass inventory to use new version of components
reset_reclass_cache()
kapitan_inventory = inventory_reclass('inventory')['nodes'][target_name]
jsonnet_libs = kapitan_inventory['parameters'].get('commodore', {}).get(
'jsonnet_libs', None)
if jsonnet_libs and not config.local:
fetch_jsonnet_libs(config, jsonnet_libs)
clean_catalog(catalog_repo)
# Generate Kapitan secret references from refs found in inventory
# parameters
update_refs(config, kapitan_inventory['parameters'])
kapitan_compile(config)
postprocess_components(config, kapitan_inventory, target_name,
config.get_components())
update_catalog(config, target_name, catalog_repo)
click.secho('Catalog compiled! 🎉', bold=True)
def test_compile_vars_target_missing(self):
inventory_path = "inventory"
target_filename = "minikube-es"
target_obj = inventory_reclass(inventory_path)["nodes"][target_filename]["parameters"]["kapitan"]
# delete vars.target
del target_obj["vars"]["target"]
with self.assertRaises(InventoryError) as ie:
validate_matching_target_name(target_filename, target_obj, inventory_path)
error_message = (
'Target missing: target "{}" is missing parameters.kapitan.vars.target\n'
"This parameter should be set to the target name"
)
self.assertTrue(error_message.format(target_filename), ie.exception.args[0])
def secret_update_validate(args, ref_controller):
"Validate and/or update target secrets"
# update recipients for all secrets in secrets_path
# use --secrets-path to set scanning path
inv = inventory_reclass(args.inventory_path)
targets = set(inv['nodes'].keys())
secrets_path = os.path.abspath(args.secrets_path)
target_token_paths = search_target_token_paths(secrets_path, targets)
ret_code = 0
ref_controller.register_backend(
GPGBackend(secrets_path)) # override gpg backend for new secrets_path
# TODO: Implement --update-targets and --validate-targets for KMS
for target_name, token_paths in target_token_paths.items():
try:
try:
recipients = inv['nodes'][target_name]['parameters'][
'kapitan']['secrets']['gpg']['recipients']
except KeyError:
# TODO: Keeping gpg recipients backwards-compatible until we make a breaking release
logger.warning(
"WARNING: parameters.kapitan.secrets.recipients is deprecated, "
+
"please move them to parameters.kapitan.secrets.gpg.recipients"
)
recipients = inv['nodes'][target_name]['parameters'][
'kapitan']['secrets']['recipients']
for token_path in token_paths:
secret_obj = ref_controller.backends['gpg'][token_path]
target_fingerprints = set(lookup_fingerprints(recipients))
secret_fingerprints = set(
lookup_fingerprints(secret_obj.recipients))
if target_fingerprints != secret_fingerprints:
if args.validate_targets:
logger.info("%s recipient mismatch", token_path)
ret_code = 1
else:
new_recipients = [
dict([
("fingerprint", f),
]) for f in target_fingerprints
]
secret_obj.update_recipients(new_recipients)
ref_controller.backends['gpg'][token_path] = secret_obj
except KeyError:
logger.debug(
"secret_gpg_update_target: target: %s has no inventory recipients, skipping",
target_name)
sys.exit(ret_code)
def _discover_components(cfg, inventory_path):
"""
Discover components in `inventory_path/`. Parse all classes found in
inventory_path and look for class includes starting with `components.`.
"""
reset_reclass_cache()
kapitan_inventory = inventory_reclass(inventory_path, ignore_class_notfound=True)[
"nodes"
]["cluster"]
components = set()
for kls in kapitan_inventory["classes"]:
if kls.startswith("components."):
component = kls.split(".")[1]
if cfg.debug:
click.echo(f" > Found component {component}")
components.add(component)
return sorted(components)
def compile(config, cluster_id):
if config.local:
cluster, target_name, catalog_repo = _local_setup(config, cluster_id)
else:
clean(config)
cluster, target_name, catalog_repo = _regular_setup(config, cluster_id)
# Compile kapitan inventory to extract component versions. Component
# versions are assumed to be defined in the inventory key
# 'parameters.component_versions'
kapitan_inventory = inventory_reclass('inventory')['nodes'][target_name]
versions = kapitan_inventory['parameters'].get('component_versions', None)
if versions and not config.local:
set_component_versions(config, versions)
update_target(config, cluster)
jsonnet_libs = kapitan_inventory['parameters'].get('commodore', {}).get(
'jsonnet_libs', None)
if jsonnet_libs and not config.local:
fetch_jsonnet_libs(jsonnet_libs)
clean_catalog(catalog_repo)
# Generate Kapitan secret references from refs found in inventory
# parameters
update_refs(config, kapitan_inventory['parameters'])
p = kapitan_compile()
if p.returncode != 0:
raise click.ClickException('Kapitan catalog compilation failed.')
postprocess_components(kapitan_inventory, target_name,
config.get_components())
update_catalog(config, target_name, catalog_repo)
click.secho('Catalog compiled! 🎉', bold=True)
def main():
"""main function for command line usage"""
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument('--version', action='version', version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file')
eval_parser.add_argument('jsonnet_file', type=str)
eval_parser.add_argument('--output',
type=str,
choices=('yaml', 'json'),
default=from_dot_kapitan('eval', 'output',
'yaml'),
help='set output format, default is "yaml"')
eval_parser.add_argument('--vars',
type=str,
default=from_dot_kapitan('eval', 'vars', []),
nargs='*',
metavar='VAR',
help='set variables')
eval_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan('eval', 'search-paths',
['.']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser = subparser.add_parser('compile', help='compile targets')
compile_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'search-paths', ['.', 'lib']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser.add_argument(
'--jinja2-filters',
'-J2F',
type=str,
default=from_dot_kapitan('compile', 'jinja2-filters',
default_jinja2_filters_path),
metavar='FPATH',
help='load custom jinja2 filters from any file, default is to put\
them inside lib/jinja2_filters.py')
compile_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'compile', 'verbose', False))
compile_parser.add_argument('--prune',
help='prune jsonnet output',
action='store_true',
default=from_dot_kapitan(
'compile', 'prune', False))
compile_parser.add_argument('--quiet',
help='set quiet mode, only critical output',
action='store_true',
default=from_dot_kapitan(
'compile', 'quiet', False))
compile_parser.add_argument('--output-path',
type=str,
default=from_dot_kapitan(
'compile', 'output-path', '.'),
metavar='PATH',
help='set output path, default is "."')
compile_parser.add_argument('--fetch',
help='fetches external dependencies',
action='store_true',
default=from_dot_kapitan(
'compile', 'fetch', False))
compile_parser.add_argument(
'--validate',
help=
'validate compile output against schemas as specified in inventory',
action='store_true',
default=from_dot_kapitan('compile', 'validate', False))
compile_parser.add_argument('--targets',
'-t',
help='targets to compile, default is all',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'targets', []),
metavar='TARGET')
compile_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=from_dot_kapitan('compile', 'parallelism', 4),
metavar='INT',
help='Number of concurrent compile processes, default is 4')
compile_parser.add_argument(
'--indent',
'-i',
type=int,
default=from_dot_kapitan('compile', 'indent', 2),
metavar='INT',
help='Indentation spaces for YAML/JSON, default is 2')
compile_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('compile', 'secrets-path', './secrets'))
compile_parser.add_argument(
'--reveal',
help='reveal secrets (warning: this will write sensitive data)',
action='store_true',
default=from_dot_kapitan('compile', 'reveal', False))
compile_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
compile_parser.add_argument(
'--cache',
'-c',
help='enable compilation caching to .kapitan_cache, default is False',
action='store_true',
default=from_dot_kapitan('compile', 'cache', False))
compile_parser.add_argument(
'--cache-paths',
type=str,
nargs='+',
default=from_dot_kapitan('compile', 'cache-paths', []),
metavar='PATH',
help='cache additional paths to .kapitan_cache, default is []')
compile_parser.add_argument('--ignore-version-check',
help='ignore the version from .kapitan',
action='store_true',
default=from_dot_kapitan(
'compile', 'ignore-version-check', False))
compile_parser.add_argument(
'--schemas-path',
default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
help='set schema cache path, default is "./schemas"')
inventory_parser = subparser.add_parser('inventory', help='show inventory')
inventory_parser.add_argument(
'--target-name',
'-t',
default=from_dot_kapitan('inventory', 'target-name', ''),
help='set target name, default is all targets')
inventory_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('inventory', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
inventory_parser.add_argument('--flat',
'-F',
help='flatten nested inventory variables',
action='store_true',
default=from_dot_kapitan(
'inventory', 'flat', False))
inventory_parser.add_argument(
'--pattern',
'-p',
default=from_dot_kapitan('inventory', 'pattern', ''),
help=
'filter pattern (e.g. parameters.mysql.storage_class, or storage_class,'
+ ' or storage_*), default is ""')
inventory_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'inventory', 'verbose', False))
searchvar_parser = subparser.add_parser(
'searchvar', help='show all inventory files where var is declared')
searchvar_parser.add_argument(
'searchvar',
type=str,
metavar='VARNAME',
help=
'e.g. parameters.mysql.storage_class, or storage_class, or storage_*')
searchvar_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('searchvar', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
searchvar_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'verbose', False))
searchvar_parser.add_argument('--pretty-print',
'-p',
help='Pretty print content of var',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'pretty-print', False))
secrets_parser = subparser.add_parser('secrets', help='manage secrets')
secrets_parser.add_argument(
'--write',
'-w',
help='write secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument(
'--update',
help='update recipients for secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument('--update-targets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'update-targets', False),
help='update target secrets')
secrets_parser.add_argument('--validate-targets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'validate-targets', False),
help='validate target secrets')
secrets_parser.add_argument('--base64',
'-b64',
help='base64 encode file content',
action='store_true',
default=from_dot_kapitan(
'secrets', 'base64', False))
secrets_parser.add_argument('--reveal',
'-r',
help='reveal secrets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'reveal', False))
secrets_parser.add_argument(
'--file',
'-f',
help='read file or directory, set "-" for stdin',
metavar='FILENAME')
secrets_parser.add_argument('--target-name',
'-t',
help='grab recipients from target name')
secrets_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('secrets', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
secrets_parser.add_argument('--recipients',
'-R',
help='set GPG recipients',
type=str,
nargs='+',
default=from_dot_kapitan(
'secrets', 'recipients', []),
metavar='RECIPIENT')
secrets_parser.add_argument('--key',
'-K',
help='set KMS key',
default=from_dot_kapitan('secrets', 'key', ''),
metavar='KEY')
secrets_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('secrets', 'secrets-path', './secrets'))
secrets_parser.add_argument(
'--verbose',
'-v',
help='set verbose mode (warning: this will show sensitive data)',
action='store_true',
default=from_dot_kapitan('secrets', 'verbose', False))
lint_parser = subparser.add_parser('lint',
help='linter for inventory and secrets')
lint_parser.add_argument(
'--fail-on-warning',
default=from_dot_kapitan('lint', 'fail-on-warning', False),
action='store_true',
help='exit with failure code if warnings exist, default is False')
lint_parser.add_argument(
'--skip-class-checks',
action='store_true',
help='skip checking for unused classes, default is False',
default=from_dot_kapitan('lint', 'skip-class-checks', False))
lint_parser.add_argument(
'--skip-yamllint',
action='store_true',
help='skip running yamllint on inventory, default is False',
default=from_dot_kapitan('lint', 'skip-yamllint', False))
lint_parser.add_argument(
'--search-secrets',
default=from_dot_kapitan('lint', 'search-secrets', False),
action='store_true',
help='searches for plaintext secrets in inventory, default is False')
lint_parser.add_argument('--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('lint', 'secrets-path',
'./secrets'))
lint_parser.add_argument('--compiled-path',
default=from_dot_kapitan('lint', 'compiled-path',
'./compiled'),
help='set compiled path, default is "./compiled"')
lint_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('lint', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
init_parser = subparser.add_parser(
'init',
help=
'initialize a directory with the recommended kapitan project skeleton.'
)
init_parser.add_argument(
'--directory',
default=from_dot_kapitan('init', 'directory', '.'),
help=
'set path, in which to generate the project skeleton, assumes directory already exists. default is "./"'
)
validate_parser = subparser.add_parser(
'validate',
help=
'validates the compile output against schemas as specified in inventory'
)
validate_parser.add_argument(
'--compiled-path',
default=from_dot_kapitan('compile', 'compiled-path', './compiled'),
help='set compiled path, default is "./compiled')
validate_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
validate_parser.add_argument('--targets',
'-t',
help='targets to validate, default is all',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'targets', []),
metavar='TARGET'),
validate_parser.add_argument(
'--schemas-path',
default=from_dot_kapitan('validate', 'schemas-path', './schemas'),
help='set schema cache path, default is "./schemas"')
validate_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=from_dot_kapitan('validate', 'parallelism', 4),
metavar='INT',
help='Number of concurrent validate processes, default is 4')
args = parser.parse_args()
logger.debug('Running with args: %s', args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
if hasattr(args, 'verbose') and args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
elif hasattr(args, 'quiet') and args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
if cmd == 'eval':
file_path = args.jsonnet_file
search_paths = [os.path.abspath(path) for path in args.search_paths]
ext_vars = {}
if args.vars:
ext_vars = dict(var.split('=') for var in args.vars)
json_output = None
def _search_imports(cwd, imp):
return search_imports(cwd, imp, search_paths)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_paths),
ext_vars=ext_vars)
if args.output == 'yaml':
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == 'compile':
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.secrets_path)
compile_targets(args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths,
fetch_dependencies=args.fetch,
validate=args.validate,
schemas_path=args.schemas_path,
jinja2_filters=args.jinja2_filters)
elif cmd == 'inventory':
if args.pattern and args.target_name == '':
parser.error("--pattern requires --target_name")
try:
inv = inventory_reclass(args.inventory_path)
if args.target_name != '':
inv = inv['nodes'][args.target_name]
if args.pattern != '':
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv,
sys.stdout,
width=10000,
default_flow_style=False)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
sys.exit(1)
elif cmd == 'searchvar':
searchvar(args.searchvar, args.inventory_path, args.pretty_print)
elif cmd == 'lint':
start_lint(args.fail_on_warning, args.skip_class_checks,
args.skip_yamllint, args.inventory_path,
args.search_secrets, args.secrets_path, args.compiled_path)
elif cmd == 'init':
initialise_skeleton(args.directory)
elif cmd == 'secrets':
ref_controller = RefController(args.secrets_path)
if args.write is not None:
secret_write(args, ref_controller)
elif args.reveal:
secret_reveal(args, ref_controller)
elif args.update:
secret_update(args, ref_controller)
elif args.update_targets or args.validate_targets:
secret_update_validate(args, ref_controller)
elif cmd == 'validate':
schema_validate_compiled(args.targets,
inventory_path=args.inventory_path,
compiled_path=args.compiled_path,
schema_cache_path=args.schemas_path,
parallel=args.parallelism)
def generate_inv_cache_hashes(inventory_path, targets, cache_paths):
"""
generates the hashes for the inventory per target and jsonnet/jinja2 folders for caching purposes
struct: {
inventory:
<target>:
classes: <sha256>
parameters: <sha256>
folder:
components: <sha256>
docs: <sha256>
lib: <sha256>
scripts: <sha256>
...
}
"""
inv = inventory_reclass(inventory_path)
cached.inv_cache = {}
cached.inv_cache["inventory"] = {}
cached.inv_cache["folder"] = {}
if targets:
for target in targets:
try:
cached.inv_cache["inventory"][target] = {}
cached.inv_cache["inventory"][target][
"classes"] = dictionary_hash(
inv["nodes"][target]["classes"])
cached.inv_cache["inventory"][target][
"parameters"] = dictionary_hash(
inv["nodes"][target]["parameters"])
except KeyError:
raise CompileError("target not found: {}".format(target))
else:
for target in inv["nodes"]:
cached.inv_cache["inventory"][target] = {}
cached.inv_cache["inventory"][target]["classes"] = dictionary_hash(
inv["nodes"][target]["classes"])
cached.inv_cache["inventory"][target][
"parameters"] = dictionary_hash(
inv["nodes"][target]["parameters"])
compile_obj = inv["nodes"][target]["parameters"]["kapitan"][
"compile"]
for obj in compile_obj:
for input_path in obj["input_paths"]:
base_folder = os.path.dirname(input_path).split("/")[0]
if base_folder == "":
base_folder = os.path.basename(input_path).split(
"/")[0]
if base_folder not in cached.inv_cache["folder"].keys():
if os.path.exists(base_folder) and os.path.isdir(
base_folder):
cached.inv_cache["folder"][
base_folder] = directory_hash(base_folder)
# Cache additional folders set by --cache-paths
for path in cache_paths:
if path not in cached.inv_cache["folder"].keys():
if os.path.exists(path) and os.path.isdir(path):
cached.inv_cache["folder"][path] = directory_hash(
path)
# Most commonly changed but not referenced in input_paths
for common in ("lib", "vendor", "secrets"):
if common not in cached.inv_cache["folder"].keys():
if os.path.exists(common) and os.path.isdir(common):
cached.inv_cache["folder"][common] = directory_hash(common)
def main():
"""main function for command line usage"""
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument('--version', action='version', version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file')
eval_parser.add_argument('jsonnet_file', type=str)
eval_parser.add_argument('--output',
type=str,
choices=('yaml', 'json', 'str'),
default=from_dot_kapitan('eval', 'output',
'yaml'),
help='set output format, default is "yaml"')
eval_parser.add_argument('--vars',
type=str,
default=from_dot_kapitan('eval', 'vars', []),
nargs='*',
metavar='VAR',
help='set variables')
eval_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan('eval', 'search-paths',
['.']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser = subparser.add_parser('compile', help='compile targets')
compile_parser.add_argument('--search-paths',
'-J',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'search-paths', ['.']),
metavar='JPATH',
help='set search paths, default is ["."]')
compile_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'compile', 'verbose', False))
compile_parser.add_argument('--prune',
help='prune jsonnet output',
action='store_true',
default=from_dot_kapitan(
'compile', 'prune', False))
compile_parser.add_argument('--quiet',
help='set quiet mode, only critical output',
action='store_true',
default=from_dot_kapitan(
'compile', 'quiet', False))
compile_parser.add_argument('--output-path',
type=str,
default=from_dot_kapitan(
'compile', 'output-path', '.'),
metavar='PATH',
help='set output path, default is "."')
compile_parser.add_argument('--targets',
'-t',
help='targets to compile, default is all',
type=str,
nargs='+',
default=from_dot_kapitan(
'compile', 'targets', []),
metavar='TARGET')
compile_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=from_dot_kapitan('compile', 'parallelism', 4),
metavar='INT',
help='Number of concurrent compile processes, default is 4')
compile_parser.add_argument(
'--indent',
'-i',
type=int,
default=from_dot_kapitan('compile', 'indent', 2),
metavar='INT',
help='Indentation spaces for YAML/JSON, default is 2')
compile_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('compile', 'secrets-path', './secrets'))
compile_parser.add_argument(
'--reveal',
help='reveal secrets (warning: this will write sensitive data)',
action='store_true',
default=from_dot_kapitan('compile', 'reveal', False))
compile_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('compile', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
compile_parser.add_argument(
'--cache',
'-c',
help='enable compilation caching to .kapitan_cache, default is False',
action='store_true',
default=from_dot_kapitan('compile', 'cache', False))
compile_parser.add_argument(
'--cache-paths',
type=str,
nargs='+',
default=from_dot_kapitan('compile', 'cache-paths', []),
metavar='PATH',
help='cache additional paths to .kapitan_cache, default is []')
compile_parser.add_argument('--ignore-version-check',
help='ignore the version from .kapitan',
action='store_true',
default=from_dot_kapitan(
'compile', 'ignore-version-check', False))
inventory_parser = subparser.add_parser('inventory', help='show inventory')
inventory_parser.add_argument(
'--target-name',
'-t',
default=from_dot_kapitan('inventory', 'target-name', ''),
help='set target name, default is all targets')
inventory_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('inventory', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
inventory_parser.add_argument('--flat',
'-F',
help='flatten nested inventory variables',
action='store_true',
default=from_dot_kapitan(
'inventory', 'flat', False))
inventory_parser.add_argument(
'--pattern',
'-p',
default=from_dot_kapitan('inventory', 'pattern', ''),
help=
'filter pattern (e.g. parameters.mysql.storage_class, or storage_class,'
+ ' or storage_*), default is ""')
inventory_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'inventory', 'verbose', False))
searchvar_parser = subparser.add_parser(
'searchvar', help='show all inventory files where var is declared')
searchvar_parser.add_argument(
'searchvar',
type=str,
metavar='VARNAME',
help=
'e.g. parameters.mysql.storage_class, or storage_class, or storage_*')
searchvar_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('searchvar', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
searchvar_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'verbose', False))
searchvar_parser.add_argument('--pretty-print',
'-p',
help='Pretty print content of var',
action='store_true',
default=from_dot_kapitan(
'searchvar', 'pretty-print', False))
secrets_parser = subparser.add_parser('secrets', help='manage secrets')
secrets_parser.add_argument(
'--write',
'-w',
help='write secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument(
'--update',
help='update recipients for secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument('--update-targets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'update-targets', False),
help='update target secrets')
secrets_parser.add_argument('--validate-targets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'validate-targets', False),
help='validate target secrets')
secrets_parser.add_argument('--base64',
'-b64',
help='base64 encode file content',
action='store_true',
default=from_dot_kapitan(
'secrets', 'base64', False))
secrets_parser.add_argument('--reveal',
'-r',
help='reveal secrets',
action='store_true',
default=from_dot_kapitan(
'secrets', 'reveal', False))
secrets_parser.add_argument(
'--file',
'-f',
help='read file or directory, set "-" for stdin',
metavar='FILENAME')
secrets_parser.add_argument('--target-name',
'-t',
help='grab recipients from target name')
secrets_parser.add_argument(
'--inventory-path',
default=from_dot_kapitan('secrets', 'inventory-path', './inventory'),
help='set inventory path, default is "./inventory"')
secrets_parser.add_argument('--recipients',
'-R',
help='set recipients',
type=str,
nargs='+',
default=from_dot_kapitan(
'secrets', 'recipients', []),
metavar='RECIPIENT')
secrets_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default=from_dot_kapitan('secrets', 'secrets-path', './secrets'))
secrets_parser.add_argument('--backend',
help='set secrets backend, default is "gpg"',
type=str,
choices=('gpg', ),
default=from_dot_kapitan(
'secrets', 'backend', 'gpg'))
secrets_parser.add_argument(
'--verbose',
'-v',
help='set verbose mode (warning: this will show sensitive data)',
action='store_true',
default=from_dot_kapitan('secrets', 'verbose', False))
args = parser.parse_args()
logger.debug('Running with args: %s', args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
if cmd == 'eval':
file_path = args.jsonnet_file
search_paths = [os.path.abspath(path) for path in args.search_paths]
ext_vars = {}
if args.vars:
ext_vars = dict(var.split('=') for var in args.vars)
json_output = None
def _search_imports(cwd, imp):
return search_imports(cwd, imp, search_paths)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_paths),
ext_vars=ext_vars)
if args.output == 'yaml':
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == 'compile':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
elif args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.secrets_path)
compile_targets(args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths)
elif cmd == 'inventory':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
if args.pattern and args.target_name == '':
parser.error("--pattern requires --target_name")
try:
inv = inventory_reclass(args.inventory_path)
if args.target_name != '':
inv = inv['nodes'][args.target_name]
if args.pattern != '':
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv, sys.stdout, width=10000)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.error("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
traceback.print_exc()
sys.exit(1)
elif cmd == 'searchvar':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
searchvar(args.searchvar, args.inventory_path, args.pretty_print)
elif cmd == 'secrets':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
ref_controller = RefController(args.secrets_path)
if args.write is not None:
if args.file is None:
parser.error('--file is required with --write')
data = None
recipients = [dict((("name", name), )) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
# TODO move into kapitan:secrets:gpg:recipients key
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
if args.file == '-':
data = ''
for line in sys.stdin:
data += line
else:
with open(args.file) as fp:
data = fp.read()
# TODO deprecate backend and move to passing ref tags in command line
if args.backend == "gpg":
secret_obj = GPGSecret(data, recipients, args.base64)
ref_controller.backends['gpg'][args.write] = secret_obj
elif args.reveal:
revealer = Revealer(ref_controller)
if args.file is None:
parser.error('--file is required with --reveal')
if args.file == '-':
# TODO deal with RefHashMismatchError or KeyError exceptions
out = revealer.reveal_raw_file(None)
sys.stdout.write(out)
elif args.file:
for rev_obj in revealer.reveal_path(args.file):
sys.stdout.write(rev_obj.content)
elif args.update:
# update recipients for secret tag
# args.recipients is a list, convert to recipients dict
recipients = [
dict([
("name", name),
]) for name in args.recipients
]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
# TODO move into kapitan:secrets:gpg:recipients key
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
if args.backend == "gpg":
secret_obj = ref_controller.backends['gpg'][args.update]
secret_obj.update_recipients(recipients)
ref_controller.backends['gpg'][args.update] = secret_obj
elif args.update_targets or args.validate_targets:
# update recipients for all secrets in secrets_path
# use --secrets-path to set scanning path
inv = inventory_reclass(args.inventory_path)
targets = set(inv['nodes'].keys())
secrets_path = os.path.abspath(args.secrets_path)
target_token_paths = search_target_token_paths(
secrets_path, targets)
ret_code = 0
# override gpg backend for new secrets_path
ref_controller.register_backend(GPGBackend(secrets_path))
for target_name, token_paths in target_token_paths.items():
try:
recipients = inv['nodes'][target_name]['parameters'][
'kapitan']['secrets']['recipients']
for token_path in token_paths:
secret_obj = ref_controller.backends['gpg'][token_path]
target_fingerprints = set(
lookup_fingerprints(recipients))
secret_fingerprints = set(
lookup_fingerprints(secret_obj.recipients))
if target_fingerprints != secret_fingerprints:
if args.validate_targets:
logger.info("%s recipient mismatch",
token_path)
ret_code = 1
else:
new_recipients = [
dict([
("fingerprint", f),
]) for f in target_fingerprints
]
secret_obj.update_recipients(new_recipients)
ref_controller.backends['gpg'][
token_path] = secret_obj
except KeyError:
logger.debug(
"secret_gpg_update_target: target: %s has no inventory recipients, skipping",
target_name)
sys.exit(ret_code)
def secret_update_validate(args, ref_controller):
"Validate and/or update target secrets"
# update gpg recipients/gkms/awskms key for all secrets in secrets_path
# use --refs-path to set scanning path
inv = inventory_reclass(args.inventory_path)
targets = set(inv["nodes"].keys())
secrets_path = os.path.abspath(args.refs_path)
target_token_paths = search_target_token_paths(secrets_path, targets)
ret_code = 0
for target_name, token_paths in target_token_paths.items():
kap_inv_params = inv["nodes"][target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in {}".format(
target_name))
try:
recipients = kap_inv_params["secrets"]["gpg"]["recipients"]
except KeyError:
recipients = None
try:
gkey = kap_inv_params["secrets"]["gkms"]["key"]
except KeyError:
gkey = None
try:
awskey = kap_inv_params["secrets"]["awskms"]["key"]
except KeyError:
awskey = None
try:
vaultkv = kap_inv_params["secrets"]["vaultkv"]["auth"]
except KeyError:
vaultkv = None
for token_path in token_paths:
if token_path.startswith("?{gpg:"):
if not recipients:
logger.debug(
"secret_update_validate: target: %s has no inventory gpg recipients, skipping %s",
target_name,
token_path,
)
continue
secret_obj = ref_controller[token_path]
target_fingerprints = set(lookup_fingerprints(recipients))
secret_fingerprints = set(
lookup_fingerprints(secret_obj.recipients))
if target_fingerprints != secret_fingerprints:
if args.validate_targets:
logger.info("%s recipient mismatch", token_path)
to_remove = secret_fingerprints.difference(
target_fingerprints)
to_add = target_fingerprints.difference(
secret_fingerprints)
if to_remove:
logger.info("%s needs removal", to_remove)
if to_add:
logger.info("%s needs addition", to_add)
ret_code = 1
else:
new_recipients = [
dict([
("fingerprint", f),
]) for f in target_fingerprints
]
secret_obj.update_recipients(new_recipients)
ref_controller[token_path] = secret_obj
elif token_path.startswith("?{gkms:"):
if not gkey:
logger.debug(
"secret_update_validate: target: %s has no inventory gkms key, skipping %s",
target_name,
token_path,
)
continue
secret_obj = ref_controller[token_path]
if gkey != secret_obj.key:
if args.validate_targets:
logger.info("%s key mismatch", token_path)
ret_code = 1
else:
secret_obj.update_key(gkey)
ref_controller[token_path] = secret_obj
elif token_path.startswith("?{awskms:"):
if not awskey:
logger.debug(
"secret_update_validate: target: %s has no inventory awskms key, skipping %s",
target_name,
token_path,
)
continue
secret_obj = ref_controller[token_path]
if awskey != secret_obj.key:
if args.validate_targets:
logger.info("%s key mismatch", token_path)
ret_code = 1
else:
secret_obj.update_key(awskey)
ref_controller[token_path] = secret_obj
else:
logger.info("Invalid secret %s, could not get type, skipping",
token_path)
sys.exit(ret_code)
def ref_write(args, ref_controller):
"Write ref to ref_controller based on cli args"
token_name = args.write
file_name = args.file
data = None
if file_name is None:
fatal_error("--file is required with --write")
if file_name == "-":
data = ""
for line in sys.stdin:
data += line
else:
with open(file_name) as fp:
data = fp.read()
if token_name.startswith("gpg:"):
type_name, token_path = token_name.split(":")
recipients = [dict((("name", name), )) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv["nodes"][
args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in inventory of target {}"
.format(args.target_name))
recipients = kap_inv_params["secrets"]["gpg"]["recipients"]
if not recipients:
raise KapitanError(
"No GPG recipients specified. Use --recipients or specify them in "
+
"parameters.kapitan.secrets.gpg.recipients and use --target-name"
)
secret_obj = GPGSecret(data, recipients, encode_base64=args.base64)
tag = "?{{gpg:{}}}".format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("gkms:"):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv["nodes"][
args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in inventory of target {}"
.format(args.target_name))
key = kap_inv_params["secrets"]["gkms"]["key"]
if not key:
raise KapitanError(
"No KMS key specified. Use --key or specify it in parameters.kapitan.secrets.gkms.key and use --target-name"
)
secret_obj = GoogleKMSSecret(data, key, encode_base64=args.base64)
tag = "?{{gkms:{}}}".format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("awskms:"):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv["nodes"][
args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in inventory of target {}"
.format(args.target_name))
key = kap_inv_params["secrets"]["awskms"]["key"]
if not key:
raise KapitanError(
"No KMS key specified. Use --key or specify it in parameters.kapitan.secrets.awskms.key and use --target-name"
)
secret_obj = AWSKMSSecret(data, key, encode_base64=args.base64)
tag = "?{{awskms:{}}}".format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("base64:"):
type_name, token_path = token_name.split(":")
_data = data.encode()
encoding = "original"
if args.base64:
_data = base64.b64encode(_data).decode()
_data = _data.encode()
encoding = "base64"
ref_obj = Base64Ref(_data, encoding=encoding)
tag = "?{{base64:{}}}".format(token_path)
ref_controller[tag] = ref_obj
elif token_name.startswith("vaultkv:"):
type_name, token_path = token_name.split(":")
_data = data.encode()
vault_params = {}
encoding = "original"
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv["nodes"][
args.target_name]["parameters"]["kapitan"]
if "secrets" not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in inventory of target {}"
.format(args.target_name))
vault_params = kap_inv_params["secrets"]["vaultkv"]
if args.vault_auth:
vault_params["auth"] = args.vault_auth
if vault_params.get("auth") is None:
raise KapitanError(
"No Authentication type parameter specified. Specify it"
" in parameters.kapitan.secrets.vaultkv.auth and use --target-name or use --vault-auth"
)
secret_obj = VaultSecret(_data, vault_params)
tag = "?{{vaultkv:{}}}".format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("plain:"):
type_name, token_path = token_name.split(":")
_data = data.encode()
encoding = "original"
if args.base64:
_data = base64.b64encode(_data).decode()
_data = _data.encode()
encoding = "base64"
ref_obj = PlainRef(_data, encoding=encoding)
tag = "?{{plain:{}}}".format(token_path)
ref_controller[tag] = ref_obj
else:
fatal_error(
"Invalid token: {name}. Try using gpg/gkms/awskms/vaultkv/base64/plain:{name}"
.format(name=token_name))
def main():
"""main function for command line usage"""
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument("--version", action="version", version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser("eval", help="evaluate jsonnet file")
eval_parser.add_argument("jsonnet_file", type=str)
eval_parser.add_argument(
"--output",
type=str,
choices=("yaml", "json"),
default=from_dot_kapitan("eval", "output", "yaml"),
help='set output format, default is "yaml"',
)
eval_parser.add_argument(
"--vars",
type=str,
default=from_dot_kapitan("eval", "vars", []),
nargs="*",
metavar="VAR",
help="set variables",
)
eval_parser.add_argument(
"--search-paths",
"-J",
type=str,
nargs="+",
default=from_dot_kapitan("eval", "search-paths", ["."]),
metavar="JPATH",
help='set search paths, default is ["."]',
)
compile_parser = subparser.add_parser("compile", help="compile targets")
compile_parser.add_argument(
"--search-paths",
"-J",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "search-paths", [".", "lib"]),
metavar="JPATH",
help='set search paths, default is ["."]',
)
compile_parser.add_argument(
"--jinja2-filters",
"-J2F",
type=str,
default=from_dot_kapitan("compile", "jinja2-filters",
defaults.DEFAULT_JINJA2_FILTERS_PATH),
metavar="FPATH",
help="load custom jinja2 filters from any file, default is to put\
them inside lib/jinja2_filters.py",
)
compile_parser.add_argument(
"--verbose",
"-v",
help="set verbose mode",
action="store_true",
default=from_dot_kapitan("compile", "verbose", False),
)
compile_parser.add_argument(
"--prune",
help="prune jsonnet output",
action="store_true",
default=from_dot_kapitan("compile", "prune", False),
)
compile_parser.add_argument(
"--quiet",
help="set quiet mode, only critical output",
action="store_true",
default=from_dot_kapitan("compile", "quiet", False),
)
compile_parser.add_argument(
"--output-path",
type=str,
default=from_dot_kapitan("compile", "output-path", "."),
metavar="PATH",
help='set output path, default is "."',
)
compile_parser.add_argument(
"--fetch",
help="fetches external dependencies",
action="store_true",
default=from_dot_kapitan("compile", "fetch", False),
)
compile_parser.add_argument(
"--validate",
help=
"validate compile output against schemas as specified in inventory",
action="store_true",
default=from_dot_kapitan("compile", "validate", False),
)
compile_parser.add_argument(
"--parallelism",
"-p",
type=int,
default=from_dot_kapitan("compile", "parallelism", 4),
metavar="INT",
help="Number of concurrent compile processes, default is 4",
)
compile_parser.add_argument(
"--indent",
"-i",
type=int,
default=from_dot_kapitan("compile", "indent", 2),
metavar="INT",
help="Indentation spaces for YAML/JSON, default is 2",
)
compile_parser.add_argument(
"--refs-path",
help='set refs path, default is "./refs"',
default=from_dot_kapitan("compile", "refs-path", "./refs"),
)
compile_parser.add_argument(
"--reveal",
help=
"reveal refs (warning: this will potentially write sensitive data)",
action="store_true",
default=from_dot_kapitan("compile", "reveal", False),
)
compile_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("compile", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
compile_parser.add_argument(
"--cache",
"-c",
help="enable compilation caching to .kapitan_cache, default is False",
action="store_true",
default=from_dot_kapitan("compile", "cache", False),
)
compile_parser.add_argument(
"--cache-paths",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "cache-paths", []),
metavar="PATH",
help="cache additional paths to .kapitan_cache, default is []",
)
compile_parser.add_argument(
"--ignore-version-check",
help="ignore the version from .kapitan",
action="store_true",
default=from_dot_kapitan("compile", "ignore-version-check", False),
)
compile_parser.add_argument(
"--schemas-path",
default=from_dot_kapitan("validate", "schemas-path", "./schemas"),
help='set schema cache path, default is "./schemas"',
)
compile_selector_parser = compile_parser.add_mutually_exclusive_group()
compile_selector_parser.add_argument(
"--targets",
"-t",
help="targets to compile, default is all",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "targets", []),
metavar="TARGET",
)
compile_selector_parser.add_argument(
"--labels",
"-l",
help="compile targets matching the labels, default is all",
type=str,
nargs="*",
default=from_dot_kapitan("compile", "labels", []),
metavar="key=value",
)
inventory_parser = subparser.add_parser("inventory", help="show inventory")
inventory_parser.add_argument(
"--target-name",
"-t",
default=from_dot_kapitan("inventory", "target-name", ""),
help="set target name, default is all targets",
)
inventory_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("inventory", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
inventory_parser.add_argument(
"--flat",
"-F",
help="flatten nested inventory variables",
action="store_true",
default=from_dot_kapitan("inventory", "flat", False),
)
inventory_parser.add_argument(
"--pattern",
"-p",
default=from_dot_kapitan("inventory", "pattern", ""),
help=
"filter pattern (e.g. parameters.mysql.storage_class, or storage_class,"
+ ' or storage_*), default is ""',
)
inventory_parser.add_argument(
"--verbose",
"-v",
help="set verbose mode",
action="store_true",
default=from_dot_kapitan("inventory", "verbose", False),
)
searchvar_parser = subparser.add_parser(
"searchvar", help="show all inventory files where var is declared")
searchvar_parser.add_argument(
"searchvar",
type=str,
metavar="VARNAME",
help=
"e.g. parameters.mysql.storage_class, or storage_class, or storage_*",
)
searchvar_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("searchvar", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
searchvar_parser.add_argument(
"--verbose",
"-v",
help="set verbose mode",
action="store_true",
default=from_dot_kapitan("searchvar", "verbose", False),
)
searchvar_parser.add_argument(
"--pretty-print",
"-p",
help="Pretty print content of var",
action="store_true",
default=from_dot_kapitan("searchvar", "pretty-print", False),
)
subparser.add_parser("secrets", help="(DEPRECATED) please use refs")
refs_parser = subparser.add_parser("refs", help="manage refs")
refs_parser.add_argument(
"--write",
"-w",
help="write ref token",
metavar="TOKENNAME",
)
refs_parser.add_argument(
"--update",
help="update GPG recipients for ref token",
metavar="TOKENNAME",
)
refs_parser.add_argument(
"--update-targets",
action="store_true",
default=from_dot_kapitan("refs", "update-targets", False),
help="update target secret refs",
)
refs_parser.add_argument(
"--validate-targets",
action="store_true",
default=from_dot_kapitan("refs", "validate-targets", False),
help="validate target secret refs",
)
refs_parser.add_argument(
"--base64",
"-b64",
help="base64 encode file content",
action="store_true",
default=from_dot_kapitan("refs", "base64", False),
)
refs_parser.add_argument(
"--reveal",
"-r",
help="reveal refs",
action="store_true",
default=from_dot_kapitan("refs", "reveal", False),
)
refs_parser.add_argument("--file",
"-f",
help='read file or directory, set "-" for stdin',
metavar="FILENAME")
refs_parser.add_argument("--target-name",
"-t",
help="grab recipients from target name")
refs_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("refs", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
refs_parser.add_argument(
"--recipients",
"-R",
help="set GPG recipients",
type=str,
nargs="+",
default=from_dot_kapitan("refs", "recipients", []),
metavar="RECIPIENT",
)
refs_parser.add_argument("--key",
"-K",
help="set KMS key",
default=from_dot_kapitan("refs", "key", ""),
metavar="KEY")
refs_parser.add_argument(
"--vault-auth",
help="set authentication type for vaultkv secrets",
default=from_dot_kapitan("refs", "vault-auth", ""),
metavar="AUTH",
)
refs_parser.add_argument(
"--refs-path",
help='set refs path, default is "./refs"',
default=from_dot_kapitan("refs", "refs-path", "./refs"),
)
refs_parser.add_argument(
"--verbose",
"-v",
help=
"set verbose mode (warning: this will potentially show sensitive data)",
action="store_true",
default=from_dot_kapitan("refs", "verbose", False),
)
lint_parser = subparser.add_parser("lint",
help="linter for inventory and refs")
lint_parser.add_argument(
"--fail-on-warning",
default=from_dot_kapitan("lint", "fail-on-warning", False),
action="store_true",
help="exit with failure code if warnings exist, default is False",
)
lint_parser.add_argument(
"--skip-class-checks",
action="store_true",
help="skip checking for unused classes, default is False",
default=from_dot_kapitan("lint", "skip-class-checks", False),
)
lint_parser.add_argument(
"--skip-yamllint",
action="store_true",
help="skip running yamllint on inventory, default is False",
default=from_dot_kapitan("lint", "skip-yamllint", False),
)
lint_parser.add_argument(
"--search-secrets",
default=from_dot_kapitan("lint", "search-secrets", False),
action="store_true",
help="searches for plaintext secrets in inventory, default is False",
)
lint_parser.add_argument(
"--refs-path",
help='set refs path, default is "./refs"',
default=from_dot_kapitan("lint", "refs-path", "./refs"),
)
lint_parser.add_argument(
"--compiled-path",
default=from_dot_kapitan("lint", "compiled-path", "./compiled"),
help='set compiled path, default is "./compiled"',
)
lint_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("lint", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
init_parser = subparser.add_parser(
"init",
help=
"initialize a directory with the recommended kapitan project skeleton."
)
init_parser.add_argument(
"--directory",
default=from_dot_kapitan("init", "directory", "."),
help="set path, in which to generate the project skeleton,"
'assumes directory already exists. default is "./"',
)
validate_parser = subparser.add_parser(
"validate",
help=
"validates the compile output against schemas as specified in inventory"
)
validate_parser.add_argument(
"--compiled-path",
default=from_dot_kapitan("compile", "compiled-path", "./compiled"),
help='set compiled path, default is "./compiled',
)
validate_parser.add_argument(
"--inventory-path",
default=from_dot_kapitan("compile", "inventory-path", "./inventory"),
help='set inventory path, default is "./inventory"',
)
validate_parser.add_argument(
"--targets",
"-t",
help="targets to validate, default is all",
type=str,
nargs="+",
default=from_dot_kapitan("compile", "targets", []),
metavar="TARGET",
),
validate_parser.add_argument(
"--schemas-path",
default=from_dot_kapitan("validate", "schemas-path", "./schemas"),
help='set schema cache path, default is "./schemas"',
)
validate_parser.add_argument(
"--parallelism",
"-p",
type=int,
default=from_dot_kapitan("validate", "parallelism", 4),
metavar="INT",
help="Number of concurrent validate processes, default is 4",
)
args = parser.parse_args()
logger.debug("Running with args: %s", args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
# cache args where key is subcommand
cached.args[sys.argv[1]] = args
if hasattr(args, "verbose") and args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(name)-12s %(levelname)-8s %(message)s")
elif hasattr(args, "quiet") and args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
if cmd == "eval":
file_path = args.jsonnet_file
search_paths = [os.path.abspath(path) for path in args.search_paths]
ext_vars = {}
if args.vars:
ext_vars = dict(var.split("=") for var in args.vars)
json_output = None
def _search_imports(cwd, imp):
return search_imports(cwd, imp, search_paths)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_paths),
ext_vars=ext_vars,
)
if args.output == "yaml":
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == "compile":
search_paths = [os.path.abspath(path) for path in args.search_paths]
if not args.ignore_version_check:
check_version()
ref_controller = RefController(args.refs_path)
# cache controller for use in reveal_maybe jinja2 filter
cached.ref_controller_obj = ref_controller
cached.revealer_obj = Revealer(ref_controller)
compile_targets(
args.inventory_path,
search_paths,
args.output_path,
args.parallelism,
args.targets,
args.labels,
ref_controller,
prune=(args.prune),
indent=args.indent,
reveal=args.reveal,
cache=args.cache,
cache_paths=args.cache_paths,
fetch_dependencies=args.fetch,
validate=args.validate,
schemas_path=args.schemas_path,
jinja2_filters=args.jinja2_filters,
)
elif cmd == "inventory":
if args.pattern and args.target_name == "":
parser.error("--pattern requires --target_name")
try:
inv = inventory_reclass(args.inventory_path)
if args.target_name != "":
inv = inv["nodes"][args.target_name]
if args.pattern != "":
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv,
sys.stdout,
width=10000,
default_flow_style=False)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.exception("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
sys.exit(1)
elif cmd == "searchvar":
searchvar(args.searchvar, args.inventory_path, args.pretty_print)
elif cmd == "lint":
start_lint(
args.fail_on_warning,
args.skip_class_checks,
args.skip_yamllint,
args.inventory_path,
args.search_secrets,
args.refs_path,
args.compiled_path,
)
elif cmd == "init":
initialise_skeleton(args.directory)
elif cmd == "secrets":
logger.error(
"Secrets have been renamed to refs, please refer to: '$ kapitan refs --help'"
)
sys.exit(1)
elif cmd == "refs":
ref_controller = RefController(args.refs_path)
if args.write is not None:
ref_write(args, ref_controller)
elif args.reveal:
ref_reveal(args, ref_controller)
elif args.update:
secret_update(args, ref_controller)
elif args.update_targets or args.validate_targets:
secret_update_validate(args, ref_controller)
elif cmd == "validate":
schema_validate_compiled(
args.targets,
inventory_path=args.inventory_path,
compiled_path=args.compiled_path,
schema_cache_path=args.schemas_path,
parallel=args.parallelism,
)
def compile_component(config: Config, component_path, value_files,
search_paths, output_path):
# Resolve all input to absolute paths to fix symlinks
component_path = P(component_path).resolve()
value_files = [P(f).resolve() for f in value_files]
search_paths = [P(d).resolve() for d in search_paths]
output_path = P(output_path).resolve()
# Ignore 'component-' prefix in dir name
component_name = component_path.stem.replace('component-', '')
click.secho(f"Compile component {component_name}...", bold=True)
temp_dir = P(tempfile.mkdtemp(prefix='component-')).resolve()
original_working_dir = os.getcwd()
os.chdir(temp_dir)
try:
if config.debug:
click.echo(f" > Created temp workspace: {temp_dir}")
_prepare_fake_inventory(temp_dir, component_name, component_path,
value_files)
# Create class for fake parameters
with open(temp_dir / 'inventory/classes/fake.yml', 'w') as file:
file.write("""
parameters:
cloud:
provider: cloudscale
region: rma1
cluster:
catalog_url: ssh://[email protected]/org/repo.git
dist: test-distribution
name: c-green-test-1234
customer:
name: t-silent-test-1234
argocd:
namespace: test
kapitan:
vars:
target: test
namespace: test
""")
# Create test target
with open(temp_dir / 'inventory/targets/test.yml', 'w') as file:
value_classes = "\n".join([f"- {c.stem}" for c in value_files])
file.write(f"""
classes:
- fake
- defaults.{component_name}
- components.{component_name}
{value_classes}
""")
# Fake Argo CD lib
(temp_dir / 'dependencies/lib').mkdir(exist_ok=True)
with open(temp_dir / 'dependencies/lib/argocd.libjsonnet',
'w') as file:
file.write("""
local ArgoApp(component, namespace, project='', secrets=true) = {};
local ArgoProject(name) = {};
{
App: ArgoApp,
Project: ArgoProject,
}
""")
# Fetch Jsonnet libs
fetch_jsonnet_libs(config, libs)
# Compile component
kapitan_compile(config,
target='test',
output_dir=output_path,
search_paths=search_paths,
fake_refs=True,
reveal=True)
click.echo(f" > Component compiled to {output_path / 'compiled/test'}")
# prepare inventory and fake component object for postprocess
inventory = inventory_reclass(temp_dir / 'inventory')['nodes']['test']
component = Component(component_name, Repo(component_path),
'https://fake.repo.url/', 'master')
# We change the working directory to the output_path directory here,
# as postprocess expects to find `compiled/<target>` in the working
# directory.
os.chdir(output_path)
postprocess_components(config, inventory, 'test',
{component_name: component})
finally:
os.chdir(original_working_dir)
if config.trace:
click.echo(f" > Temp dir left in place {temp_dir}")
else:
if config.debug:
click.echo(f" > Remove temp dir {temp_dir}")
shutil.rmtree(temp_dir)
def main():
"main function for command line usage"
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument('--version', action='version', version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file')
eval_parser.add_argument('jsonnet_file', type=str)
eval_parser.add_argument('--output',
type=str,
choices=('yaml', 'json'),
default='yaml',
help='set output format, default is "yaml"')
eval_parser.add_argument('--vars',
type=str,
default=[],
nargs='*',
metavar='VAR',
help='set variables')
eval_parser.add_argument('--search-path',
'-J',
type=str,
default='.',
metavar='JPATH',
help='set search path, default is "."')
compile_parser = subparser.add_parser('compile',
help='compile target files')
compile_parser.add_argument('--target-file',
'-f',
type=str,
nargs='+',
default=[],
metavar='TARGET',
help='target files')
compile_parser.add_argument('--search-path',
'-J',
type=str,
default='.',
metavar='JPATH',
help='set search path, default is "."')
compile_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=False)
compile_parser.add_argument('--no-prune',
help='do not prune jsonnet output',
action='store_true',
default=False)
compile_parser.add_argument('--quiet',
help='set quiet mode, only critical output',
action='store_true',
default=False)
compile_parser.add_argument(
'--output-path',
type=str,
default='compiled',
metavar='PATH',
help='set output path, default is "./compiled"')
compile_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=4,
metavar='INT',
help='Number of concurrent compile processes, default is 4')
compile_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default='./secrets',
)
compile_parser.add_argument(
'--reveal',
help='reveal secrets (warning: this will write sensitive data)',
action='store_true',
default=False)
inventory_parser = subparser.add_parser('inventory', help='show inventory')
inventory_parser.add_argument(
'--target-name',
'-t',
default='',
help='set target name, default is all targets')
inventory_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
inventory_parser.add_argument('--flat',
'-F',
help='flatten nested inventory variables',
action='store_true',
default=False)
searchvar_parser = subparser.add_parser(
'searchvar', help='show all inventory files where var is declared')
searchvar_parser.add_argument(
'searchvar',
type=str,
metavar='VARNAME',
help='flattened full variable name. Example: ' +
'parameters.cluster.type')
searchvar_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
secrets_parser = subparser.add_parser('secrets', help='manage secrets')
secrets_parser.add_argument(
'--write',
'-w',
help='write secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument('--reveal',
'-r',
help='reveal secrets',
action='store_true',
default=False)
secrets_parser.add_argument('--file',
'-f',
help='read file, set "-" for stdin',
required=True,
metavar='FILENAME')
secrets_parser.add_argument('--target-name',
'-t',
help='grab recipients from target name')
secrets_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
secrets_parser.add_argument('--recipients',
'-R',
help='set recipients',
type=str,
nargs='+',
default=[],
metavar='RECIPIENT')
secrets_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default='./secrets',
)
secrets_parser.add_argument('--backend',
help='set secrets backend, default is "gpg"',
type=str,
choices=('gpg', ),
default='gpg')
secrets_parser.add_argument(
'--verbose',
'-v',
help='set verbose mode (warning: this will show sensitive data)',
action='store_true',
default=False)
secrets_parser.add_argument('--no-verify',
help='do not verify secret hashes on reveal',
action='store_true',
default=False)
args = parser.parse_args()
logger.debug('Running with args: %s', args)
cmd = sys.argv[1]
if cmd == 'eval':
file_path = args.jsonnet_file
search_path = os.path.abspath(args.search_path)
ext_vars = {}
if args.vars:
ext_vars = dict(var.split('=') for var in args.vars)
json_output = None
_search_imports = lambda cwd, imp: search_imports(
cwd, imp, search_path)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_path),
ext_vars=ext_vars)
if args.output == 'yaml':
json_obj = json.loads(json_output)
yaml_output = yaml.safe_dump(json_obj, default_flow_style=False)
print yaml_output
elif json_output:
print json_output
elif cmd == 'compile':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
elif args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
search_path = os.path.abspath(args.search_path)
gpg_obj = secret_gpg_backend()
if args.target_file:
pool = multiprocessing.Pool(args.parallelism)
worker = partial(compile_target_file,
search_path=search_path,
output_path=args.output_path,
prune=(not args.no_prune),
secrets_path=args.secrets_path,
secrets_reveal=args.reveal,
gpg_obj=gpg_obj)
try:
pool.map(worker, args.target_file)
except RuntimeError:
# if compile worker fails, terminate immediately
pool.terminate()
raise
else:
logger.error("Nothing to compile")
elif cmd == 'inventory':
inv = inventory_reclass(args.inventory_path)
if args.target_name != '':
inv = inv['nodes'][args.target_name]
if args.flat:
inv = flatten_dict(inv)
print yaml.dump(inv, width=10000)
else:
print yaml.dump(inv, Dumper=PrettyDumper, default_flow_style=False)
elif cmd == 'searchvar':
searchvar(args.searchvar, args.inventory_path)
elif cmd == 'secrets':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
gpg_obj = secret_gpg_backend()
if args.write is not None:
data = None
recipients = args.recipients
if args.target_name:
inv = inventory_reclass(args.inventory_path)
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
if args.file == '-':
data = ''
for line in sys.stdin:
data += line
else:
with open(args.file) as fp:
data = fp.read()
secret_gpg_write(gpg_obj, args.secrets_path, args.write, data,
recipients)
elif args.reveal:
if args.file == '-':
secret_gpg_reveal(gpg_obj,
args.secrets_path,
None,
verify=(not args.no_verify))
elif args.file:
# TODO if it is a directory, reveal every file there
with open(args.file) as fp:
secret_gpg_reveal(gpg_obj,
args.secrets_path,
args.file,
verify=(not args.no_verify))
def secret_update_validate(args, ref_controller):
"Validate and/or update target secrets"
# update gpg recipients/gkms key for all secrets in secrets_path
# use --secrets-path to set scanning path
inv = inventory_reclass(args.inventory_path)
targets = set(inv['nodes'].keys())
secrets_path = os.path.abspath(args.secrets_path)
target_token_paths = search_target_token_paths(secrets_path, targets)
ret_code = 0
for target_name, token_paths in target_token_paths.items():
kap_inv_params = inv['nodes'][target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in {}".format(
target_name))
try:
try:
recipients = kap_inv_params['secrets']['gpg']['recipients']
except KeyError:
# TODO: Keeping gpg recipients backwards-compatible until we make a breaking release
logger.warning(
"WARNING: parameters.kapitan.secrets.recipients is deprecated, "
+ "please use parameters.kapitan.secrets.gpg.recipients")
recipients = kap_inv_params['secrets']['recipients']
except KeyError:
recipients = None
try:
key = kap_inv_params['secrets']['gkms']['key']
except KeyError:
key = None
for token_path in token_paths:
if token_path.startswith("?{gpg:"):
if not recipients:
logger.debug(
"secret_update_validate: target: %s has no inventory gpg recipients, skipping %s",
target_name, token_path)
continue
secret_obj = ref_controller[token_path]
target_fingerprints = set(lookup_fingerprints(recipients))
secret_fingerprints = set(
lookup_fingerprints(secret_obj.recipients))
if target_fingerprints != secret_fingerprints:
if args.validate_targets:
logger.info("%s recipient mismatch", token_path)
to_remove = secret_fingerprints.difference(
target_fingerprints)
to_add = target_fingerprints.difference(
secret_fingerprints)
if to_remove:
logger.info("%s needs removal", to_remove)
if to_add:
logger.info("%s needs addition", to_add)
ret_code = 1
else:
new_recipients = [
dict([
("fingerprint", f),
]) for f in target_fingerprints
]
secret_obj.update_recipients(new_recipients)
ref_controller[token_path] = secret_obj
elif token_path.startswith("?{gkms:"):
if not key:
logger.debug(
"secret_update_validate: target: %s has no inventory gkms key, skipping %s",
target_name, token_path)
continue
secret_obj = ref_controller[token_path]
if key != secret_obj.key:
if args.validate_targets:
logger.info("%s key mismatch", token_path)
ret_code = 1
else:
secret_obj.update_key(key)
ref_controller[token_path] = secret_obj
else:
logger.info("Invalid secret %s, could not get type, skipping",
token_path)
ret_code = 1
sys.exit(ret_code)
def main():
"main function for command line usage"
parser = argparse.ArgumentParser(prog=PROJECT_NAME,
description=DESCRIPTION)
parser.add_argument('--version', action='version', version=VERSION)
subparser = parser.add_subparsers(help="commands")
eval_parser = subparser.add_parser('eval', help='evaluate jsonnet file')
eval_parser.add_argument('jsonnet_file', type=str)
eval_parser.add_argument('--output',
type=str,
choices=('yaml', 'json'),
default='yaml',
help='set output format, default is "yaml"')
eval_parser.add_argument('--vars',
type=str,
default=[],
nargs='*',
metavar='VAR',
help='set variables')
eval_parser.add_argument('--search-path',
'-J',
type=str,
default='.',
metavar='JPATH',
help='set search path, default is "."')
compile_parser = subparser.add_parser('compile', help='compile targets')
compile_parser.add_argument('--search-path',
'-J',
type=str,
default='.',
metavar='JPATH',
help='set search path, default is "."')
compile_parser.add_argument('--verbose',
'-v',
help='set verbose mode',
action='store_true',
default=False)
compile_parser.add_argument('--no-prune',
help='do not prune jsonnet output',
action='store_true',
default=False)
compile_parser.add_argument('--quiet',
help='set quiet mode, only critical output',
action='store_true',
default=False)
compile_parser.add_argument('--output-path',
type=str,
default='.',
metavar='PATH',
help='set output path, default is "."')
compile_parser.add_argument('--targets',
'-t',
help='targets to compile, default is all',
type=str,
nargs='+',
default=[],
metavar='TARGET')
compile_parser.add_argument(
'--parallelism',
'-p',
type=int,
default=4,
metavar='INT',
help='Number of concurrent compile processes, default is 4')
compile_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default='./secrets',
)
compile_parser.add_argument(
'--reveal',
help='reveal secrets (warning: this will write sensitive data)',
action='store_true',
default=False)
compile_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
inventory_parser = subparser.add_parser('inventory', help='show inventory')
inventory_parser.add_argument(
'--target-name',
'-t',
default='',
help='set target name, default is all targets')
inventory_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
inventory_parser.add_argument('--flat',
'-F',
help='flatten nested inventory variables',
action='store_true',
default=False)
inventory_parser.add_argument(
'--pattern',
'-p',
default='',
help='filter pattern (e.g. parameters.mysql), default is ""')
searchvar_parser = subparser.add_parser(
'searchvar', help='show all inventory files where var is declared')
searchvar_parser.add_argument(
'searchvar',
type=str,
metavar='VARNAME',
help='flattened full variable name. Example: ' +
'parameters.cluster.type')
searchvar_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
secrets_parser = subparser.add_parser('secrets', help='manage secrets')
secrets_parser.add_argument(
'--write',
'-w',
help='write secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument(
'--update',
help='update recipients for secret token',
metavar='TOKENNAME',
)
secrets_parser.add_argument('--update-targets',
action='store_true',
default=False,
help='update target secrets')
secrets_parser.add_argument('--validate-targets',
action='store_true',
default=False,
help='validate target secrets')
secrets_parser.add_argument('--base64',
'-b64',
help='base64 encode file content',
action='store_true',
default=False)
secrets_parser.add_argument('--reveal',
'-r',
help='reveal secrets',
action='store_true',
default=False)
secrets_parser.add_argument(
'--file',
'-f',
help='read file or directory, set "-" for stdin',
metavar='FILENAME')
secrets_parser.add_argument('--target-name',
'-t',
help='grab recipients from target name')
secrets_parser.add_argument(
'--inventory-path',
default='./inventory',
help='set inventory path, default is "./inventory"')
secrets_parser.add_argument('--recipients',
'-R',
help='set recipients',
type=str,
nargs='+',
default=[],
metavar='RECIPIENT')
secrets_parser.add_argument(
'--secrets-path',
help='set secrets path, default is "./secrets"',
default='./secrets',
)
secrets_parser.add_argument('--backend',
help='set secrets backend, default is "gpg"',
type=str,
choices=('gpg', ),
default='gpg')
secrets_parser.add_argument(
'--verbose',
'-v',
help='set verbose mode (warning: this will show sensitive data)',
action='store_true',
default=False)
secrets_parser.add_argument('--no-verify',
help='do not verify secret hashes on reveal',
action='store_true',
default=False)
args = parser.parse_args()
logger.debug('Running with args: %s', args)
try:
cmd = sys.argv[1]
except IndexError:
parser.print_help()
sys.exit(1)
if cmd == 'eval':
file_path = args.jsonnet_file
search_path = os.path.abspath(args.search_path)
ext_vars = {}
if args.vars:
ext_vars = dict(var.split('=') for var in args.vars)
json_output = None
_search_imports = lambda cwd, imp: search_imports(
cwd, imp, search_path)
json_output = jsonnet_file(
file_path,
import_callback=_search_imports,
native_callbacks=resource_callbacks(search_path),
ext_vars=ext_vars)
if args.output == 'yaml':
json_obj = json.loads(json_output)
yaml.safe_dump(json_obj, sys.stdout, default_flow_style=False)
elif json_output:
print(json_output)
elif cmd == 'compile':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
elif args.quiet:
logging.basicConfig(level=logging.CRITICAL, format="%(message)s")
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
search_path = os.path.abspath(args.search_path)
gpg_obj = secret_gpg_backend()
compile_targets(args.inventory_path,
search_path,
args.output_path,
args.parallelism,
args.targets,
prune=(not args.no_prune),
secrets_path=args.secrets_path,
secrets_reveal=args.reveal,
gpg_obj=gpg_obj)
elif cmd == 'inventory':
try:
logging.basicConfig(level=logging.INFO, format="%(message)s")
inv = inventory_reclass(args.inventory_path)
if args.target_name != '':
inv = inv['nodes'][args.target_name]
if args.pattern != '':
pattern = args.pattern.split(".")
inv = deep_get(inv, pattern)
if args.flat:
inv = flatten_dict(inv)
yaml.dump(inv, sys.stdout, width=10000)
else:
yaml.dump(inv,
sys.stdout,
Dumper=PrettyDumper,
default_flow_style=False)
except Exception as e:
if not isinstance(e, KapitanError):
logger.error("\n~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~")
traceback.print_exc()
sys.exit(1)
elif cmd == 'searchvar':
searchvar(args.searchvar, args.inventory_path)
elif cmd == 'secrets':
if args.verbose:
logging.basicConfig(
level=logging.DEBUG,
format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
else:
logging.basicConfig(level=logging.INFO, format="%(message)s")
gpg_obj = secret_gpg_backend()
if args.write is not None:
if args.file is None:
parser.error('--file is required with --write')
data = None
recipients = [dict((("name", name), )) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
if args.file == '-':
data = ''
for line in sys.stdin:
data += line
else:
with open(args.file) as fp:
data = fp.read()
secret_gpg_write(gpg_obj, args.secrets_path, args.write, data,
args.base64, recipients)
elif args.reveal:
if args.file is None:
parser.error('--file is required with --reveal')
if args.file == '-':
secret_gpg_reveal_raw(gpg_obj,
args.secrets_path,
None,
verify=(not args.no_verify))
elif args.file:
if os.path.isfile(args.file):
out = secret_gpg_reveal_file(gpg_obj,
args.secrets_path,
args.file,
verify=(not args.no_verify))
sys.stdout.write(out)
elif os.path.isdir(args.file):
secret_gpg_reveal_dir(gpg_obj,
args.secrets_path,
args.file,
verify=(not args.no_verify))
elif args.update:
# update recipients for secret tag
recipients = [
dict([
("name", name),
]) for name in args.recipients
]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
recipients = inv['nodes'][args.target_name]['parameters'][
'kapitan']['secrets']['recipients']
secret_gpg_update_recipients(gpg_obj, args.secrets_path,
args.update, recipients)
elif args.update_targets or args.validate_targets:
# update recipients for all secrets in secrets_path
# use --secrets-path to set scanning path
inv = inventory_reclass(args.inventory_path)
targets = set(inv['nodes'].keys())
secrets_path = os.path.abspath(args.secrets_path)
target_token_paths = search_target_token_paths(
secrets_path, targets)
ret_code = 0
for target_name, token_paths in target_token_paths.items():
try:
recipients = inv['nodes'][target_name]['parameters'][
'kapitan']['secrets']['recipients']
for token_path in token_paths:
target_fingerprints = set(
lookup_fingerprints(gpg_obj, recipients))
secret_fingerprints = set(
secret_gpg_raw_read_fingerprints(
secrets_path, token_path))
if target_fingerprints != secret_fingerprints:
if args.validate_targets:
logger.info("%s recipient mismatch",
token_path)
ret_code = 1
else:
new_recipients = [
dict([
("fingerprint", f),
]) for f in target_fingerprints
]
secret_gpg_update_recipients(
gpg_obj, secrets_path, token_path,
new_recipients)
except KeyError:
logger.debug(
"secret_gpg_update_target: target: %s has no inventory recipients, skipping",
target_name)
sys.exit(ret_code)
def secret_write(args, ref_controller):
"Write secret to ref_controller based on cli args"
token_name = args.write
file_name = args.file
data = None
if file_name is None:
fatal_error('--file is required with --write')
if file_name == '-':
data = ''
for line in sys.stdin:
data += line
else:
with open(file_name) as fp:
data = fp.read()
if token_name.startswith("gpg:"):
type_name, token_path = token_name.split(":")
recipients = [dict((("name", name), )) for name in args.recipients]
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv['nodes'][
args.target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in {}".format(
args.target_name))
recipients = kap_inv_params['secrets']['gpg']['recipients']
if not recipients:
raise KapitanError(
"No GPG recipients specified. Use --recipients or specify them in "
+ "parameters.kapitan.secrets.gpg.recipients and use --target")
secret_obj = GPGSecret(data, recipients, encode_base64=args.base64)
tag = '?{{gpg:{}}}'.format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("gkms:"):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv['nodes'][
args.target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in {}".format(
args.target_name))
key = kap_inv_params['secrets']['gkms']['key']
if not key:
raise KapitanError(
"No KMS key specified. Use --key or specify it in parameters.kapitan.secrets.gkms.key and use --target"
)
secret_obj = GoogleKMSSecret(data, key, encode_base64=args.base64)
tag = '?{{gkms:{}}}'.format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("awskms:"):
type_name, token_path = token_name.split(":")
key = args.key
if args.target_name:
inv = inventory_reclass(args.inventory_path)
kap_inv_params = inv['nodes'][
args.target_name]['parameters']['kapitan']
if 'secrets' not in kap_inv_params:
raise KapitanError(
"parameters.kapitan.secrets not defined in {}".format(
args.target_name))
key = kap_inv_params['secrets']['awskms']['key']
if not key:
raise KapitanError(
"No KMS key specified. Use --key or specify it in parameters.kapitan.secrets.awskms.key and use --target"
)
secret_obj = AWSKMSSecret(data, key, encode_base64=args.base64)
tag = '?{{awskms:{}}}'.format(token_path)
ref_controller[tag] = secret_obj
elif token_name.startswith("ref:"):
type_name, token_path = token_name.split(":")
_data = data.encode()
encoding = 'original'
if args.base64:
_data = base64.b64encode(_data).decode()
_data = _data.encode()
encoding = 'base64'
ref_obj = Ref(_data, encoding=encoding)
tag = '?{{ref:{}}}'.format(token_path)
ref_controller[tag] = ref_obj
else:
fatal_error(
"Invalid token: {name}. Try using gpg/gkms/awskms/ref:{name}".
format(name=token_name))
