def raw_file(client, src, dest, opt):
"""Write the contents of a vault path/key to a file"""
path, key = path_pieces(src)
resp = client.read(path)
if not resp:
client.revoke_self_token()
raise aomi.exceptions.VaultData("Unable to retrieve %s" % path)
else:
if 'data' in resp and key in resp['data']:
secret = resp['data'][key]
if is_base64(secret):
log('decoding base64 entry', opt)
secret = portable_b64decode(secret)
if is_aws(resp['data']):
renew_secret(client, resp, opt)
secret_file = None
if sys.version_info >= (3, 0):
if not isinstance(secret, str):
secret_file = open(abspath(dest), 'wb')
if not secret_file:
secret_file = open(abspath(dest), 'w')
secret_file.write(secret)
else:
client.revoke_self_token()
e_msg = "Key %s not found in %s" % (key, path)
raise aomi.exceptions.VaultData(e_msg)
def gitignore(opt):
"""Will check directories upwards from the Secretfile in order
to ensure the gitignore file is set properly"""
directory = os.path.dirname(abspath(opt.secretfile))
gitignore_file = find_file('.gitignore', directory)
if gitignore_file:
secrets_path = subdir_file(abspath(opt.secrets), gitignore_file)
if not secrets_path:
problems("Unable to determine relative location of secretfile")
if not in_file(secrets_path, gitignore_file):
problems("The path %s was not found in %s" %
(secrets_path, gitignore_file))
else:
problems("You should really have a .gitignore")
def template(client, src, dest, paths, opt):
"""Writes a template using variables from a vault path"""
key_map = cli_hash(opt.key_map)
obj = {}
for path in paths:
response = client.read(path)
if not response:
raise aomi.exceptions.VaultData("Unable to retrieve %s" % path)
if is_aws(response['data']) and 'sts' not in path:
renew_secret(client, response, opt)
for s_k, s_v in response['data'].items():
o_key = s_k
if s_k in key_map:
o_key = key_map[s_k]
k_name = secret_key_name(path, o_key, opt) \
.lower() \
.replace('-', '_')
obj[k_name] = s_v
template_obj = blend_vars(obj, opt)
output = render(grok_template_file(src),
template_obj)
write_raw_file(output, abspath(dest))
def render(filename, obj):
"""Render a template, maybe mixing in extra variables"""
template_path = abspath(filename)
fs_loader = FileSystemLoader(os.path.dirname(template_path))
env = Environment(loader=fs_loader, autoescape=True)
env.filters['b64encode'] = portable_b64encode
env.filters['b64decode'] = portable_b64decode
template_src = env.loader.get_source(env, os.path.basename(template_path))
parsed_content = env.parse(template_src)
template_vars = meta.find_undeclared_variables(parsed_content)
if template_vars:
missing_vars = []
default_vars = grok_default_vars(parsed_content)
for var in template_vars:
if var not in default_vars and var not in obj:
missing_vars.append(var)
if missing_vars:
e_msg = "Missing required variables %s" % ','.join(missing_vars)
raise aomi.exceptions.AomiData(e_msg)
template_obj = env.get_template(os.path.basename(template_path))
output = template_obj.render(**obj)
return output
def grok_template_file(src):
"""Determine the real deal template file"""
if not src.startswith('builtin:'):
return abspath(src)
builtin = src.split(':')[1]
builtin = "templates/%s.j2" % builtin
return resource_filename(__name__, builtin)
def gitignore(opt):
"""Will check directories upwards from the Secretfile in order
to ensure the gitignore file is set properly"""
directory = os.path.dirname(abspath(opt.secretfile))
gitignore_file = find_file('.gitignore', directory)
if gitignore_file:
secrets_path = subdir_path(abspath(opt.secrets), gitignore_file)
if secrets_path:
if not in_file(secrets_path, gitignore_file):
e_msg = "The path %s was not found in %s" \
% (secrets_path, gitignore_file)
raise aomi.exceptions.AomiFile(e_msg)
else:
log("Using a non-relative secret directory", opt)
else:
raise aomi.exceptions.AomiFile("You should really have a .gitignore")
def load_var_files(opt):
"""Load variable files, merge, return contents"""
obj = {}
for var_file in opt.extra_vars_file:
yamlz = yaml.safe_load(open(abspath(var_file)).read())
obj = merge_dicts(obj.copy(), yamlz)
return obj
def grok_template_file(src):
"""Determine the real deal template file"""
if not src.startswith('builtin:'):
return abspath(src)
builtin = src.split(':')[1]
builtin = "templates/%s.j2" % builtin
return resource_filename(__name__, builtin)
def gitignore(opt):
"""Will check directories upwards from the Secretfile in order
to ensure the gitignore file is set properly"""
directory = os.path.dirname(abspath(opt.secretfile))
gitignore_file = find_file('.gitignore', directory)
if gitignore_file:
secrets_path = subdir_path(abspath(opt.secrets), gitignore_file)
if secrets_path:
if not in_file(secrets_path, gitignore_file):
e_msg = "The path %s was not found in %s" \
% (secrets_path, gitignore_file)
raise aomi.exceptions.AomiFile(e_msg)
else:
LOG.debug("Using a non-relative secret directory")
else:
raise aomi.exceptions.AomiFile("You should really have a .gitignore")
def find_file(name, directory):
"""Searches up from a directory looking for a file"""
path_bits = directory.split(os.sep)
for i in range(0, len(path_bits) - 1):
check_path = path_bits[0:len(path_bits) - i]
check_file = "%s%s%s" % (os.sep.join(check_path), os.sep, name)
if os.path.exists(check_file):
return abspath(check_file)
return None
def initial_token(vault_client, opt):
"""Generate our first token based on workstation configuration"""
home = os.environ['HOME'] if 'HOME' in os.environ else \
os.environ['USERPROFILE']
token_file = os.environ.get('VAULT_TOKEN_FILE',
os.path.join(home, ".vault-token"))
app_file = os.environ.get('AOMI_APP_FILE',
os.path.join(home, ".aomi-app-token"))
token_file = abspath(token_file)
app_file = abspath(app_file)
if 'VAULT_TOKEN' in os.environ and os.environ['VAULT_TOKEN']:
log('Token derived from VAULT_TOKEN environment variable', opt)
return os.environ['VAULT_TOKEN'].strip()
elif 'VAULT_USER_ID' in os.environ and \
'VAULT_APP_ID' in os.environ and \
os.environ['VAULT_USER_ID'] and os.environ['VAULT_APP_ID']:
token = app_token(vault_client,
os.environ['VAULT_APP_ID'].strip(),
os.environ['VAULT_USER_ID'].strip())
log("Token derived from VAULT_APP_ID and VAULT_USER_ID", opt)
return token
elif 'VAULT_ROLE_ID' in os.environ and \
'VAULT_SECRET_ID' in os.environ and \
os.environ['VAULT_ROLE_ID'] and os.environ['VAULT_SECRET_ID']:
token = approle_token(vault_client,
os.environ['VAULT_ROLE_ID'],
os.environ['VAULT_SECRET_ID'])
log("Token derived from VAULT_ROLE_ID and VAULT_SECRET_ID", opt)
return token
elif os.path.exists(app_file):
token = yaml.safe_load(open(app_file).read().strip())
if 'app_id' in token and 'user_id' in token:
token = app_token(vault_client,
token['app_id'],
token['user_id'])
log("Token derived from %s" % app_file, opt)
return token
elif os.path.exists(token_file):
log("Token derived from %s" % token_file, opt)
return open(token_file, 'r').read().strip()
else:
raise aomi.exceptions.AomiCredentials('unknown method')
def find_file(name, directory):
"""Searches up from a directory looking for a file"""
path_bits = directory.split(os.sep)
for i in range(0, len(path_bits) - 1):
check_path = path_bits[0:len(path_bits) - i]
check_file = "%s%s%s" % (os.sep.join(check_path), os.sep, name)
if os.path.exists(check_file):
return abspath(check_file)
return None
def builtin_list():
"""Show a listing of all our builtin templates"""
for template in resource_listdir(__name__, "templates"):
builtin, ext = os.path.splitext(os.path.basename(abspath(template)))
if ext == '.yml':
continue
help_obj = load_template_help(builtin)
if 'name' in help_obj:
print("%-*s %s" % (20, builtin, help_obj['name']))
else:
print("%s" % builtin)
def write_raw_file(secret, dest):
"""Writes an actual secret out to a file"""
secret_file = None
secret_filename = abspath(dest)
if sys.version_info >= (3, 0):
if not isinstance(secret, str):
secret_file = open(secret_filename, 'wb')
if not secret_file:
secret_file = open(secret_filename, 'w')
secret_file.write(secret)
secret_file.close()
os.chmod(secret_filename, 0o600)
def write_raw_file(secret, dest):
"""Writes an actual secret out to a file"""
secret_file = None
secret_filename = abspath(dest)
if sys.version_info >= (3, 0):
if not isinstance(secret, str):
secret_file = open(secret_filename, 'wb')
if not secret_file:
secret_file = open(secret_filename, 'w')
secret_file.write(secret)
secret_file.close()
os.chmod(secret_filename, 0o600)
def vault_file(env, default):
"""The path to a misc Vault file
This function will check for the env override on a file
path, compute a fully qualified OS appropriate path to
the desired file and return it if it exists. Otherwise
returns None
"""
home = os.environ['HOME'] if 'HOME' in os.environ else \
os.environ['USERPROFILE']
filename = os.environ.get(env, os.path.join(home, default))
filename = abspath(filename)
if os.path.exists(filename):
return filename
return None
def vault_file(env, default):
"""The path to a misc Vault file
This function will check for the env override on a file
path, compute a fully qualified OS appropriate path to
the desired file and return it if it exists. Otherwise
returns None
"""
home = os.environ['HOME'] if 'HOME' in os.environ else \
os.environ['USERPROFILE']
filename = os.environ.get(env, os.path.join(home, default))
filename = abspath(filename)
if os.path.exists(filename):
return filename
return None
def secret_file(filename):
"""Will check the permissions of things which really
should be secret files"""
filestat = os.stat(abspath(filename))
if stat.S_ISREG(filestat.st_mode) == 0 and \
stat.S_ISLNK(filestat.st_mode) == 0:
e_msg = "Secret file %s must be a real file or symlink" % filename
raise aomi.exceptions.AomiFile(e_msg)
if platform.system() != "Windows":
if filestat.st_mode & stat.S_IROTH or \
filestat.st_mode & stat.S_IWOTH or \
filestat.st_mode & stat.S_IWGRP:
e_msg = "Secret file %s has too loose permissions" % filename
raise aomi.exceptions.AomiFile(e_msg)
def secret_file(filename):
"""Will check the permissions of things which really
should be secret files"""
filestat = os.stat(abspath(filename))
if stat.S_ISREG(filestat.st_mode) == 0 and \
stat.S_ISLNK(filestat.st_mode) == 0:
e_msg = "Secret file %s must be a real file or symlink" % filename
raise aomi.exceptions.AomiFile(e_msg)
if platform.system() != "Windows":
if filestat.st_mode & stat.S_IROTH or \
filestat.st_mode & stat.S_IWOTH or \
filestat.st_mode & stat.S_IWGRP:
e_msg = "Secret file %s has too loose permissions" % filename
raise aomi.exceptions.AomiFile(e_msg)
def raw_file(client, src, dest, opt):
"""Write the contents of a vault path/key to a file"""
path, key = path_pieces(src)
resp = client.read(path)
if not resp:
client.revoke_self_token()
raise aomi.exceptions.VaultData("Unable to retrieve %s" % path)
else:
if 'data' in resp and key in resp['data']:
secret = resp['data'][key]
if is_aws(resp['data']):
renew_secret(client, resp, opt)
open(abspath(dest), 'w').write(secret)
else:
client.revoke_self_token()
e_msg = "Key %s not found in %s" % (key, path)
raise aomi.exceptions.VaultData(e_msg)
def render(filename, obj):
"""Render a template, maybe mixing in extra variables"""
template_path = abspath(filename)
env = jinja_env(template_path)
template_base = os.path.basename(template_path)
try:
parsed_content = env.parse(env.loader.get_source(env, template_base))
template_vars = meta.find_undeclared_variables(parsed_content)
if template_vars:
missing_vars(template_vars, parsed_content, obj)
LOG.debug("rendering %s with %s vars", template_path,
len(template_vars))
return env \
.get_template(template_base) \
.render(**obj)
except jinja2.exceptions.TemplateSyntaxError as exception:
template_trace = traceback.format_tb(sys.exc_info()[2])
# Different error context depending on whether it is the
# pre-render variable scan or not
if exception.filename:
template_line = template_trace[len(template_trace) - 1]
raise aomi_excep.Validation("Bad template %s %s" %
(template_line, str(exception)))
template_str = ''
if isinstance(exception.source, tuple):
# PyLint seems confused about whether or not this is a tuple
# pylint: disable=locally-disabled, unsubscriptable-object
template_str = "Embedded Template\n%s" % exception.source[0]
raise aomi_excep.Validation("Bad template %s" % str(exception),
source=template_str)
except jinja2.exceptions.UndefinedError as exception:
template_traces = [
x.strip() for x in traceback.format_tb(sys.exc_info()[2])
if 'template code' in x
]
raise aomi_excep.Validation("Missing template variable %s" %
' '.join(template_traces))
def template(client, src, dest, paths, opt):
"""Writes a template using variables from a vault path"""
key_map = cli_hash(opt.key_map)
obj = {}
for path in paths:
response = client.read(path)
if is_aws(response['data']):
renew_secret(client, response, opt)
for s_k, s_v in response['data'].items():
o_key = s_k
if s_k in key_map:
o_key = key_map[s_k]
k_name = secret_key_name(path, o_key, opt) \
.lower() \
.replace('-', '_')
obj[k_name] = s_v
template_obj = blend_vars(obj, opt)
output = render(grok_template_file(src), template_obj)
open(abspath(dest), 'w').write(output)
def template(client, src, dest, paths, opt):
"""Writes a template using variables from a vault path"""
key_map = cli_hash(opt.key_map)
obj = {}
for path in paths:
response = client.read(path)
if not response:
raise aomi.exceptions.VaultData("Unable to retrieve %s" % path)
if is_aws(response['data']) and 'sts' not in path:
renew_secret(client, response, opt)
for s_k, s_v in response['data'].items():
o_key = s_k
if s_k in key_map:
o_key = key_map[s_k]
k_name = secret_key_name(path, o_key, opt) \
.lower() \
.replace('-', '_')
obj[k_name] = s_v
template_obj = blend_vars(obj, opt)
output = render(grok_template_file(src), template_obj)
write_raw_file(output, abspath(dest))
def get_secretfile(opt):
"""Renders, YAMLs, and returns the Secretfile construct"""
secretfile_path = abspath(opt.secretfile)
obj = merge_dicts(load_var_files(opt),
cli_hash(opt.extra_vars))
return yaml.safe_load(render(secretfile_path, obj))
def render_secretfile(opt):
"""Renders and returns the Secretfile construct"""
LOG.debug("Using Secretfile %s", opt.secretfile)
secretfile_path = abspath(opt.secretfile)
obj = load_vars(opt)
return render(secretfile_path, obj)
