def approle(client, approle_obj, opt):
"""Will seed application role information into a Vault"""
aomi.validation.approle_obj(approle_obj)
name = approle_obj['name']
if not aomi.validation.tag_check(approle_obj, "approle/%s" % name, opt):
return
ensure_auth(client, 'approle')
policies = approle_obj['policies']
role_obj = {'policies': ','.join(policies)}
if 'cidr_list' in approle_obj:
role_obj['bound_cidr_list'] = ','.join(approle_obj['cidr_list'])
else:
role_obj['bound_cidr_list'] = ''
if 'secret_uses' in approle_obj:
role_obj['secret_id_num_uses'] = approle_obj['secret_uses']
if 'secret_ttl' in approle_obj:
role_obj['secret_id_ttl'] = approle_obj['secret_ttl']
log("creating approle %s" % name, opt)
client.create_role(name, **role_obj)
def thaw(src_file, opt):
"""Given the combination of a Secretfile and the output of
a freeze operation, will restore secrets to usable locations"""
if not os.path.exists(src_file):
raise aomi.exceptions.AomiFile("%s does not exist" % src_file)
tmp_dir = tempfile.mkdtemp('aomi-freeze')
zip_file = thaw_decrypt(src_file, tmp_dir, opt)
archive = zipfile.ZipFile(zip_file, 'r')
for archive_file in archive.namelist():
archive.extract(archive_file, tmp_dir)
os.chmod("%s/%s" % (tmp_dir, archive_file), 0o640)
log("Extracted %s from archive" % archive_file, opt)
log("Thawing secrets into %s" % opt.secrets, opt)
config = get_secretfile(opt)
for app in config.get('apps', []):
thaw_secret(app['app_file'], tmp_dir, 'App', opt)
for user in config.get('users', []):
thaw_secret(user['password_file'], tmp_dir, 'User', opt)
for secret in config.get('secrets', []):
if 'var_file' in secret:
thaw_var_file(secret, tmp_dir, opt)
elif 'aws_file' in secret:
thaw_aws_file(secret, tmp_dir, opt)
elif 'files' in secret:
thaw_files(secret, tmp_dir, opt)
for duo in config.get('duo', []):
thaw_secret(duo['creds'], tmp_dir, 'DUO', opt)
def files(client, secret, opt):
"""Seed files into Vault"""
if 'mount' not in secret or 'path' not in secret:
problems("Invalid files specification %s" % secret)
obj = {}
vault_path = "%s/%s" % (secret['mount'], secret['path'])
if not is_tagged(secret.get('tags', []), opt.tags):
log("Skipping %s as it does not have appropriate tags" % vault_path,
opt)
return
for f in secret.get('files', []):
if 'source' not in f or 'name' not in f:
problems("Invalid file specification %s" % f)
filename = hard_path(f['source'], opt.secrets)
data = open(filename, 'r').read()
obj[f['name']] = data
log('writing file %s into %s/%s' % (filename, vault_path, f['name']),
opt)
ensure_mounted(client, 'generic', secret['mount'])
client.write(vault_path, **obj)
def initial_token(vault_client, opt):
"""Generate our first token based on workstation configuration"""
app_filename = appid_file()
token_filename = token_file()
if 'VAULT_TOKEN' in os.environ and os.environ['VAULT_TOKEN']:
log('Token derived from VAULT_TOKEN environment variable', opt)
return os.environ['VAULT_TOKEN'].strip()
elif 'VAULT_USER_ID' in os.environ and \
'VAULT_APP_ID' in os.environ and \
os.environ['VAULT_USER_ID'] and os.environ['VAULT_APP_ID']:
token = app_token(vault_client, os.environ['VAULT_APP_ID'].strip(),
os.environ['VAULT_USER_ID'].strip())
log("Token derived from VAULT_APP_ID and VAULT_USER_ID", opt)
return token
elif 'VAULT_ROLE_ID' in os.environ and \
'VAULT_SECRET_ID' in os.environ and \
os.environ['VAULT_ROLE_ID'] and os.environ['VAULT_SECRET_ID']:
token = approle_token(vault_client, os.environ['VAULT_ROLE_ID'],
os.environ['VAULT_SECRET_ID'])
log("Token derived from VAULT_ROLE_ID and VAULT_SECRET_ID", opt)
return token
elif app_filename:
token = yaml.safe_load(open(app_filename).read().strip())
if 'app_id' in token and 'user_id' in token:
token = app_token(vault_client, token['app_id'], token['user_id'])
log("Token derived from %s" % app_filename, opt)
return token
elif token_filename:
log("Token derived from %s" % token_filename, opt)
return open(token_filename, 'r').read().strip()
else:
raise aomi.exceptions.AomiCredentials('unknown method')
def raw_file(client, src, dest, opt):
"""Write the contents of a vault path/key to a file"""
path, key = path_pieces(src)
resp = client.read(path)
if not resp:
client.revoke_self_token()
raise aomi.exceptions.VaultData("Unable to retrieve %s" % path)
else:
if 'data' in resp and key in resp['data']:
secret = resp['data'][key]
if is_base64(secret):
log('decoding base64 entry', opt)
secret = portable_b64decode(secret)
if is_aws(resp['data']):
renew_secret(client, resp, opt)
secret_file = None
if sys.version_info >= (3, 0):
if not isinstance(secret, str):
secret_file = open(abspath(dest), 'wb')
if not secret_file:
secret_file = open(abspath(dest), 'w')
secret_file.write(secret)
else:
client.revoke_self_token()
e_msg = "Key %s not found in %s" % (key, path)
raise aomi.exceptions.VaultData(e_msg)
def tag_check(obj, path, opt):
"""If we require tags, validate for that"""
if not is_tagged(opt.tags, obj.get('tags', [])):
log("Skipping %s as it does not have requested tags" %
path, opt)
return False
return True
def sync(self, vault_client):
"""Synchronizes the local and remote Vault resources. Has the net
effect of adding backend if needed"""
if not self.existing:
self.actually_mount(vault_client)
log("Mounting %s backend on %s" % (self.backend, self.path),
self.opt)
else:
log("%s backend already mounted on %s" % (self.backend, self.path),
self.opt)
def thaw_secret(filename, tmp_dir, flav, opt):
"""Will perform some validation and copy a
decrypted secret to it's final location"""
src_file = "%s/%s" % (tmp_dir, filename)
dest_file = "%s/%s" % (opt.secrets, filename)
if not os.path.exists(src_file):
raise aomi.exceptions.IceFile("%s file %s missing" % (flav, filename))
shutil.copy(src_file, dest_file)
log("Thawed %s %s" % (flav, filename), opt)
def freeze_secret(src, dest, flav, tmp_dir, opt):
"""Copies a secret into a particular location"""
src_file = hard_path(src, opt.secrets)
dest_file = "%s/%s" % (tmp_dir, dest)
dest_dir = os.path.dirname(dest_file)
if not os.path.isdir(dest_dir):
os.mkdir(dest_dir, 0o700)
shutil.copy(src_file, dest_file)
log("Froze %s %s" % (flav, src), opt)
def from_keybase(username, opt):
"""Will attempt to retrieve a GPG public key from
Keybase, importing if neccesary"""
public_key = key_from_keybase(username)
fingerprint = public_key['fingerprint'][-8:].upper().encode('ascii')
key = public_key['bundle'].encode('ascii')
if not has_gpg_key(fingerprint):
log("Importing gpg key for %s" % username, opt)
if not import_gpg_key(key):
raise aomi.exceptions.KeybaseAPI("import key for %s" % username)
return fingerprint
def duo_enable(client, backend, opt):
"""Set the MFA type to DUO"""
obj = {'type': 'duo'}
path = "auth/%s/mfa_config" % backend
existing = client.read(path)
if existing \
and 'data' in existing \
and 'type' in existing['data'] \
and existing['data']['type'] == 'duo':
log("Auth backend %s already configured for DUO" % backend, opt)
else:
write(client, path, obj, opt)
log("Auth backend %s now configured for DUO" % backend, opt)
def freeze(dest_dir, opt):
"""Iterates over the Secretfile looking for secrets to freeze"""
tmp_dir = ensure_tmpdir()
dest_prefix = "%s/dest" % tmp_dir
ensure_dir(dest_dir)
ensure_dir(dest_prefix)
config = get_secretfile(opt)
ctx = Context.load(config, opt)
ctx.freeze(dest_prefix)
zip_filename = freeze_archive(tmp_dir, dest_prefix)
ice_file = freeze_encrypt(dest_dir, zip_filename, config, opt)
shutil.rmtree(tmp_dir)
log("Generated file is %s" % ice_file, opt)
def thaw_aws_file(secret, tmp_dir, opt):
"""Thaw the contents of an AWS file"""
path = "%s/config/root" % (sanitize_mount(secret['mount']))
if not validate_entry(secret, path, opt):
return
dest_file = "%s/%s" % (opt.secrets, secret['aws_file'])
aws_file = os.path.basename(dest_file)
src_file = "%s/%s" % (tmp_dir, aws_file)
if not os.path.exists(src_file):
raise aomi.exceptions.IceFile("AWS file %s missing" % aws_file)
shutil.copy(src_file, dest_file)
log("Thawed aws_file %s" % aws_file, opt)
def thaw_var_file(secret, tmp_dir, opt):
"""Thaw the contents of a var file"""
path = "%s/%s" % (sanitize_mount(secret['mount']), secret['path'])
if not validate_entry(secret, path, opt):
return
dest_file = "%s/%s" % (opt.secrets, secret['var_file'])
var_file = os.path.basename(dest_file)
src_file = "%s/%s" % (tmp_dir, var_file)
if not os.path.exists(src_file):
raise aomi.exceptions.IceFile("Var file %s missing" % var_file)
shutil.copy(src_file, dest_file)
log("Thawed var_file %s" % var_file, opt)
def initial_token(vault_client, opt):
"""Generate our first token based on workstation configuration"""
token_file = os.environ.get('VAULT_TOKEN_FILE',
"%s/.vault-token" % os.environ['HOME'])
app_file = os.environ.get('AOMI_APP_FILE',
"%s/.aomi-app-token" % os.environ['HOME'])
token_file = os.path.abspath(token_file)
app_file = os.path.abspath(app_file)
if 'VAULT_TOKEN' in os.environ and len(os.environ['VAULT_TOKEN']) > 0:
log('Token derived from VAULT_TOKEN environment variable', opt)
return os.environ['VAULT_TOKEN'].strip()
elif 'VAULT_USER_ID' in os.environ and \
'VAULT_APP_ID' in os.environ and \
len(os.environ['VAULT_USER_ID']) > 0 and \
len(os.environ['VAULT_APP_ID']) > 0:
token = app_token(vault_client, os.environ['VAULT_APP_ID'].strip(),
os.environ['VAULT_USER_ID'].strip())
log("Token derived from VAULT_APP_ID and VAULT_USER_ID", opt)
return token
elif os.path.exists(app_file):
token = yaml.load(open(app_file).read().strip())
if 'app_id' in token and 'user_id' in token:
token = app_token(vault_client, token['app_id'], token['user_id'])
log("Token derived from %s" % app_file, opt)
return token
elif os.path.exists(token_file):
log("Token derived from %s" % token_file, opt)
return open(token_file, 'r').read().strip()
else:
problems('Unable to determine vault authentication method')
def policy(client, secret, opt):
"""Seed a standalone policy into Vault"""
aomi.validation.policy_obj(secret)
policy_name = secret['name']
if not validate_entry(secret, "policy/%s" % policy_name, opt):
return
if secret.get('state', 'present') == 'present':
data = policy_data(secret['file'], secret.get('vars', {}), opt)
write_policy(policy_name, data, client, opt)
else:
log('removing policy %s' % policy_name, opt)
client.delete_policy(policy_name)
def freeze(self, tmp_dir):
"""Copies a secret into a particular location"""
for sfile in self.secrets():
src_file = hard_path(sfile, self.opt.secrets)
if not os.path.exists(src_file):
raise aomi.exceptions.IceFile("%s secret not found at %s" %
(self, src_file))
dest_file = "%s/%s" % (tmp_dir, sfile)
dest_dir = os.path.dirname(dest_file)
if not os.path.isdir(dest_dir):
os.mkdir(dest_dir, 0o700)
shutil.copy(src_file, dest_file)
log("Froze %s %s" % (self, sfile), self.opt)
def freeze(dest_dir, opt):
"""Iterates over the Secretfile looking for secrets to freeze"""
if not (os.path.exists(dest_dir) and
os.path.isdir(dest_dir)):
os.mkdir(dest_dir)
tmp_dir = tempfile.mkdtemp('aomi-freeze')
dest_prefix = "%s/dest" % tmp_dir
os.mkdir(dest_prefix)
config = get_secretfile(opt)
freeze_files(config, dest_prefix, opt)
zip_filename = freeze_archive(tmp_dir, dest_prefix)
ice_file = freeze_encrypt(dest_dir, zip_filename, config, opt)
shutil.rmtree(tmp_dir)
log("Generated file is %s" % ice_file, opt)
def decrypt(source, dest, opt):
"""Attempts to decrypt a file"""
cmd = flatten([
gnupg_bin(), "--output", dest, "--decrypt",
gnupg_home(),
passphrase_file(), source
])
# we confirm the source file exists in filez.thaw
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT) # nosec
except subprocess.CalledProcessError as exception:
log("GPG Command %s" % ' '.join(exception.cmd), opt)
log("GPG Output %s" % exception.output, opt)
raise aomi.exceptions.GPG()
return True
def filtered(self):
"""Determines whether or not resource is filtered.
Resources may be filtered if the tags do not match
or the user has specified explict paths to include
or exclude via command line options"""
if not is_tagged(self.tags, self.opt.tags):
log("Skipping %s as it does not have requested tags" % self.path,
self.opt)
return False
if not aomi.validation.specific_path_check(self.path, self.opt):
log("Skipping %s as it does not match specified paths" % self.path,
self.opt)
return False
return True
def mount_path(client, obj, opt):
"""Manage a Vault mountpoint"""
aomi.validation.mount_obj(obj)
path = obj['path']
if not validate_entry(obj, path, opt):
return
backends = client.list_secret_backends()
mounted = is_mounted(path, backends, 'generic')
if obj.get('state', 'present') == 'present':
if not mounted:
actually_mount(client, 'generic', path)
log("Mounted %s" % (path), opt)
else:
if mounted:
unmount(client, 'generic', path)
log("Mounted %s" % (path), opt)
def gitignore(opt):
"""Will check directories upwards from the Secretfile in order
to ensure the gitignore file is set properly"""
directory = os.path.dirname(abspath(opt.secretfile))
gitignore_file = find_file('.gitignore', directory)
if gitignore_file:
secrets_path = subdir_path(abspath(opt.secrets), gitignore_file)
if secrets_path:
if not in_file(secrets_path, gitignore_file):
e_msg = "The path %s was not found in %s" \
% (secrets_path, gitignore_file)
raise aomi.exceptions.AomiFile(e_msg)
else:
log("Using a non-relative secret directory", opt)
else:
raise aomi.exceptions.AomiFile("You should really have a .gitignore")
def encrypt(source, dest, keys, opt):
"""Encrypts a file using the given keys"""
recipients = [["--recipient", key.encode('ASCII')] for key in keys]
cmd = flatten([
gnupg_bin(), "--armor", "--output", dest,
gnupg_home(),
passphrase_file(), recipients, "--encrypt", source
])
# gpg keys are validated in filez.grok_keys
try:
subprocess.check_output(cmd, stderr=subprocess.STDOUT) # nosec
except subprocess.CalledProcessError as exception:
log("GPG Command %s" % ' '.join(exception.cmd), opt)
log("GPG Output %s" % exception.output, opt)
raise aomi.exceptions.GPG()
return True
def app(client, app_obj, opt):
"""Seed an app file into Vault"""
if 'app_file' not in app_obj:
problems("Invalid app definition %s" % app_obj)
name = None
if 'name' in app_obj:
name = app_obj['name']
else:
name = os.path.splitext(os.path.basename(app_obj['app_file']))[0]
if not is_tagged(app_obj.get('tags', []), opt.tags):
log("Skipping %s as it does not have appropriate tags" % name, opt)
return
app_file = hard_path(app_obj['app_file'], opt.secrets)
data = yaml.load(open(app_file).read())
if 'app_id' not in data \
or 'policy' not in data:
problems("Invalid app file %s" % app_file)
policy_name = None
if 'policy_name' in data:
policy_name = data['policy_name']
else:
policy_name = name
policy = open(hard_path(data['policy'], opt.policies), 'r').read()
client.set_policy(name, policy)
app_path = "auth/app-id/map/app-id/%s" % data['app_id']
app_obj = {'value': policy_name, 'display_name': name}
client.write(app_path, **app_obj)
users = data.get('users', [])
for user in users:
if 'id' not in user:
problems("Invalid user definition %s" % user)
user_path = "auth/app-id/map/user-id/%s" % user['id']
user_obj = {'value': data['app_id']}
if 'cidr' in user:
user_obj['cidr_block'] = user['cidr']
client.write(user_path, **user_obj)
log('created %d users in application %s' % (len(users), name), opt)
def token_meta(operation, opt):
"""Generates metadata for a token"""
meta = {'operation': operation, 'hostname': socket.gethostname()}
if 'USER' in os.environ:
meta['unix_user'] = os.environ['USER']
if opt.metadata:
meta_bits = opt.metadata.split(',')
for meta_bit in meta_bits:
k, v = meta_bit.split('=')
if k not in meta:
meta[k] = v
for k, v in meta.items():
log("Token metadata %s %s" % (k, v), opt)
return meta
def grok_keys(config, opt):
"""Will retrieve a GPG key from either Keybase or GPG directly"""
key_ids = []
for key in config['pgp_keys']:
if key.startswith('keybase:'):
key_id = from_keybase(key[8:], opt)
log("Encrypting for keybase user %s" % key[8:], opt)
else:
if not has_gpg_key(key):
raise aomi.exceptions.GPG("Do not actually have key %s" % key)
log("Encrypting for gpg id %s" % key, opt)
key_id = key
validate_gpg_fingerprint(key_id)
key_ids.append(key_id)
return key_ids
def thaw(self, tmp_dir):
"""Will perform some validation and copy a
decrypted secret to it's final location"""
for sfile in self.secrets():
src_file = "%s/%s" % (tmp_dir, sfile)
if not os.path.exists(src_file):
raise aomi \
.exceptions \
.IceFile("%s secret missing from icefile" %
(self))
dest_file = "%s/%s" % (self.opt.secrets, sfile)
dest_dir = os.path.dirname(dest_file)
if not os.path.exists(dest_dir):
os.mkdir(dest_dir)
shutil.copy(src_file, dest_file)
log("Thawed %s %s" % (self, sfile), self.opt)
def thaw(src_file, opt):
"""Given the combination of a Secretfile and the output of
a freeze operation, will restore secrets to usable locations"""
if not os.path.exists(src_file):
raise aomi.exceptions.AomiFile("%s does not exist" % src_file)
tmp_dir = ensure_tmpdir()
zip_file = thaw_decrypt(src_file, tmp_dir, opt)
archive = zipfile.ZipFile(zip_file, 'r')
for archive_file in archive.namelist():
archive.extract(archive_file, tmp_dir)
os.chmod("%s/%s" % (tmp_dir, archive_file), 0o640)
log("Extracted %s from archive" % archive_file, opt)
log("Thawing secrets into %s" % opt.secrets, opt)
config = get_secretfile(opt)
ctx = Context.load(config, opt)
ctx.thaw(tmp_dir)
def generate_obj(self):
"""Generates the secret object, respecting existing information
and user specified options"""
secret_obj = {}
if self.existing:
secret_obj = deepcopy(self.existing)
for key in self.keys:
key_name = key['name']
if self.existing and \
key_name in self.existing and \
not key.get('overwrite'):
log("Not overwriting %s/%s" % (self.path, key_name), self.opt)
continue
else:
secret_obj[key_name] = generated_key(key, self.opt)
return secret_obj
def initial_token(vault_client, opt):
"""Generate our first token based on workstation configuration"""
home = os.environ['HOME'] if 'HOME' in os.environ else \
os.environ['USERPROFILE']
token_file = os.environ.get('VAULT_TOKEN_FILE',
os.path.join(home, ".vault-token"))
app_file = os.environ.get('AOMI_APP_FILE',
os.path.join(home, ".aomi-app-token"))
token_file = abspath(token_file)
app_file = abspath(app_file)
if 'VAULT_TOKEN' in os.environ and os.environ['VAULT_TOKEN']:
log('Token derived from VAULT_TOKEN environment variable', opt)
return os.environ['VAULT_TOKEN'].strip()
elif 'VAULT_USER_ID' in os.environ and \
'VAULT_APP_ID' in os.environ and \
os.environ['VAULT_USER_ID'] and os.environ['VAULT_APP_ID']:
token = app_token(vault_client,
os.environ['VAULT_APP_ID'].strip(),
os.environ['VAULT_USER_ID'].strip())
log("Token derived from VAULT_APP_ID and VAULT_USER_ID", opt)
return token
elif 'VAULT_ROLE_ID' in os.environ and \
'VAULT_SECRET_ID' in os.environ and \
os.environ['VAULT_ROLE_ID'] and os.environ['VAULT_SECRET_ID']:
token = approle_token(vault_client,
os.environ['VAULT_ROLE_ID'],
os.environ['VAULT_SECRET_ID'])
log("Token derived from VAULT_ROLE_ID and VAULT_SECRET_ID", opt)
return token
elif os.path.exists(app_file):
token = yaml.safe_load(open(app_file).read().strip())
if 'app_id' in token and 'user_id' in token:
token = app_token(vault_client,
token['app_id'],
token['user_id'])
log("Token derived from %s" % app_file, opt)
return token
elif os.path.exists(token_file):
log("Token derived from %s" % token_file, opt)
return open(token_file, 'r').read().strip()
else:
raise aomi.exceptions.AomiCredentials('unknown method')