def get_real_file(self, file_path):
"""
If the file is vault encrypted return a path to a temporary decrypted file
If the file is not encrypted then the path is returned
Temporary files are cleanup in the destructor
"""
if not file_path or not isinstance(file_path, string_types):
raise AnsibleParserError("Invalid filename: '%s'" %
to_native(file_path))
b_file_path = to_bytes(file_path, errors='surrogate_or_strict')
if not self.path_exists(b_file_path) or not self.is_file(b_file_path):
raise AnsibleFileNotFound(
"the file_name '%s' does not exist, or is not readable" %
to_native(file_path))
if not self._vault:
self._vault = VaultLib(password="")
real_path = self.path_dwim(file_path)
try:
with open(to_bytes(real_path), 'rb') as f:
# Limit how much of the file is read since we do not know
# whether this is a vault file and therefore it could be very
# large.
if is_encrypted_file(f, count=len(b_HEADER)):
# if the file is encrypted and no password was specified,
# the decrypt call would throw an error, but we check first
# since the decrypt function doesn't know the file name
data = f.read()
if not self._vault_password:
raise AnsibleParserError(
"A vault password must be specified to decrypt %s"
% file_path)
data = self._vault.decrypt(data, filename=real_path)
# Make a temp file
real_path = self._create_content_tempfile(data)
self._tempfiles.add(real_path)
return real_path
except (IOError, OSError) as e:
raise AnsibleParserError(
"an error occurred while trying to read the file '%s': %s" %
(to_native(real_path), to_native(e)))
def load_yaml(yaml_file, vault_secret=None):
"""
Load a YAML file into a python dictionary.
The YAML file can be fully encrypted by Ansible-Vault or can contain
multiple inline Ansible-Vault encrypted values. Ansible Vault
encryption is ideal to store passwords or encrypt the entire file
with sensitive data if required.
"""
vault = VaultLib()
if vault_secret:
secret_file = get_file_vault_secret(filename=vault_secret,
loader=DataLoader())
secret_file.load()
vault.secrets = [('default', secret_file)]
data = None
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
# Render environment variables using jinja templates
contents = stream.read()
template = Template(contents)
stream = StringIO(template.render(env_var=os.environ))
try:
if is_encrypted_file(stream):
file_data = stream.read()
data = yaml.load(vault.decrypt(file_data, None))
else:
loader = AnsibleLoader(stream, None, vault.secrets)
try:
data = loader.get_single_data()
except Exception as exc:
raise Exception(
f'Error when loading YAML config at {yaml_file} {exc}'
)
finally:
loader.dispose()
except yaml.YAMLError as exc:
raise Exception(
f'Error when loading YAML config at {yaml_file} {exc}')
else:
LOGGER.debug('No file at %s', yaml_file)
return data
def load_yaml(yaml_file, vault_secret=None):
'''
Load a YAML file into a python dictionary.
The YAML file can be fully encrypted by Ansible-Vault or can contain
multiple inline Ansible-Vault encrypted values. Ansible Vault
encryption is ideal to store passwords or encrypt the entire file
with sensitive data if required.
'''
vault = VaultLib()
if vault_secret:
secret_file = get_file_vault_secret(filename=vault_secret,
loader=DataLoader())
secret_file.load()
vault.secrets = [('default', secret_file)]
data = None
if os.path.isfile(yaml_file):
with open(yaml_file, 'r') as stream:
try:
if is_encrypted_file(stream):
file_data = stream.read()
data = yaml.load(vault.decrypt(file_data, None))
else:
loader = AnsibleLoader(stream, None, vault.secrets)
try:
data = loader.get_single_data()
except Exception as exc:
raise Exception(
"Error when loading YAML config at {} {}".format(
yaml_file, exc))
finally:
loader.dispose()
except yaml.YAMLError as exc:
raise Exception(
"Error when loading YAML config at {} {}".format(
yaml_file, exc))
else:
logger.debug("No file at {}".format(yaml_file))
return data
def get_real_file(self, file_path):
"""
If the file is vault encrypted return a path to a temporary decrypted file
If the file is not encrypted then the path is returned
Temporary files are cleanup in the destructor
"""
if not file_path or not isinstance(file_path, string_types):
raise AnsibleParserError("Invalid filename: '%s'" % to_native(file_path))
b_file_path = to_bytes(file_path, errors='surrogate_or_strict')
if not self.path_exists(b_file_path) or not self.is_file(b_file_path):
raise AnsibleFileNotFound("the file_name '%s' does not exist, or is not readable" % to_native(file_path))
if not self._vault:
self._vault = VaultLib(password="")
real_path = self.path_dwim(file_path)
try:
with open(to_bytes(real_path), 'rb') as f:
# Limit how much of the file is read since we do not know
# whether this is a vault file and therefore it could be very
# large.
if is_encrypted_file(f, count=len(b_HEADER)):
# if the file is encrypted and no password was specified,
# the decrypt call would throw an error, but we check first
# since the decrypt function doesn't know the file name
data = f.read()
if not self._vault_password:
raise AnsibleParserError("A vault password must be specified to decrypt %s" % file_path)
data = self._vault.decrypt(data, filename=real_path)
# Make a temp file
real_path = self._create_content_tempfile(data)
self._tempfiles.add(real_path)
return real_path
except (IOError, OSError) as e:
raise AnsibleParserError("an error occurred while trying to read the file '%s': %s" % (to_native(real_path), to_native(e)))
def test_utf8_encrypted(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
b_data = data.encode('utf8')
b_data_fo = io.BytesIO(b_data)
self.assertTrue(vault.is_encrypted_file(b_data_fo))
def test_binary_file_handle_invalid(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
b_data = to_bytes(data)
b_data_fo = io.BytesIO(b_data)
self.assertFalse(vault.is_encrypted_file(b_data_fo))
def test_file_with_offset_and_count(self):
b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
vault_length = len(b_data)
b_data = b'JUNK' + b_data + u'ァ ア'.encode('utf-8')
b_data_fo = io.BytesIO(b_data)
self.assertTrue(vault.is_encrypted_file(b_data_fo, start_pos=4, count=vault_length))
def test_utf8_not_encrypted(self):
b_data = "foobar".encode('utf8')
b_data_fo = io.BytesIO(b_data)
self.assertFalse(vault.is_encrypted_file(b_data_fo))
def test_file_already_read_from_finds_header(self):
b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(
b"ansible\ntesting\nfile pos")
b_data_fo = io.BytesIO(b_data)
b_data_fo.read(42) # Arbitrary number
self.assertTrue(vault.is_encrypted_file(b_data_fo))
def main(password_file, varsfile, code_path, dry_run, keep_backups, debug):
"""(Re)keys Ansible Vault repos."""
if debug:
log_console.setLevel(logging.DEBUG)
if not os.path.isdir(code_path):
log.error("{} doesn't seem to exist".format(code_path))
sys.exit(1)
code_path = os.path.realpath(code_path)
backup_path = os.path.join(code_path, ".rekey-backups")
log.debug('Backup path set to: {}'.format(backup_path))
if not password_file:
password_file = os.path.join(code_path, 'vault-password.txt')
else:
if not os.path.isfile(password_file):
log.error("{} doesn't seem to exist".format(password_file))
sys.exit(1)
password_file = os.path.realpath(password_file)
# find all files
files = [os.path.realpath(varsfile)
] if varsfile else rekey.find_files(code_path)
vault_files = []
for f in files:
with open(f, 'rb') as stream:
if is_encrypted_file(stream):
vault_files.append({'file': f})
continue
if b'$ANSIBLE_VAULT;1.1;AES256' in stream.read():
# inline secrets
try:
data = rekey.parse_yaml(f)
except Exception as e:
log.warning(
'Unable to parse file, probably not valid yaml: {} {}'.
format(happy_relpath(f), e))
continue
# enh, generator. w/e.
secrets = list(rekey.find_yaml_secrets(data)) if data else None
if secrets and len(secrets) > 0:
vault_files.append({'file': f, 'secrets': secrets})
vflog = []
for i in vault_files:
suffix = " (whole)" if 'secrets' not in i.keys() else ""
vflog.append("{}{}".format(happy_relpath(i['file']), suffix))
log.info('Found {} vault-enabled files: {}'.format(len(vflog),
', '.join(vflog)))
log.info('Backing up encrypted and password files...')
# backup password file
rekey.backup_files([password_file], backup_path, code_path)
# decrypt and write files out to unencbackup location (same relative paths)
for f in vault_files:
newpath = os.path.join(backup_path, f['file'][len(code_path) + 1:])
log.debug('Decrypting {} to {} using {}'.format(
happy_relpath(f['file']), happy_relpath(newpath),
happy_relpath(password_file)))
rekey.decrypt_file(f['file'], password_file, newpath)
# generate new password file
log.info('Generating new password file...')
if dry_run:
log.info('>> Dry run enabled, skipping overwrite. <<')
else:
rekey.write_password_file(password_file, overwrite=True)
log.info('Password file written: {}'.format(
happy_relpath(password_file)))
# loop through encrypted asset list, re-encrypt and overwrite originals
log.info('Re-encrypting assets with new password file...')
for f in vault_files:
# log.debug('Raw file obj: {}'.format(f))
oldpath = os.path.join(backup_path, happy_relpath(f['file']))
newpath = os.path.realpath(f['file'])
log.debug('Encrypting {} to {}'.format(happy_relpath(oldpath),
happy_relpath(newpath)))
if dry_run:
log.info('>> Dry run enabled, skipping overwrite. <<')
r = True
else:
r = rekey.encrypt_file(oldpath, password_file, newpath,
f.get('secrets', None))
if not r:
log.error('Encryption failed on {}'.format(oldpath))
# test decryption of newly written assets?
# remove backups
if not keep_backups:
log.info('Removing backups...')
shutil.rmtree(backup_path)
log.info('Done!')
def test_binary_file_handle_encrypted(self):
b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
b_data_fo = io.BytesIO(b_data)
self.assertTrue(vault.is_encrypted_file(b_data_fo))
def test_binary_file_handle_invalid(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
b_data = to_bytes(data)
b_data_fo = io.BytesIO(b_data)
self.assertFalse(vault.is_encrypted_file(b_data_fo))
def test_binary_file_handle_not_encrypted(self):
b_data = b"foobar"
b_data_fo = io.BytesIO(b_data)
self.assertFalse(vault.is_encrypted_file(b_data_fo))
def test_text_file_handle_encrypted(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % to_text(hexlify(b"ansible"))
data_fo = io.StringIO(data)
self.assertTrue(vault.is_encrypted_file(data_fo))
def test_binary_file_handle_encrypted(self):
b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible")
b_data_fo = io.BytesIO(b_data)
self.assertTrue(vault.is_encrypted_file(b_data_fo))
def test_text_file_handle_not_encrypted(self):
data = u"foobar"
data_fo = io.StringIO(data)
self.assertFalse(vault.is_encrypted_file(data_fo))
def test_binary_file_handle_not_encrypted(self):
b_data = b"foobar"
b_data_fo = io.BytesIO(b_data)
self.assertFalse(vault.is_encrypted_file(b_data_fo))
def test_text_file_handle_not_encrypted(self):
data = u"foobar"
data_fo = io.StringIO(data)
self.assertFalse(vault.is_encrypted_file(data_fo))
def test_text_file_handle_invalid(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
data_fo = io.StringIO(data)
self.assertFalse(vault.is_encrypted_file(data_fo))
def test_text_file_handle_encrypted(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % to_text(hexlify(b"ansible"))
data_fo = io.StringIO(data)
self.assertTrue(vault.is_encrypted_file(data_fo))
def test_file_already_read_from_finds_header(self):
b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
b_data_fo = io.BytesIO(b_data)
b_data_fo.read(42) # Arbitrary number
self.assertTrue(vault.is_encrypted_file(b_data_fo))
def test_text_file_handle_invalid(self):
data = u"$ANSIBLE_VAULT;9.9;TEST\n%s" % u"ァ ア ィ イ ゥ ウ ェ エ ォ オ カ ガ キ ギ ク グ ケ "
data_fo = io.StringIO(data)
self.assertFalse(vault.is_encrypted_file(data_fo))
def test_file_already_read_from_saves_file_pos(self):
b_data = b"$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
b_data_fo = io.BytesIO(b_data)
b_data_fo.read(69) # Arbitrary number
vault.is_encrypted_file(b_data_fo)
self.assertEqual(b_data_fo.tell(), 69)
def test_file_with_offset(self):
b_data = b"JUNK$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(
b"ansible\ntesting\nfile pos")
b_data_fo = io.BytesIO(b_data)
self.assertTrue(vault.is_encrypted_file(b_data_fo, start_pos=4))
def test_file_with_offset(self):
b_data = b"JUNK$ANSIBLE_VAULT;9.9;TEST\n%s" % hexlify(b"ansible\ntesting\nfile pos")
b_data_fo = io.BytesIO(b_data)
self.assertTrue(vault.is_encrypted_file(b_data_fo, start_pos=4))
