class TestDataLoaderWithVault(unittest.TestCase):
def setUp(self):
self._loader = DataLoader()
vault_secrets = [('default', TextVaultSecret('ansible'))]
self._loader.set_vault_secrets(vault_secrets)
def tearDown(self):
pass
@patch.multiple(DataLoader,
path_exists=lambda s, x: True,
is_file=lambda s, x: True)
def test_parse_from_vault_1_1_file(self):
vaulted_data = """$ANSIBLE_VAULT;1.1;AES256
33343734386261666161626433386662623039356366656637303939306563376130623138626165
6436333766346533353463636566313332623130383662340a393835656134633665333861393331
37666233346464636263636530626332623035633135363732623332313534306438393366323966
3135306561356164310a343937653834643433343734653137383339323330626437313562306630
3035
"""
if PY3:
builtins_name = 'builtins'
else:
builtins_name = '__builtin__'
with patch(builtins_name + '.open',
mock_open(read_data=vaulted_data.encode('utf-8'))):
output = self._loader.load_from_file('dummy_vault.txt')
self.assertEqual(output, dict(foo='bar'))
def _play_prereqs():
options = context.CLIARGS
# all needs loader
loader = DataLoader()
basedir = options.get('basedir', False)
if basedir:
loader.set_basedir(basedir)
add_all_plugin_dirs(basedir)
set_collection_playbook_paths(basedir)
vault_ids = list(options['vault_ids'])
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
vault_secrets = CLI.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=list(options['vault_password_files']),
ask_vault_pass=options['ask_vault_pass'],
auto_prompt=False)
loader.set_vault_secrets(vault_secrets)
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader, sources=options['inventory'])
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager(loader=loader, inventory=inventory, version_info=CLI.version_info(gitinfo=False))
return loader, inventory, variable_manager
def get_ansible_variablemanager(self, **kwargs):
from ansible.parsing.dataloader import DataLoader
from ansible.utils.vars import load_extra_vars
from ansible.vars.manager import VariableManager
if 'options' in kwargs:
options = kwargs['options']
else:
options = AnsibleOptions()
if 'loader' in kwargs:
loader = kwargs['loader']
else:
loader = DataLoader()
vault_secret = get_vault_password_source(self.master.main_config)
if not isinstance(vault_secret, NullSource):
loader.set_vault_secrets([(vault_secret.id, vault_secret)])
basedir = get_playbooks_directory(self.master.ctrl.config)
loader.set_basedir(basedir)
if 'inventory' in kwargs:
inventory = kwargs['inventory']
else:
inventory = self.get_ansible_inventorymanager()
if 'variable_manager' in kwargs:
variable_manager = kwargs['variable_manager']
else:
variable_manager = VariableManager(loader=loader, inventory=inventory)
variable_manager.extra_vars = load_extra_vars(loader=loader, options=options)
return (options, loader, inventory, variable_manager)
def _configure_root(self, rootResource):
rootResource.imports = self.imports
if (
self.manifest.vault and self.manifest.vault.secrets
): # setBaseDir() may create a new templar
rootResource._templar._loader.set_vault_secrets(self.manifest.vault.secrets)
rootResource.envRules = self.context.get("variables") or CommentedMap()
if not self.localEnv:
return
# use the password associated with the project the repository appears in.
repos = set(self.repositories.values())
project = self.localEnv.project or self.localEnv.homeProject
while project:
loader = DataLoader()
vault = project.make_vault_lib()
if vault:
yaml = make_yaml(vault)
loader.set_vault_secrets(vault.secrets)
for repoview in project.workingDirs.values():
if repoview in repos:
repoview.load_secrets(loader)
repoview.yaml = yaml
repos.remove(repoview)
project = project.parentProject
# left over:
for repository in repos:
repository.load_secrets(rootResource._templar._loader)
repository.yaml = self.yaml
def get_play_prereqs_2_4(self, options):
loader = DataLoader()
if self.vault_pass:
loader.set_vault_secrets([
('default', VaultSecret(_bytes=to_bytes(self.vault_pass)))
])
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader, sources=options.inventory)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
try:
# Ansible 2.8
variable_manager = VariableManager(
loader=loader,
inventory=inventory,
version_info=self.version_info(ansible_version))
variable_manager._extra_vars = self.extra_vars
except TypeError:
variable_manager = VariableManager(loader=loader,
inventory=inventory)
variable_manager.extra_vars = self.extra_vars
variable_manager.options_vars = {
'ansible_version': self.version_info(ansible_version)
}
return loader, inventory, variable_manager
def _prepare_for_run(self, host, extra_vars, playbook):
module_path = f'{self._base_plays_path}/modules'
options = InventoryOptions(
remote_user=self._remote_user,
private_key_file=self._private_key_file,
module_path=module_path)
loader = DataLoader()
loader.set_vault_secrets(self._default_secret)
inventory = InventoryManager(loader=loader, sources=f'{host},')
variable_manager = VariableManager(loader=loader, inventory=inventory)
variable_manager.extra_vars = extra_vars
self.play_executor = PlaybookExecutor(
playbooks=[playbook],
options=options,
loader=loader,
inventory=inventory,
variable_manager=variable_manager,
passwords={}
)
self.logger_cb = LogCallBack(self._verbosity)
# TODO: make a PR to ansible core to allowing access to task queue
# manager property in order to get all execution logs
self.play_executor._tqm._stdout_callback = self.logger_cb
def _play_prereqs():
options = context.CLIARGS
# all needs loader
loader = DataLoader()
basedir = options.get('basedir', False)
if basedir:
loader.set_basedir(basedir)
add_all_plugin_dirs(basedir)
AnsibleCollectionConfig.playbook_paths = basedir
default_collection = _get_collection_name_from_path(basedir)
if default_collection:
display.warning(u'running with default collection {0}'.format(default_collection))
AnsibleCollectionConfig.default_collection = default_collection
vault_ids = list(options['vault_ids'])
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
vault_secrets = CLI.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=list(options['vault_password_files']),
ask_vault_pass=options['ask_vault_pass'],
auto_prompt=False)
loader.set_vault_secrets(vault_secrets)
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader, sources=options['inventory'], cache=(not options.get('flush_cache')))
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager(loader=loader, inventory=inventory, version_info=CLI.version_info(gitinfo=False))
return loader, inventory, variable_manager
def _play_prereqs(options):
# all needs loader
loader = DataLoader()
basedir = getattr(options, 'basedir', False)
if basedir:
loader.set_basedir(basedir)
vault_ids = options.vault_ids
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
vault_secrets = CLI.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=options.vault_password_files,
ask_vault_pass=options.ask_vault_pass,
auto_prompt=False)
loader.set_vault_secrets(vault_secrets)
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader, sources=options.inventory)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager(loader=loader, inventory=inventory)
# load vars from cli options
variable_manager.extra_vars = load_extra_vars(loader=loader, options=options)
variable_manager.options_vars = load_options_vars(options, CLI.version_info(gitinfo=False))
return loader, inventory, variable_manager
class Runner(object):
def __init__(self, hosts_file, playbook_file, cfg_file=None, vault_id=None, private_key_file=None, module_path=None, become_pass=None, verbosity=0):
self.hosts_file = hosts_file
self.playbook_file = playbook_file
self.vault_id = vault_id
if cfg_file:
os.environ['ANSIBLE_CONFIG']=cfg_file
self.options = Options()
if private_key_file:
self.options.private_key_file = private_key_file
self.options.verbosity = verbosity
if module_path:
self.options.module_path = module_path
self.options.connection = 'ssh'
# set verbosity
self.display = Display()
self.display.verbosity = self.options.verbosity
playbook_executor.verbosity = self.options.verbosity
# Become Pass Needed if not logging in as user root
if become_pass:
self.options.become = True
self.options.become_method = 'sudo'
self.options.become_user = '******'
passwords = {'become_pass': become_pass}
else:
passwords = {}
# Gets data from YAML/JSON files
self.loader = DataLoader()
# vault secrets valut_id=name@password
vault_secrets = []
if self.vault_id:
vault_ids = vault_id.split(',', vault_id) if ',' in vault_id else [vault_id]
for vid in vault_ids:
vault_secrets.append(tuple(vid.split(',',1)))
if vault_secrets:
self.loader.set_vault_secrets(vault_secrets)
# Set inventory, using most of above objects
self.inventory = InventoryManager(loader=self.loader, sources=self.hosts_file)
self.variable_manager = VariableManager(loader=self.loader, inventory=self.inventory)
# Setup playbook executor, but don't run until run() called
self.pbex = playbook_executor.PlaybookExecutor(
playbooks=[self.playbook_file],
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords=passwords)
def run(self):
# Results of PlaybookExecutor
self.pbex.run()
def _play_prereqs(options):
# all needs loader
loader = DataLoader()
vault_ids = options.vault_ids
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
vault_secrets = CLI.setup_vault_secrets(
loader,
vault_ids=vault_ids,
vault_password_files=options.vault_password_files,
ask_vault_pass=options.ask_vault_pass,
auto_prompt=False)
loader.set_vault_secrets(vault_secrets)
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader, sources=options.inventory)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager(loader=loader, inventory=inventory)
# load vars from cli options
variable_manager.extra_vars = load_extra_vars(loader=loader,
options=options)
variable_manager.options_vars = load_options_vars(
options, CLI.version_info(gitinfo=False))
return loader, inventory, variable_manager
def runplaybook(inventory, playbook, project, bname, secret):
loader = DataLoader()
invlist = []
invlist.append(inventory)
inv = InventoryManager(loader=loader, sources=invlist)
variable_manager = VariableManager(loader=loader, inventory=inv)
pblist = []
pblist.append(playbook)
if secret != '':
loader.set_vault_secrets([('default',
VaultSecret(_bytes=to_bytes(secret)))])
passwords = {}
Options = namedtuple('Options', [
'connection', 'remote_user', 'ask_sudo_pass', 'verbosity', 'ack_pass',
'module_path', 'forks', 'become', 'become_method', 'become_user',
'check', 'listhosts', 'listtasks', 'listtags', 'syntax', 'sudo_user',
'sudo', 'diff'
])
options = Options(connection='smart',
remote_user=None,
ack_pass=None,
sudo_user=None,
forks=5,
sudo=None,
ask_sudo_pass=False,
verbosity=5,
module_path=None,
become=None,
become_method=None,
become_user=None,
check=False,
diff=False,
listhosts=None,
listtasks=None,
listtags=None,
syntax=None)
pb = PlaybookExecutor(loader=loader,
playbooks=pblist,
inventory=inv,
variable_manager=variable_manager,
passwords=passwords,
options=options)
f = open('/opt/Projects/' + project + '/logs/' + bname + 'out.log', 'a+')
# ferr=open('/opt/Projects/'+project+'/logs/'+bname+'err.log','a+')
sys.stdout = f
# sys.stderr = ferr
result = pb.run()
# ferr.close()
f.close()
return result
def ansibleHelper(vault=None):
'''
Creates the ansible inventory object
TODO: Split out into its own class???
'''
global ANSIBLE_DL
ANSIBLE_DL = DataLoader()
if vault is not None:
ANSIBLE_DL.set_vault_secrets(vault)
inv = InventoryManager(loader=ANSIBLE_DL, sources=[ANSIBLE_HOSTS])
return inv
def run(self):
loader = DataLoader()
vault_ids = self.options.vault_ids
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
encrypt_vault_id = None
vault_secrets = self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
vault_pass=self.options.vault_pass)
encrypt_secret = match_encrypt_secret(vault_secrets,
encrypt_vault_id=encrypt_vault_id)
self.encrypt_vault_id = encrypt_secret[0]
self.encrypt_secret = encrypt_secret[1]
loader.set_vault_secrets(vault_secrets)
vault = VaultLib(vault_secrets)
self.editor = VaultEditor(vault)
def inventory_load(inventory_sources, vault_secret):
""" Load the inventory
"""
loader = DataLoader()
vault_secrets = [('default', TextVaultSecret(vault_secret))]
loader.set_vault_secrets(vault_secrets)
inventory = InventoryManager(loader=loader, sources=inventory_sources)
result = {}
for hostname in inventory.hosts:
host = inventory.get_host(hostname)
variable_manager = VariableManager(loader=loader, inventory=inventory)
magic_vars = ['ansible_playbook_python', 'groups', 'group_names', 'inventory_dir',
'inventory_file', 'inventory_hostname', 'inventory_hostname_short',
'omit', 'playbook_dir']
all_vars = variable_manager.get_vars(host=host, include_hostvars=True)
cleaned = ({k: v for (k, v) in all_vars.items() if k not in magic_vars})
result[hostname] = cleaned
return result
def _play_prereqs():
options = context.CLIARGS
# all needs loader
loader = DataLoader()
basedir = options.get('basedir', False)
if basedir:
loader.set_basedir(basedir)
vault_ids = list(options['vault_ids'])
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
vault_secrets = CLI.setup_vault_secrets(
loader,
vault_ids=vault_ids,
vault_password_files=list(options['vault_password_files']),
ask_vault_pass=options['ask_vault_pass'],
auto_prompt=False)
loader.set_vault_secrets(vault_secrets)
# create the inventory, and filter it based on the subset specified (if any)
inventory = InventoryManager(loader=loader,
sources=options['inventory'])
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager(loader=loader, inventory=inventory)
# If the basedir is specified as the empty string then it results in cwd being used. This
# is not a safe location to load vars from
if options.get('basedir', False) is not False:
if basedir:
variable_manager.safe_basedir = True
else:
variable_manager.safe_basedir = True
# load vars from cli options
variable_manager.extra_vars = load_extra_vars(loader=loader)
variable_manager.options_vars = load_options_vars(
CLI.version_info(gitinfo=False))
return loader, inventory, variable_manager
def _initialize_ssh_agent(self, password):
"""Initializes the SSH agent and loads the required private keys.
This to prevent private key material being stored on disk."""
dl = DataLoader()
dl.set_vault_secrets(self._initialize_vault_secrets(self, password))
# databaaaaaaase
# maybe make distinction between multiple secrets depending on deploy
ds = dl.load_from_file('/etc/ansible/key.ssh')
key = self._return_valid_key(self, bytes(ds, encoding='utf-8'))
ssh_add_cmd = "ssh-add -"
ret = Popen((ssh_add_cmd.split()), stdin=PIPE)
ret.communicate(key)
if ret.returncode:
raise OSError('--- something went wrong while loading the key')
def _initialize_play(cls,
play_source,
vault_key,
canary,
added_by,
source,
spath=None):
"""Initializes Ansible playbook with the default and required
options."""
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'become', 'become_method',
'become_user', 'check', 'diff'
])
options = Options(connection='ssh',
module_path=False,
forks=10,
become=False,
become_method='sudo',
become_user='******',
check=False,
diff=False)
loader = DataLoader()
vault_secrets = cls._initialize_vault_secrets(cls, vault_key)
loader.set_vault_secrets(vault_secrets)
inventory = InventoryManager(loader=loader, sources=source)
if not inventory.parse_source(source, loader):
raise ValueError('Invalid inventory source specified.')
variable_manager = VariableManager(loader=loader, inventory=inventory)
play = Play().load(play_source,
variable_manager=variable_manager,
loader=loader)
return cls._run_task(play, inventory, variable_manager, loader,
options, canary, added_by, play_source, spath)
def __init__(
self,
ansibleinventory,
ansible_vault_password_file,
moleculeenv):
# Leverage the ansible python api
# to run a playbook against a molecule host.
#
# see: ansible python api
# https://docs.ansible.com/ansible/latest/dev_guide/developing_api.html
self._moleculeenv = moleculeenv
context.CLIARGS = ImmutableDict(
connection='local',
module_path=[''],
forks=10,
become=None,
become_method=None,
become_user=None,
check=False,
diff=False)
loader = DataLoader()
# Load ansible vault secrets if environment variable is set
if ansible_vault_password_file:
vault_id_name = 'default'
file_vault_secret = get_file_vault_secret(
filename=ansible_vault_password_file,
vault_id=vault_id_name,
loader=loader)
file_vault_secret.load()
loader.set_vault_secrets(
[(ansible_vault_password_file, file_vault_secret)])
self._loader = loader
self._inventory = ansibleinventory
self._variable_manager = VariableManager(
loader=loader,
inventory=ansibleinventory)
def read_vault_yaml(path: str) -> dict:
"""Read YAML with vault-encrypted values.
"""
# Read YAML without decrypting.
raw_clean = open(path).read().replace('!vault', '')
data = yaml.load(raw_clean, Loader=yaml.FullLoader)
# Pop out PW file, if provided.
pw_file = data.get('vault_password_file')
pw_files = [pw_file] if pw_file else None
loader = DataLoader()
vault_secrets = CLI.setup_vault_secrets(
loader=loader,
vault_ids=C.DEFAULT_VAULT_IDENTITY_LIST,
vault_password_files=pw_files,
)
loader.set_vault_secrets(vault_secrets)
# Re-read with decryption.
return loader.load_from_file(path)
class TestDataLoaderWithVault(unittest.TestCase):
def setUp(self):
self._loader = DataLoader()
vault_secrets = [('default', TextVaultSecret('ansible'))]
self._loader.set_vault_secrets(vault_secrets)
self.test_vault_data_path = os.path.join(os.path.dirname(__file__),
'fixtures', 'vault.yml')
def tearDown(self):
pass
def test_get_real_file_vault(self):
real_file_path = self._loader.get_real_file(self.test_vault_data_path)
self.assertTrue(os.path.exists(real_file_path))
def test_get_real_file_vault_no_vault(self):
self._loader.set_vault_secrets(None)
self.assertRaises(AnsibleParserError, self._loader.get_real_file,
self.test_vault_data_path)
def test_get_real_file_vault_wrong_password(self):
wrong_vault = [('default', TextVaultSecret('wrong_password'))]
self._loader.set_vault_secrets(wrong_vault)
self.assertRaises(AnsibleVaultError, self._loader.get_real_file,
self.test_vault_data_path)
def test_get_real_file_not_a_path(self):
self.assertRaisesRegexp(AnsibleParserError, 'Invalid filename',
self._loader.get_real_file, None)
@patch.multiple(DataLoader,
path_exists=lambda s, x: True,
is_file=lambda s, x: True)
def test_parse_from_vault_1_1_file(self):
vaulted_data = """$ANSIBLE_VAULT;1.1;AES256
33343734386261666161626433386662623039356366656637303939306563376130623138626165
6436333766346533353463636566313332623130383662340a393835656134633665333861393331
37666233346464636263636530626332623035633135363732623332313534306438393366323966
3135306561356164310a343937653834643433343734653137383339323330626437313562306630
3035
"""
if PY3:
builtins_name = 'builtins'
else:
builtins_name = '__builtin__'
with patch(builtins_name + '.open',
mock_open(read_data=vaulted_data.encode('utf-8'))):
output = self._loader.load_from_file('dummy_vault.txt')
self.assertEqual(output, dict(foo='bar'))
def applyTemplate(value, ctx, overrides=None):
if not isinstance(value, six.string_types):
msg = "Error rendering template: source must be a string, not %s" % type(
value)
if ctx.strict:
raise UnfurlError(msg)
else:
return "<<%s>>" % msg
value = value.strip()
# implementation notes:
# see https://github.com/ansible/ansible/test/units/template/test_templar.py
# dataLoader is only used by _lookup and to set _basedir (else ./)
if not ctx.templar or (ctx.baseDir
and ctx.templar._basedir != ctx.baseDir):
# we need to create a new templar
loader = DataLoader()
if ctx.baseDir:
loader.set_basedir(ctx.baseDir)
if ctx.templar and ctx.templar._loader._vault.secrets:
loader.set_vault_secrets(ctx.templar._loader._vault.secrets)
templar = Templar(loader)
ctx.templar = templar
else:
templar = ctx.templar
overrides = Templar.findOverrides(value, overrides)
if overrides:
# returns the original values
overrides = templar._applyTemplarOverrides(overrides)
templar.environment.trim_blocks = False
# templar.environment.lstrip_blocks = False
fail_on_undefined = ctx.strict
vars = _VarTrackerDict(__unfurl=ctx)
vars.update(ctx.vars)
vars.ctx = ctx
# replaces current vars
# don't use setter to avoid isinstance(dict) check
templar._available_variables = vars
oldvalue = value
index = ctx.referenced.start()
# set referenced to track references (set by Ref.resolve)
# need a way to turn on and off
try:
# strip whitespace so jinija native types resolve even with extra whitespace
# disable caching so we don't need to worry about the value of a cached var changing
# use do_template because we already know it's a template
try:
value = templar.template(value,
fail_on_undefined=fail_on_undefined)
except Exception as e:
value = "<<Error rendering template: %s>>" % str(e)
if ctx.strict:
logger.debug(value, exc_info=True)
raise UnfurlError(value)
else:
logger.warning(value[2:100] +
"... see debug log for full report")
logger.debug(value, exc_info=True)
else:
if value != oldvalue:
ctx.trace("successfully processed template:", value)
for result in ctx.referenced.getReferencedResults(index):
if isSensitive(result):
# note: even if the template rendered a list or dict
# we still need to wrap the entire result as sensitive because we
# don't know how the referenced senstive results were transformed by the template
ctx.trace("setting template result as sensitive")
return wrapSensitiveValue(
value, templar._loader._vault
) # mark the template result as sensitive
if isinstance(value, Results):
# mapValue now because wrap_var() fails with Results types
value = mapValue(value, ctx, applyTemplates=False)
# wrap result as AnsibleUnsafe so it isn't evaluated again
return wrap_var(value)
finally:
ctx.referenced.stop()
if overrides:
# restore original values
templar._applyTemplarOverrides(overrides)
return value
def run(self):
super(VaultCLI, self).run()
loader = DataLoader()
# set default restrictive umask
old_umask = os.umask(0o077)
vault_ids = list(context.CLIARGS['vault_ids'])
# there are 3 types of actions, those that just 'read' (decrypt, view) and only
# need to ask for a password once, and those that 'write' (create, encrypt) that
# ask for a new password and confirm it, and 'read/write (rekey) that asks for the
# old password, then asks for a new one and confirms it.
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
action = context.CLIARGS['action']
# TODO: instead of prompting for these before, we could let VaultEditor
# call a callback when it needs it.
if action in ['decrypt', 'view', 'rekey', 'edit']:
vault_secrets = self.setup_vault_secrets(loader, vault_ids=vault_ids,
vault_password_files=list(context.CLIARGS['vault_password_files']),
ask_vault_pass=context.CLIARGS['ask_vault_pass'])
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
if action in ['encrypt', 'encrypt_string', 'create']:
encrypt_vault_id = None
# no --encrypt-vault-id context.CLIARGS['encrypt_vault_id'] for 'edit'
if action not in ['edit']:
encrypt_vault_id = context.CLIARGS['encrypt_vault_id'] or C.DEFAULT_VAULT_ENCRYPT_IDENTITY
vault_secrets = None
vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=list(context.CLIARGS['vault_password_files']),
ask_vault_pass=context.CLIARGS['ask_vault_pass'],
create_new_password=True)
if len(vault_secrets) > 1 and not encrypt_vault_id:
raise AnsibleOptionsError("The vault-ids %s are available to encrypt. Specify the vault-id to encrypt with --encrypt-vault-id" %
','.join([x[0] for x in vault_secrets]))
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
encrypt_secret = match_encrypt_secret(vault_secrets,
encrypt_vault_id=encrypt_vault_id)
# only one secret for encrypt for now, use the first vault_id and use its first secret
# TODO: exception if more than one?
self.encrypt_vault_id = encrypt_secret[0]
self.encrypt_secret = encrypt_secret[1]
if action in ['rekey']:
encrypt_vault_id = context.CLIARGS['encrypt_vault_id'] or C.DEFAULT_VAULT_ENCRYPT_IDENTITY
# print('encrypt_vault_id: %s' % encrypt_vault_id)
# print('default_encrypt_vault_id: %s' % default_encrypt_vault_id)
# new_vault_ids should only ever be one item, from
# load the default vault ids if we are using encrypt-vault-id
new_vault_ids = []
if encrypt_vault_id:
new_vault_ids = default_vault_ids
if context.CLIARGS['new_vault_id']:
new_vault_ids.append(context.CLIARGS['new_vault_id'])
new_vault_password_files = []
if context.CLIARGS['new_vault_password_file']:
new_vault_password_files.append(context.CLIARGS['new_vault_password_file'])
new_vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=new_vault_ids,
vault_password_files=new_vault_password_files,
ask_vault_pass=context.CLIARGS['ask_vault_pass'],
create_new_password=True)
if not new_vault_secrets:
raise AnsibleOptionsError("A new vault password is required to use Ansible's Vault rekey")
# There is only one new_vault_id currently and one new_vault_secret, or we
# use the id specified in --encrypt-vault-id
new_encrypt_secret = match_encrypt_secret(new_vault_secrets,
encrypt_vault_id=encrypt_vault_id)
self.new_encrypt_vault_id = new_encrypt_secret[0]
self.new_encrypt_secret = new_encrypt_secret[1]
loader.set_vault_secrets(vault_secrets)
# FIXME: do we need to create VaultEditor here? its not reused
vault = VaultLib(vault_secrets)
self.editor = VaultEditor(vault)
context.CLIARGS['func']()
# and restore umask
os.umask(old_umask)
def run(self):
super(VaultCLI, self).run()
loader = DataLoader()
# set default restrictive umask
old_umask = os.umask(0o077)
vault_ids = self.options.vault_ids
# there are 3 types of actions, those that just 'read' (decrypt, view) and only
# need to ask for a password once, and those that 'write' (create, encrypt) that
# ask for a new password and confirm it, and 'read/write (rekey) that asks for the
# old password, then asks for a new one and confirms it.
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
# TODO: instead of prompting for these before, we could let VaultEditor
# call a callback when it needs it.
if self.action in ['decrypt', 'view', 'rekey', 'edit']:
vault_secrets = self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
ask_vault_pass=self.options.ask_vault_pass)
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
if self.action in ['encrypt', 'encrypt_string', 'create']:
if len(vault_ids) > 1:
raise AnsibleOptionsError("Only one --vault-id can be used for encryption")
vault_secrets = None
vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
ask_vault_pass=self.options.ask_vault_pass,
create_new_password=True)
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
encrypt_secret = match_encrypt_secret(vault_secrets)
# only one secret for encrypt for now, use the first vault_id and use its first secret
# self.encrypt_vault_id = list(vault_secrets.keys())[0]
# self.encrypt_secret = vault_secrets[self.encrypt_vault_id][0]
self.encrypt_vault_id = encrypt_secret[0]
self.encrypt_secret = encrypt_secret[1]
if self.action in ['rekey']:
new_vault_ids = []
if self.options.new_vault_id:
new_vault_ids.append(self.options.new_vault_id)
new_vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=new_vault_ids,
vault_password_files=self.options.new_vault_password_files,
ask_vault_pass=self.options.ask_vault_pass,
create_new_password=True)
if not new_vault_secrets:
raise AnsibleOptionsError("A new vault password is required to use Ansible's Vault rekey")
# There is only one new_vault_id currently and one new_vault_secret
new_encrypt_secret = match_encrypt_secret(new_vault_secrets)
self.new_encrypt_vault_id = new_encrypt_secret[0]
self.new_encrypt_secret = new_encrypt_secret[1]
loader.set_vault_secrets(vault_secrets)
# FIXME: do we need to create VaultEditor here? its not reused
vault = VaultLib(vault_secrets)
self.editor = VaultEditor(vault)
self.execute()
# and restore umask
os.umask(old_umask)
class SimpleProvider(object):
__metaclass__ = ABCMeta
def __init__(self, name, general_type=None, box_extra_type=None):
self.env = Environment(name, general_type, box_extra_type)
self.loader = DataLoader()
self.provisioned = False
self.tags = {}
self.extra_vars = {'prudentia_dir': io.prudentia_python_dir()}
self.load_tags()
self.active_user = pwd.getpwuid(os.geteuid())[0]
def boxes(self):
return self.env.boxes.values()
def get_box(self, box_name):
b = self.env.get(box_name)
if not b:
print ('The box \'%s\' you entered does not exists.\n\n' \
'After typing the command press Tab for box suggestions.\n' % box_name)
return None
else:
return b
def _show_current_vars(self):
print('Current set variables:\n%s\n' % '\n'.join(
[n + ' -> ' + str(v) for n, v in self.extra_vars.iteritems()]))
def set_var(self, var, value):
if var in self.extra_vars:
print ('NOTICE: Variable \'{0}\' is already set to this value: \'{1}\' ' \
'and it will be overwritten.'.format(var, self.extra_vars[var]))
self.extra_vars[var] = value
if provisioning.VERBOSITY > 0:
print("Set \'{0}\' -> {1}\n".format(var, value))
def unset_var(self, var):
if not var:
print('Please provide a valid variable name to unset.\n')
self._show_current_vars()
elif var not in self.extra_vars:
print(
'WARNING: Variable \'{0}\' is NOT present so cannot be unset.\n'
.format(var))
self._show_current_vars()
else:
self.extra_vars.pop(var, None)
print("Unset \'{0}\'\n".format(var))
def set_vault_password(self):
vault_pwd = io.input_value('Ansible vault password', hidden=True)
try:
# Ansible 2.4
self.loader.set_vault_secrets(vault_pwd)
except:
# Ansible 2.3
self.loader.set_vault_password(vault_pwd)
def load_vars(self, vars_file):
if not vars_file:
vars_file = io.input_path('path of the variables file')
vars_dict = self.loader.load_from_file(vars_file)
for key, value in vars_dict.items():
self.set_var(key, value)
def add_box(self, box):
self.env.add(box)
self.load_tags(box)
def load_tags(self, box=None):
for b in [box] if box else self.boxes():
if not os.path.exists(b.playbook):
print ('WARNING: Box \'{0}\' points to a NON existing playbook. ' \
'Please `reconfigure` or `unregister` the box.\n'.format(b.name))
else:
plays = Playbook.load(
b.playbook,
variable_manager=provisioning.get_variable_manager(
self.loader),
loader=self.loader).get_plays()
all_tags = set()
for p in plays:
for block in p.compile():
for task in block.block:
all_tags.update(task.tags)
self.tags[b.name] = list(all_tags)
def remove_box(self, box):
if box.name in self.tags:
self.tags.pop(box.name)
return self.env.remove(box)
def register(self):
try:
box = self.define_box()
if box:
self.add_box(box)
print("\nBox %s added." % box)
except Exception as ex:
io.track_error('cannot add box', ex)
@abstractmethod
def define_box(self):
pass
def reconfigure(self, previous_box):
try:
box = self.redefine_box(previous_box)
if box:
self.remove_box(previous_box)
self.add_box(box)
print("\nBox %s reconfigured." % box)
except Exception as ex:
io.track_error('cannot reconfigure box', ex)
@abstractmethod
def redefine_box(self, previous_box):
pass
def unregister(self, box):
self.remove_box(box)
print("\nBox %s removed.\n" % box.name)
def fetch_box_hosts(self, playbook):
ds = self.loader.load_from_file(playbook)
if ds:
return ds[0][
'hosts'] # a playbook is an array of plays we take the first one
def suggest_name(self, hostname):
if hostname not in self.env.boxes:
return hostname
else:
return hostname + '-' + str(random.randint(0, 100))
def provision(self, box, tags):
self.provisioned = provisioning.run_playbook(
playbook_file=box.playbook,
inventory_file=provisioning.generate_inventory(box),
loader=self.loader,
remote_user=box.get_remote_user(),
remote_pass=box.get_remote_pwd(),
transport=box.get_transport(),
extra_vars=self.extra_vars,
only_tags=tags)
@staticmethod
def verbose(value):
if value:
try:
iv = int(value)
except ValueError:
iv = -1
if 0 <= iv <= 4:
provisioning.VERBOSITY = iv
else:
print(
'Verbosity value \'{0}\' not allowed, should be a number between 0 and 4.'
.format(value))
else:
print('Current verbosity: {0}'.format(provisioning.VERBOSITY))
def facts(self, box, regex='*'):
return provisioning.gather_facts(box, regex, self.loader)
def run(self):
super(VaultCLI, self).run()
loader = DataLoader()
# set default restrictive umask
old_umask = os.umask(0o077)
vault_ids = self.options.vault_ids
# there are 3 types of actions, those that just 'read' (decrypt, view) and only
# need to ask for a password once, and those that 'write' (create, encrypt) that
# ask for a new password and confirm it, and 'read/write (rekey) that asks for the
# old password, then asks for a new one and confirms it.
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
# TODO: instead of prompting for these before, we could let VaultEditor
# call a callback when it needs it.
if self.action in ['decrypt', 'view', 'rekey', 'edit']:
vault_secrets = self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
ask_vault_pass=self.options.ask_vault_pass)
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
if self.action in ['encrypt', 'encrypt_string', 'create']:
if len(vault_ids) > 1:
raise AnsibleOptionsError("Only one --vault-id can be used for encryption")
vault_secrets = None
vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
ask_vault_pass=self.options.ask_vault_pass,
create_new_password=True)
if len(vault_secrets) > 1:
raise AnsibleOptionsError("Only one --vault-id can be used for encryption. This includes passwords from configuration and cli.")
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
encrypt_secret = match_encrypt_secret(vault_secrets)
# only one secret for encrypt for now, use the first vault_id and use its first secret
# self.encrypt_vault_id = list(vault_secrets.keys())[0]
# self.encrypt_secret = vault_secrets[self.encrypt_vault_id][0]
self.encrypt_vault_id = encrypt_secret[0]
self.encrypt_secret = encrypt_secret[1]
if self.action in ['rekey']:
new_vault_ids = []
if self.options.new_vault_id:
new_vault_ids.append(self.options.new_vault_id)
new_vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=new_vault_ids,
vault_password_files=self.options.new_vault_password_files,
ask_vault_pass=self.options.ask_vault_pass,
create_new_password=True)
if not new_vault_secrets:
raise AnsibleOptionsError("A new vault password is required to use Ansible's Vault rekey")
# There is only one new_vault_id currently and one new_vault_secret
new_encrypt_secret = match_encrypt_secret(new_vault_secrets)
self.new_encrypt_vault_id = new_encrypt_secret[0]
self.new_encrypt_secret = new_encrypt_secret[1]
loader.set_vault_secrets(vault_secrets)
# FIXME: do we need to create VaultEditor here? its not reused
vault = VaultLib(vault_secrets)
self.editor = VaultEditor(vault)
self.execute()
# and restore umask
os.umask(old_umask)
class Runner(object):
"""
wraps ansible playbook execution
"""
def __init__(self,
hostnames,
action,
playbook,
private_key_file,
run_data,
internal_data,
location,
become_pass,
request_id,
started_at,
config,
dbsession,
tr,
verbosity=5):
self.logger = app.logger
self.logger.debug('initializing ansible runbook executor')
self.action = action
self.config = config
self.dbsession = dbsession
self.transition_request = tr
self.location = location
self.run_data = run_data
self.internal_data = internal_data
self.run_variables = {}
self.run_variables.update(self.run_data)
self.run_variables.update(self.location)
self.run_variables.update(self.internal_data)
self.logger.debug(str(self.location))
self.logger.debug(str(self.run_data))
self.logger.debug(str(self.run_variables))
self.request_id = request_id
self.started_at = started_at
self.finished_at = None
self.resInstance = {}
# NEW 2.9
# handles options now
context.CLIARGS = ImmutableDict(
connection='ssh',
module_path=['/var/alm_ansible_rm/library'],
forks=20,
become=None,
become_method='sudo',
become_user='******',
check=False,
diff=False,
ansible_python_interpreter='/usr/bin/python3',
host_key_checking=False,
vault_password_file='/etc/ansible/tslvault.txt',
private_key_file=private_key_file,
listhosts=None,
listtasks=None,
listtags=None,
syntax=None,
start_at_task=None)
# Gets data from YAML/JSON files
self.loader = DataLoader()
self.loader.set_vault_secrets([
('default', VaultSecret(_bytes=to_bytes('TSLDem0')))
])
# create temporary inventory file
self.hosts = NamedTemporaryFile(delete=False)
self.hosts.write(b'[run_hosts]\n')
self.hosts.write(
b'localhost ansible_connection=local ansible_python_interpreter="/usr/bin/env python3" host_key_checking=False'
)
self.hosts.close()
# set Inventory
self.inventory = InventoryManager(loader=self.loader,
sources=self.hosts.name)
# All the variables from all the various places
self.variable_manager = VariableManager(loader=self.loader,
inventory=self.inventory)
# MOD 2.9 - has been renamed, not sure this is the proposed way to treat extra_vars, but it works
self.variable_manager._extra_vars = self.run_variables
# Become Pass Needed if not logging in as user root
passwords = {'become_pass': become_pass}
# Setup playbook executor, but don't run until run() called
self.pbex = playbook_executor.PlaybookExecutor(
playbooks=[playbook],
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
passwords=passwords)
if (self.transition_request) and (isinstance(self.transition_request,
TransitionRequest)):
# log only if transition request, not for netowrk/image scans)
self.logger.debug('transition request ' +
str(self.transition_request))
self.log_request_status('PENDING', 'playbook initialized', '', '')
self.logger.debug('ansible runbook executor instantiated for ' +
str(playbook))
self.callback = OutputCallback(self.action, self.logger)
self.pbex._tqm._stdout_callback = self.callback
def run(self):
"""
run an ansible playbook (sync mode) and return Results
"""
self.logger.debug('request ' + str(self.request_id) + ' action ' +
self.action)
self.pbex._tqm._stdout_callback = self.callback
self.pbex.run()
return self.callback.properties, self.callback.is_run_ok()
def run_async(self):
"""
run an ansible playbook asynchronously and return results when done
"""
self.logger.debug('request ' + str(self.request_id) + ' action ' +
self.action)
self.log_request_status('IN_PROGRESS', 'running playbook', '', '')
self.pbex._tqm._stdout_callback = self.callback
self.pbex.run()
self.finished_at = datetime.now()
self.logger.debug('request ' + str(self.request_id) + ' action ' +
self.action + " ansible facts" +
json.dumps(self.callback.facts))
if self.callback.is_run_ok():
# if not self.callback.resource_id:
# self.logger.error(str(self.request_id) + ': ' + 'Resource ID MUST be set')
# else:
# resource_id = self.callback.resource_id
# self.logger.debug(str(self.request_id) + ': ' + 'resource created id ' + resource_id)
if not self.callback.internal_resource_instances:
if self.transition_request.transition_name in ('Install',
'Uninstall'):
self.logger.warning(
str(self.request_id) + ': ' +
'Internal Resources may be missing')
internal_resources = []
else:
internal_resources = self.callback.internal_resource_instances
self.logger.debug(
str(self.request_id) + ': ' + 'internal resources ' +
str(internal_resources))
prop_output = {}
prop_output.update(self.run_data)
prop_output.update(self.callback.properties)
# set resource_id=metric_key
resource_id = prop_output['metric_key']
self.logger.debug(
str(self.request_id) + ': ' + 'resource created id ' +
resource_id)
# remove location and ansible variables (hard wired for now :-()
del prop_output['user_id']
del prop_output['keys_dir']
del prop_output['metric_key']
if 'request_id' in prop_output:
del prop_output['request_id']
# self.logger.debug(str(prop_output))
# self.logger.debug(str(self.callback.properties))
properties = dict(
set(prop_output.items()) - set(self.location.items()))
self.logger.debug(
str(self.request_id) + ': ' + 'properties: ' + str(properties))
internalProperties = self.internal_data
internalProperties.update(self.callback.internal_properties)
self.logger.debug(
str(self.request_id) + ': ' + 'internal properties: ' +
str(internalProperties))
if self.transition_request.transition_name == 'Install':
self.logger.debug(
str(self.request_id) + ': ' + 'creating instance')
self.resInstance = self.create_instance(
resource_id, properties, internal_resources,
internalProperties)
elif self.transition_request.transition_name == 'Uninstall':
if not (('protocol' in prop_output) and
(prop_output['protocol'] == 'SOL003')):
self.logger.debug(
str(self.request_id) + ': ' + 'deleting instance')
self.delete_instance(
resource_id,
self.transition_request.deployment_location)
elif self.transition_request.transition_name in ('Start', 'Stop',
'Configure',
'Integrity'):
self.logger.debug(
str(self.request_id) + ': ' +
'updating instance properties')
self.resInstance = self.update_instance_props(
resource_id, properties, internalProperties)
else:
self.logger.debug(
str(self.request_id) + ': ' +
'update internal properties ')
self.resInstance = self.update_instance_props(
resource_id, None, internalProperties)
if ('protocol' in prop_output) and (
prop_output['protocol']
== 'SOL003') and (self.transition_request.transition_name
in ('Install', 'Uninstall')):
self.log_request_status('IN_PROGRESS', '', '', resource_id)
else:
self.log_request_status(
'COMPLETED', 'Done in ' + str(
(self.finished_at - self.started_at).total_seconds()) +
' seconds', '', resource_id)
return
else:
if self.callback.failed_task != '':
last_task = self.callback.failed_task
else:
last_task = 'Unknown'
self.log_request_status(
'FAILED', last_task + ' ' + self.callback.failure_reason,
self.callback.failure_code, '')
return
def log_request_status(self, status, freason, fcode, resource_id):
"""
write log status to db or push to kafka
"""
is_async_mode = self.config.getSupportedFeatures(
)['AsynchronousTransitionResponses']
ttl = self.config.getTTL()
#self.logger.info('async request mode is ' + str(is_async_mode))
if (status == 'COMPLETED') or (status == 'FAILED'):
finished = self.finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
finished = ''
started = self.started_at.strftime('%Y-%m-%dT%H:%M:%SZ')
if resource_id != '':
resource_id = uuid.UUID(resource_id)
else:
resource_id = None
if status == 'FAILED':
self.logger.error('request ' + str(self.request_id) + ' action ' +
self.action + ' FAILED:' + freason +
' failure code: ' + fcode)
elif status == 'COMPLETED':
self.logger.info('request ' + str(self.request_id) + ' action ' +
self.action + ' finished OK')
try:
if status == 'PENDING':
self.logger.debug(
str(self.request_id) + ': ' +
'Writing PENDING request to db')
self.dbsession.execute(
"""
INSERT INTO requests (requestId, requestState, requestStateReason,startedAt, context)
VALUES (%s, %s, %s, %s, %s)
USING TTL """ + str(ttl) + """
""", (self.request_id, status, freason, started,
self.config.getSupportedFeatures()))
else:
self.logger.debug(
str(self.request_id) + ': ' +
'Writing COMPLETED or FAILED request to db')
self.dbsession.execute(
"""
INSERT INTO requests (requestId, requestState, requestStateReason, requestFailureCode, resourceId, finishedAt)
VALUES (%s, %s, %s, %s, %s, %s)
USING TTL """ + str(ttl) + """
""", (self.request_id, status, freason, fcode, resource_id,
finished))
except Exception as err:
# handle any other exception
self.logger.error(str(err))
raise
self.logger.debug('request result logged to DB: ' +
str(self.request_id))
if is_async_mode:
if (status == 'COMPLETED') or (status == 'FAILED'):
# call kafka
self.logger.debug(
str(self.request_id) + ': ' + 'async mode and status is ' +
status)
kafkaClient = Kafka(self.logger)
kmsg = {}
kmsg['resourceInstance'] = dict(self.resInstance)
kmsg['requestId'] = str(self.request_id)
kmsg[
'resourceManagerId'] = self.transition_request.resource_manager_id
kmsg[
'deploymentLocation'] = self.transition_request.deployment_location
kmsg['resourceType'] = self.transition_request.resource_type
kmsg[
'transitionName'] = self.transition_request.transition_name
kmsg['context'] = {}
kmsg['requestState'] = status
kmsg['requestStateReason'] = freason
kmsg['requestFailureCode'] = fcode
kmsg['startedAt'] = started
kmsg['finishedAt'] = finished
self.logger.debug(
str(self.request_id) + ': ' +
'sending message to kafka: ' + str(kmsg))
kafkaClient.sendLifecycleEvent(kmsg)
return
def create_instance(self, resource_id, out_props, internal_resources,
internal_properties):
"""
save instance details to db
"""
self.logger.debug('create instance ' + resource_id)
# a little cheating, need to get this from OS
created_at = self.finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
last_modified_at = created_at
pitem = {
'resourceId': resource_id,
'deploymentLocation': self.transition_request.deployment_location,
'resourceType': self.transition_request.resource_type,
'resourceName': self.transition_request.resource_name,
'resourceManagerId': self.transition_request.resource_manager_id,
'properties': out_props,
'internalProperties': internal_properties,
'internalResourceInstances': internal_resources,
'metricKey': self.transition_request.metric_key,
'createdAt': created_at,
'lastModifiedAt': last_modified_at
}
try:
self.dbsession.execute(
"""
INSERT INTO instances
(resourceId, resourceType, resourceName, resourceManagerId,
deploymentLocation, createdAt, lastModifiedAt,
properties, internalProperties, internalResourceInstances, metricKey)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s,%s, %s, %s)
""", (uuid.UUID(pitem['resourceId']), pitem['resourceType'],
pitem['resourceName'], pitem['resourceManagerId'],
pitem['deploymentLocation'], pitem['createdAt'],
pitem['lastModifiedAt'], pitem['properties'],
pitem['internalProperties'],
pitem['internalResourceInstances'], pitem['metricKey']))
except Exception as err:
# handle any other exception
self.logger.error(str(err))
raise
self.logger.debug('instance created and logger to database ' +
str(pitem))
return pitem
def update_instance_props(self, resource_id, out_props,
internal_properties):
"""
update instance properties to db
"""
self.logger.debug('update instance ' + resource_id)
# a little cheating, need to get this from OS
created_at = self.finished_at.strftime('%Y-%m-%dT%H:%M:%SZ')
last_modified_at = created_at
pitem = {
'resourceId': resource_id,
'properties': out_props,
'internalProperties': internal_properties,
'deploymentLocation': self.transition_request.deployment_location,
'lastModifiedAt': last_modified_at
}
try:
if out_props is None:
self.dbsession.execute(
"""
UPDATE instances
SET
lastModifiedAt=%s,
internalProperties=%s
WHERE resourceId=%s
AND deploymentLocation=%s
""",
(pitem['lastModifiedAt'], pitem['internalProperties'],
uuid.UUID(
pitem['resourceId']), pitem['deploymentLocation']))
else:
self.dbsession.execute(
"""
UPDATE instances
SET
lastModifiedAt=%s,
properties=%s,
internalProperties=%s
WHERE resourceId=%s
AND deploymentLocation=%s
""",
(pitem['lastModifiedAt'], pitem['properties'],
pitem['internalProperties'], uuid.UUID(
pitem['resourceId']), pitem['deploymentLocation']))
except Exception as err:
# handle any other exception
self.logger.error(str(err))
raise
self.logger.debug('instance updated ' + str(pitem))
return pitem
def delete_instance(self, resource_id, deployment_location):
"""
delete instance details from db
"""
self.logger.debug('deleting instance ' + resource_id)
try:
self.dbsession.execute(
"""
DELETE FROM instances
WHERE resourceId = %s and deploymentLocation = %s
""", [uuid.UUID(resource_id), deployment_location])
except Exception as err:
# handle any other exception
self.logger.error(str(err))
raise
self.logger.debug('instance deleted ' + resource_id)
return
display = AnsibleDisplay(verbosity=0)
from ansible import context
from ansible.cli import CLI
from ansible.module_utils.common.collections import ImmutableDict
from ansible.executor.playbook_executor import PlaybookExecutor
from ansible.parsing.dataloader import DataLoader
from ansible.inventory.manager import InventoryManager
from ansible.vars.manager import VariableManager
from ansible.module_utils._text import to_bytes
from ansible.parsing.vault import VaultSecret
code_path = "../../.."
loader = DataLoader()
loader.set_vault_secrets([('default', VaultSecret(_bytes=to_bytes('123456')))])
context.CLIARGS = ImmutableDict(tags={},
listtags=False,
listtasks=False,
listhosts=False,
syntax=False,
module_path=None,
forks=100,
private_key_file=None,
start_at_task=None)
inventory = InventoryManager(loader=loader,
sources=[code_path + '/ansible/inventory'])
variable_manager = VariableManager(
def run(self):
super(VaultCLI, self).run()
loader = DataLoader()
# set default restrictive umask
old_umask = os.umask(0o077)
vault_ids = self.options.vault_ids
# there are 3 types of actions, those that just 'read' (decrypt, view) and only
# need to ask for a password once, and those that 'write' (create, encrypt) that
# ask for a new password and confirm it, and 'read/write (rekey) that asks for the
# old password, then asks for a new one and confirms it.
default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
vault_ids = default_vault_ids + vault_ids
# TODO: instead of prompting for these before, we could let VaultEditor
# call a callback when it needs it.
if self.action in ['decrypt', 'view', 'rekey', 'edit']:
vault_secrets = self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
ask_vault_pass=self.options.ask_vault_pass)
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
if self.action in ['encrypt', 'encrypt_string', 'create']:
encrypt_vault_id = None
# no --encrypt-vault-id self.options.encrypt_vault_id for 'edit'
if self.action not in ['edit']:
encrypt_vault_id = self.options.encrypt_vault_id or C.DEFAULT_VAULT_ENCRYPT_IDENTITY
vault_secrets = None
vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=vault_ids,
vault_password_files=self.options.vault_password_files,
ask_vault_pass=self.options.ask_vault_pass,
create_new_password=True)
if len(vault_secrets) > 1 and not encrypt_vault_id:
raise AnsibleOptionsError("The vault-ids %s are available to encrypt. Specify the vault-id to encrypt with --encrypt-vault-id" %
','.join([x[0] for x in vault_secrets]))
if not vault_secrets:
raise AnsibleOptionsError("A vault password is required to use Ansible's Vault")
encrypt_secret = match_encrypt_secret(vault_secrets,
encrypt_vault_id=encrypt_vault_id)
# only one secret for encrypt for now, use the first vault_id and use its first secret
# TODO: exception if more than one?
self.encrypt_vault_id = encrypt_secret[0]
self.encrypt_secret = encrypt_secret[1]
if self.action in ['rekey']:
encrypt_vault_id = self.options.encrypt_vault_id or C.DEFAULT_VAULT_ENCRYPT_IDENTITY
# print('encrypt_vault_id: %s' % encrypt_vault_id)
# print('default_encrypt_vault_id: %s' % default_encrypt_vault_id)
# new_vault_ids should only ever be one item, from
# load the default vault ids if we are using encrypt-vault-id
new_vault_ids = []
if encrypt_vault_id:
new_vault_ids = default_vault_ids
if self.options.new_vault_id:
new_vault_ids.append(self.options.new_vault_id)
new_vault_password_files = []
if self.options.new_vault_password_file:
new_vault_password_files.append(self.options.new_vault_password_file)
new_vault_secrets = \
self.setup_vault_secrets(loader,
vault_ids=new_vault_ids,
vault_password_files=new_vault_password_files,
ask_vault_pass=self.options.ask_vault_pass,
create_new_password=True)
if not new_vault_secrets:
raise AnsibleOptionsError("A new vault password is required to use Ansible's Vault rekey")
# There is only one new_vault_id currently and one new_vault_secret, or we
# use the id specified in --encrypt-vault-id
new_encrypt_secret = match_encrypt_secret(new_vault_secrets,
encrypt_vault_id=encrypt_vault_id)
self.new_encrypt_vault_id = new_encrypt_secret[0]
self.new_encrypt_secret = new_encrypt_secret[1]
loader.set_vault_secrets(vault_secrets)
# FIXME: do we need to create VaultEditor here? its not reused
vault = VaultLib(vault_secrets)
self.editor = VaultEditor(vault)
self.execute()
# and restore umask
os.umask(old_umask)
class Loader:
def __init__(self):
self._config = ConfigManager()
self._loader = DataLoader()
self._file_vault_secrets = {}
self._inventory: InventoryManager = None
self._variables: VariableManager = None
def _load_file_vault_secrets(self, password_file_path=None):
if password_file_path is None:
password_file_path = self._config.get_config_value(
"DEFAULT_VAULT_PASSWORD_FILE")
password_file = Path(password_file_path)
if password_file.name in self._file_vault_secrets:
return
if not password_file.is_file():
raise FileNotFoundError(password_file)
self._file_vault_secrets[password_file.name] = get_file_vault_secret(
filename=password_file, loader=self._loader)
self._file_vault_secrets[password_file.name].load()
self._loader.set_vault_secrets([
(password_file.name, self._file_vault_secrets[password_file.name])
])
def _get_inventory(self, source=None):
if source is None:
source = self._config.get_config_value("DEFAULT_HOST_LIST")
if self._inventory is None:
self._inventory = InventoryManager(loader=self._loader,
sources=source)
else:
sources = source if isinstance(source, list) else [source]
(self._inventory.parse_source(s) for s in sources)
self._inventory.reconcile_inventory()
return self._inventory
def _get_variables(self, source=None):
self._load_file_vault_secrets()
if self._variables is None:
self._variables = VariableManager(
loader=self._loader, inventory=self._get_inventory(source))
return self._variables
def get_hosts(self, pattern='all', source=None):
return self._get_inventory(source).get_hosts(pattern)
def get_host(self, hostname, source=None):
return self._get_inventory(source).get_host(hostname)
def get_vars(self, host, data={}):
play = Play.load(data,
loader=self._loader,
variable_manager=self._get_variables())
return self._get_variables().get_vars(host=host, play=play)
class Runner(object):
"""
This is a General object for parallel execute modules.
本接口依赖hosts文件，在调用本接口之前，需要先生成hosts文件，可以通过手动编辑或者动态生成的方式生成hosts文件。
如果在hosts文件里指定了ansible_ssh_private_key_file、ansible_python_interpreter、ansible_ssh_user等变量，
调用本接口时就无需再传入这些变量了,即使传了这些变量也没用，因为hosts文件中变量的优先级最高
使用密码或公钥指定一个即可，另一个设为None。比如：如果传入了公钥参数，则self.passwords就设为None
"""
def __init__(self, resource, *args, **kwargs):
# resource： 可以为hosts文件的绝对路径（或者绝对路径列表）， 也可以是以逗号隔开的ip字符串，如：'1.1.1.1,2.2.2.2,3.3.3.3',只有一个ip时也要加逗号，
# 这个ip字符串会被ansible当做hosts文件
self.resource = resource
self.variable_manager = None
self.loader = None
self.options = None
self.passwords = None # 主机登录密码
self.private_key_file = None # 登录私钥
self.ansible_vault_key = kwargs.get(
'ansible_vault_key', '') # ansible加解密密码
self.callback = ResultsCollector() # ansible回调函数，处理返回结果
self.ansible_ssh_user = kwargs.get(
'ansible_ssh_user', 'root') # 主机登录用户
self.become_user = kwargs.get('module_name', '') # 登录主机后切换的用户
self.ansible_python_interpreter = kwargs.get(
'ansible_python_interpreter', "/usr/bin/python") # 远程机器使用的python解释器
self.__initializeData()
self.results_raw = {}
# ip_list是执行ansible任务的ip列表，这个ip列表中的ip必须全都在self.resource里
self.ip_list = kwargs.get('ip_list', [])
self.module_name = kwargs.get('module_name', '') # ansible调用的模块名
self.module_args = kwargs.get('module_args', '') # 模块需要的参数
def __initializeData(self):
"""
初始化ansible
"""
Options = namedtuple('Options',
['ansible_python_interpreter', 'connection', 'module_path', 'forks', 'become',
'become_method', 'become_user', 'check',
'diff', 'private_key_file', 'remote_user'])
self.loader = DataLoader()
# 加载hosts文件解密密码
self.loader.set_vault_secrets(
[('default', VaultSecret(_bytes=to_bytes(self.ansible_vault_key)))])
self.options = Options(ansible_python_interpreter=self.ansible_python_interpreter,
connection='ssh', module_path='', forks=100, become=None,
become_method=None, become_user=self.become_user,
check=False, diff=False, private_key_file=self.private_key_file,
remote_user=self.ansible_ssh_user)
self.passwords = dict(conn_pass=self.passwords)
self.inventory = InventoryManager(
loader=self.loader, sources=self.resource)
self.variable_manager = VariableManager(
loader=self.loader, inventory=self.inventory)
def get_all_hosts(self):
inventory = InventoryManager(
loader=self.loader, sources=self.resource)
return inventory.get_hosts()
def run(self):
"""
运行ansible任务.
"""
# create play with tasks
play_source = dict(
name="Ansible Play",
hosts=self.ip_list,
gather_facts='no',
tasks=[
dict(action=dict(module=self.module_name, args=self.module_args))
]
)
play = Play().load(play_source, variable_manager=self.variable_manager, loader=self.loader)
# 运行
tqm = None
logger = mp.log_to_stderr()
try:
tqm = TaskQueueManager(
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords=self.passwords,
stdout_callback='default',
)
tqm._stdout_callback = self.callback
result = tqm.run(play)
except Exception as e:
logger.setLevel(logging.DEBUG)
finally:
if tqm is not None:
tqm.cleanup()
return result
def get_result(self):
# 将ansible返回的数据组装为自己需要的数据格式
self.results_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
for host, result in self.callback.host_ok.items():
self.results_raw['success'][host] = result._result
for host, result in self.callback.host_failed.items():
self.results_raw['failed'][host] = result._result
for host, result in self.callback.host_unreachable.items():
self.results_raw['unreachable'][host] = result._result['msg']
return self.results_raw
def apply_template(value, ctx, overrides=None):
if not isinstance(value, six.string_types):
msg = f"Error rendering template: source must be a string, not {type(value)}"
if ctx.strict:
raise UnfurlError(msg)
else:
return f"<<{msg}>>"
value = value.strip()
# implementation notes:
# see https://github.com/ansible/ansible/test/units/template/test_templar.py
# dataLoader is only used by _lookup and to set _basedir (else ./)
if not ctx.templar or (ctx.base_dir and ctx.templar._basedir != ctx.base_dir):
# we need to create a new templar
loader = DataLoader()
if ctx.base_dir:
loader.set_basedir(ctx.base_dir)
if ctx.templar and ctx.templar._loader._vault.secrets:
loader.set_vault_secrets(ctx.templar._loader._vault.secrets)
templar = Templar(loader)
ctx.templar = templar
else:
templar = ctx.templar
overrides = Templar.find_overrides(value, overrides)
if overrides:
# returns the original values
overrides = templar._apply_templar_overrides(overrides)
templar.environment.trim_blocks = False
# templar.environment.lstrip_blocks = False
fail_on_undefined = ctx.strict
if not fail_on_undefined:
templar.environment.undefined = UnfurlUndefined
vars = _VarTrackerDict(__unfurl=ctx, __python_executable=sys.executable)
if hasattr(ctx.currentResource, "attributes"):
vars['SELF'] = ctx.currentResource.attributes
vars.update(ctx.vars)
vars.ctx = ctx
# replaces current vars
# don't use setter to avoid isinstance(dict) check
templar._available_variables = vars
oldvalue = value
index = ctx.referenced.start()
# set referenced to track references (set by Ref.resolve)
# need a way to turn on and off
try:
# strip whitespace so jinija native types resolve even with extra whitespace
# disable caching so we don't need to worry about the value of a cached var changing
# use do_template because we already know it's a template
try:
value = templar.template(value, fail_on_undefined=fail_on_undefined)
except Exception as e:
msg = str(e)
match = re.search(r"has no attribute '(\w+)'", msg)
if match:
msg = f'missing attribute or key: "{match.group(1)}"'
value = f"<<Error rendering template: {msg}>>"
if ctx.strict:
logger.debug(value, exc_info=True)
raise UnfurlError(value)
else:
logger.warning(value[2:100] + "... see debug log for full report")
logger.debug(value, exc_info=True)
if ctx.task:
ctx.task._errors.append( UnfurlTaskError(ctx.task, msg) )
else:
if value != oldvalue:
ctx.trace("successfully processed template:", value)
external_result = None
for result in ctx.referenced.getReferencedResults(index):
if is_sensitive(result):
# note: even if the template rendered a list or dict
# we still need to wrap the entire result as sensitive because we
# don't know how the referenced senstive results were transformed by the template
ctx.trace("setting template result as sensitive")
# mark the template result as sensitive
return wrap_sensitive_value(value)
if result.external:
external_result = result
if (
external_result
and ctx.wantList == "result"
and value == external_result.external.get()
):
# return the external value instead
return external_result
# wrap result as AnsibleUnsafe so it isn't evaluated again
return wrap_var(value)
finally:
ctx.referenced.stop()
if overrides:
# restore original values
templar._apply_templar_overrides(overrides)
return value
class VaultpwCLI(CLI):
'''
'''
VALID_ACTIONS = frozenset(("show", "store"))
def __init__(self, args):
super().__init__(args)
self.encrypt_secret = None
self.encrypt_vault_id = None
def set_action(self):
super().set_action()
if self.action == "show":
self.parser.set_usage(
"usage: %prog show /path/to/example_password.yml")
elif self.action == "store":
self.parser.set_usage(
"usage: %prog store /path/to/example_password.yml [-c command]"
)
self.parser.add_option('-c',
'--command',
dest='password_command',
action='store',
type='string',
help="command to run to obtain a password")
self.parser.add_option(
'--encrypt-vault-id',
default=[],
dest='encrypt_vault_id',
action='store',
type='string',
help=
'the vault id used to encrypt (required if more than one vault-id is provided)'
)
def init_parser(self):
super().init_parser(
usage="usage: %%prog [%s] [options] /path/to/example_password.yml"
% "|".join(sorted(self.VALID_ACTIONS)),
desc=
"utility to store or fetch vault-encrypted passwords in YAML inventory files",
epilog=
"\nSee '%s <command> --help' for more information on a specific command.\n\n"
% os.path.basename(sys.argv[0]))
opt_help.add_vault_options(self.parser)
self.set_action()
def post_process_args(self, options, args):
options, args = super().post_process_args(options, args)
self.validate_conflicts(options,
vault_opts=True,
vault_rekey_opts=False)
display.verbosity = options.verbosity
if options.vault_ids:
for vault_id in options.vault_ids:
if u';' in vault_id:
raise AnsibleOptionsError(
"Invalid character ';' found in vault id: %s" %
vault_id)
return options, args
def run(self):
super().run()
self.loader = DataLoader()
vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST + list(
context.CLIARGS['vault_ids'])
vault_secrets = self.setup_vault_secrets(
self.loader,
vault_ids=vault_ids,
vault_password_files=list(context.CLIARGS['vault_password_files']),
ask_vault_pass=context.CLIARGS['ask_vault_pass'])
if not vault_secrets:
raise AnsibleOptionsError(
"A vault password is required to use ansible-vault")
encrypt_vault_id = context.CLIARGS.get(
'encrypt_vault_id') or C.DEFAULT_VAULT_ENCRYPT_IDENTITY
if len(vault_secrets) > 1 and not encrypt_vault_id:
raise AnsibleOptionsError(
"Use '--encrypt-vault-id id' to choose one of the following vault ids to use for encryption: %s"
% ','.join([x[0] for x in vault_secrets]))
encrypt_secret = match_encrypt_secret(
vault_secrets, encrypt_vault_id=encrypt_vault_id)
self.encrypt_vault_id = encrypt_secret[0]
self.encrypt_secret = encrypt_secret[1]
self.loader.set_vault_secrets(vault_secrets)
self.vault = VaultLib(vault_secrets)
if len(context.CLIARGS['args']) != 1:
raise AnsibleOptionsError(
"Exactly one inventory file must be specified")
self.file = os.path.expanduser(context.CLIARGS['args'][0])
old_umask = os.umask(0o077)
self.execute()
os.umask(old_umask)
def execute_store(self):
'''
Takes the path to an inventory file such as
inventory/group_vars/tag_Cluster_xxx/secrets/example_password.yml and
overwrites the file with an assignment of "example_password: password"
in vault-encrypted YAML format. The password is obtained by prompting
the user or, if a command is specified, by running the command and
reading stdout.
'''
b_plaintext = b''
command = context.CLIARGS['password_command']
if command:
try:
pw = subprocess.run(command, capture_output=True)
if pw.returncode != 0:
raise Exception('non-zero exit code: %s' % pw.returncode)
b_plaintext = pw.stdout.strip()
except Exception as e:
print("ERROR: password command failed: %s" % str(e),
file=sys.stderr)
sys.exit(-1)
else:
b_plaintext = to_bytes(display.prompt("Password: "******"ERROR: cannot encrypt password: %s" % str(e),
file=sys.stderr)
sys.exit(-1)
name = os.path.basename(self.file).replace('.yml', '')
lines = []
lines.append("%s: !vault |\n" % name)
for l in to_text(b_ciphertext).splitlines():
lines.append(" %s\n" % l)
try:
fh = open(self.file, 'wb')
fh.write(to_bytes(''.join(lines)))
fh.close()
except Exception as e:
print("ERROR: cannot write output to %s: %s" % (self.file, str(e)),
file=sys.stderr)
sys.exit(-1)
def execute_show(self):
'''
Takes the path to an inventory file such as
inventory/group_vars/tag_Cluster_xxx/secrets/example_password.yml and
prints the password defined therein. The file must contain a variable
assignment of the form "example_password: password"; either the whole
file is vault-encrypted, or only the password is.
'''
if not os.path.exists(self.file):
print("ERROR: inventory file does not exist: %s" % self.file,
file=sys.stderr)
sys.exit(-1)
try:
name = os.path.basename(self.file).replace('.yml', '')
y = self.loader.load_from_file(self.file)
print(y[name])
except Exception as e:
print("ERROR: cannot show password from %s: %s" %
(self.file, str(e)),
file=sys.stderr)
sys.exit(-1)
