def build_inventory(loader, variable_manager, group_names, playbook_basedir):
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['localhost'])
# because we just pass in a "host list" which isn't a real inventory file,
# we explicitly have to add all of the desired groups to the inventory. By
# default an "all" group is created whenever a new inventory is created
for group_name in group_names:
if not inventory.get_group(group_name):
inventory.add_group(Group(group_name))
# because we are explicitly adding groups, we also need to make sure that a
# playbook basedir is set so that `group_vars` can be loaded from the
# correct directory.
inventory.set_playbook_basedir(playbook_basedir)
inventory_hosts = inventory.get_hosts()
# for each group specified, ensure that the inventory's host (localhost) is
# explicitly in the group.
for group_name in group_names:
group = inventory.get_group(group_name)
if group.get_hosts():
continue
for host in inventory.get_hosts():
group.add_host(host)
return inventory
class InventoryWrapper:
def __init__(self, host_list):
self.loader = DataLoader()
# this code is a bit ugly, because Ansible 2.4 switched the order
# of the object and the pattern (InventoryManager is a argument to
# VariableManager, and vice-versa on version <2.3)
if ANSIBLE_24:
self.im = InventoryManager(loader=self.loader,
sources=[host_list, ])
self.vm = VariableManager(loader=self.loader, inventory=self.im)
else:
self.vm = VariableManager()
self.im = Inventory(self.loader, self.vm, host_list)
def get_loader(self):
return self.loader
def get_variable_manager(self):
return VariableManagerWrapper(self.vm)
def get_groups(self):
if ANSIBLE_24:
return self.im.groups
return self.im.get_groups()
def get_hosts(self, group):
if ANSIBLE_24:
return self.im.get_hosts(pattern=group)
return self.im.get_hosts(group)
def get_host(self, host):
return self.im.get_host(host)
def refresh_inventory(self):
return self.im.refresh_inventory()
def inventory_html():
loader = DataLoader()
variable_manager = VariableManager()
sa_inv = Inventory(loader=loader,
variable_manager=variable_manager,
host_list='%s/hosts' % app.config.get('ANSIBLE_PATH'))
# print sa_inv.serialize()
if request.args.get('group'):
hosts = sa_inv.get_hosts(request.args.get('group'))
group_vars = sa_inv.get_group_variables(
groupname=request.args.get('group'))
else:
hosts = sa_inv.get_hosts()
group_vars = None
if request.args.get('host'):
host_vars = sa_inv.get_host_vars(
host=sa_inv.get_host(request.args.get('host')))
else:
host_vars = None
return render_template('inventory.html',
group=request.args.get('group'),
host=request.args.get('host'),
groups=sa_inv.get_groups(),
hosts=hosts,
group_vars=group_vars,
host_vars=host_vars)
def get_ip():
tag = "tag_Name_" + request.args.get('host')
env = request.args.get('env')
eip = request.args.get('eip').capitalize() if 'eip' in request.url else False
inventory_file = ec2_inventory_path + env + "/ec2.py"
if env not in envs:
return make_response(json.dumps({'error': env + ' Environment not found'}),400)
inv = Inventory(inventory_file)
if len(inv.get_hosts(tag)) == 0:
msg = "No Host match for {} in {}".format(request.args.get('host'), env)
return make_response(json.dumps({'error': msg}),400)
if eip == False:
#print "Checking IP of {} for Env: {}".format(tag,env)
ipaddr = ','.join([ ip.name for ip in inv.get_hosts(tag)])
else:
#print "Checking EIP of {} for Env: {}".format(tag,env)
data = ansible.runner.Runner(
module_name='debug',
pattern=tag,
inventory = inv,
transport = 'local',
module_args="var=hostvars['{{ inventory_hostname }}']['ec2_ip_address']"
).run()
element = data['contacted'].keys()
ipaddr = ','.join([ ''.join(data['contacted'][e]['var'].values()) for e in element])
if len(ipaddr) == 0:
msg = "No EIP Found for {} in {}".format(request.args.get('host'), env)
return make_response(json.dumps({'error': msg}),400)
return ipaddr
class AnsibleInventory(object):
def __init__(self, inventory_generator):
"""
:type inventory_generator: ops.inventory.generator.InventoryGenerator
"""
self.inventory_generator = inventory_generator
self.generated_path, self.ssh_config_path = inventory_generator.generate(
)
# clean up variables cache for tests
ansible_vars.VARIABLE_CACHE = dict()
ansible_vars.HOSTVARS_CACHE = dict()
ansible_inventory.HOSTS_PATTERNS_CACHE = dict()
self.variable_manager = VariableManager()
loader = DataLoader()
self.inventory = Inventory(loader=loader,
variable_manager=self.variable_manager,
host_list=self.generated_path)
self.variable_manager.set_inventory(self.inventory)
self.inventory.set_playbook_basedir(self.generated_path)
def get_hosts(self, limit):
return self.inventory.get_hosts(limit)
def get_host(self, host):
return self.inventory.get_host(unicode(host))
def get_vars(self, host):
return self.inventory.get_vars(unicode(host))
def get_ssh_config(self):
return self.ssh_config_path
class Inventory(object):
def __init__(self, inventory_path):
"""
The Inventory class provides methods to extract information from the
specified ansible inventory file.
:param inventory_path: The path to the inventory file
:type inventory_path: String
"""
from ansible.inventory import Inventory as AnsibleInventory
from ansible.parsing.dataloader import DataLoader
from ansible.vars import VariableManager
self._inventory = AnsibleInventory(loader=DataLoader(),
variable_manager=VariableManager(),
host_list=inventory_path)
def get_ansible_inventory(self):
return self._inventory
def get_hosts(self, pattern='all'):
return self._inventory.get_hosts(pattern)
def get_groups(self):
return self._inventory.get_groups()
def build_inventory(loader, variable_manager, group_names, playbook_basedir):
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['localhost'])
# because we just pass in a "host list" which isn't a real inventory file,
# we explicitly have to add all of the desired groups to the inventory. By
# default an "all" group is created whenever a new inventory is created
for group_name in group_names:
if not inventory.get_group(group_name):
inventory.add_group(Group(group_name))
# because we are explicitly adding groups, we also need to make sure that a
# playbook basedir is set so that `group_vars` can be loaded from the
# correct directory.
inventory.set_playbook_basedir(playbook_basedir)
# for each group specified, ensure that the inventory's host (localhost) is
# explicitly in the group.
for group_name in group_names:
group = inventory.get_group(group_name)
if group.get_hosts():
continue
for host in inventory.get_hosts():
group.add_host(host)
return inventory
def parse_hosts(filename):
variable_manager = VariableManager()
loader = DataLoader()
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=filename)
workers = inventory.get_hosts("backends")
return workers
class AnsibleEnvironment():
_cache = {}
def __init__(self):
ansible_basedir = os.path.join(
os.environ.get("PROJECT_ENVIRONMENT_FILES_PATH"), "ansible")
loader = CustomLoader()
loader.set_basedir(ansible_basedir)
var_manager = VariableManager()
# load the inventory, set the basic playbook directory
self._inventory = Inventory(loader=loader,
variable_manager=var_manager)
self._inventory.set_playbook_basedir(ansible_basedir)
group = self._inventory.get_group("all")
# make sure we load all magic variables on top of the global variables
self._vars = combine_vars(
self._inventory.get_group_vars(group, return_results=True),
var_manager._get_magic_variables(loader, False, None, None, False,
False))
self._vars['groups'] = self._inventory.get_group_dict()
self._vars['env'] = os.environ
hostvars = {}
for host in self._inventory.get_hosts():
hostvars[host.name] = host.get_vars()
self._vars['hostvars'] = hostvars
# create the template renderer
self._templar = Templar(loader=loader, variables=self._vars)
# setup some easy variables that we use a lot
self._vars['control_ip'] = self.get_var(
"hostvars[groups['control'][0]]['ansible_host']")
self._vars['edge_ip'] = self.get_var(
"hostvars[groups['edge'][0]]['ansible_host']")
self._vars['monitor_ip'] = self.get_var(
"hostvars[groups['monitor'][0]]['ansible_host']")
def get_var(self, name, cache=True):
if name not in self._cache or not cache:
try:
self._cache[name] = self._templar.template("{{%s}}" % name)
except AnsibleUndefinedVariable:
self._cache[name] = None
return self._cache.get(name)
def set_var(self, name, value):
self._vars[name] = value
def template(self, *templates):
return '\n'.join([self._templar.template(tpl) for tpl in templates])
def render_GET(self, request):
i = Inventory()
inv = []
try:
hosts = i.get_hosts()
groups = i.get_groups()
except:
return self.error_page(request)
inv.extend([{"name": x.name, "type":"host"} for x in sorted(hosts)])
inv.extend([{"name": x.name, "type":"group"} for x in sorted(groups)])
return jsonify(request, inv)
def get_ip():
tag = "tag_Name_" + request.args.get('host')
env = request.args.get('env')
eip = request.args.get(
'eip').capitalize() if 'eip' in request.url else False
inventory_file = ec2_inventory_path + env + "/ec2.py"
if env not in envs:
return make_response(
json.dumps({'error': env + ' Environment not found'}), 400)
inv = Inventory(inventory_file)
if len(inv.get_hosts(tag)) == 0:
msg = "No Host match for {} in {}".format(request.args.get('host'),
env)
return make_response(json.dumps({'error': msg}), 400)
if eip == False:
#print "Checking IP of {} for Env: {}".format(tag,env)
ipaddr = ','.join([ip.name for ip in inv.get_hosts(tag)])
else:
#print "Checking EIP of {} for Env: {}".format(tag,env)
data = ansible.runner.Runner(
module_name='debug',
pattern=tag,
inventory=inv,
transport='local',
module_args=
"var=hostvars['{{ inventory_hostname }}']['ec2_ip_address']").run(
)
element = data['contacted'].keys()
ipaddr = ','.join(
[''.join(data['contacted'][e]['var'].values()) for e in element])
if len(ipaddr) == 0:
msg = "No EIP Found for {} in {}".format(request.args.get('host'),
env)
return make_response(json.dumps({'error': msg}), 400)
return ipaddr
def notifyCanary():
inven = Inventory(host_list=path["hosts"]);
groups = inven.get_groups();
for group in groups:
# print group.get_variables()
if group.name == "canary":
hosts = inven.get_hosts(group.name)
for host in hosts:
host_ip = host.get_variables()['ansible_ssh_host']
url = 'http://'+proxy_server+':'+proxy_port+"/canary"
payload = {'host': host_ip}
headers = {'content-type': 'application/json'}
response = requests.post(url, data=payload, headers=headers)
def configure_hosts(self, reservation_obj, playbook):
inven = Inventory(host_list=reservation_obj.keys())
for host in inven.get_hosts():
for key, value in reservation_obj[host.name].items():
host.set_variable(key, value)
stats = callbacks.AggregateStats()
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = PlayBook(inventory=inven,
playbook=playbook,
stats=stats,
callbacks=playbook_cb,
runner_callbacks=runner_cb)
pb.run()
def extract_list_hosts_git(revision, path):
""" Extract the hosts list from git, after deduplciating it
and resolving variables """
result = []
if revision == '0000000000000000000000000000000000000000':
return result
try:
host_content = subprocess.check_output(
['git', 'show', '%s:hosts' % revision], cwd=path)
# usually, this is done when we can't check the list of hosts
except subprocess.CalledProcessError:
return result
# beware, not portable on windows
tmp_dir = tempfile.mkdtemp()
tmp_file = tempfile.NamedTemporaryFile('w+', dir=tmp_dir)
tmp_file.write(host_content)
tmp_file.flush()
os.fsync(tmp_file.fileno())
variable_manager = VariableManager()
loader = DataLoader()
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=tmp_file.name)
for group in inventory.get_groups():
for host in inventory.get_hosts(group):
vars_host = variable_manager.get_vars(loader, host=host)
result.append({
'name':
vars_host.get('ansible_ssh_host', host.name),
'ssh_args':
vars_host.get('ansible_ssh_common_args', ''),
'connection':
vars_host.get('ansible_connection', 'ssh')
})
# for some reason, there is some kind of global cache that need to be
# cleaned
inventory.refresh_inventory()
shutil.rmtree(tmp_dir)
return result
def __init__(self, inventory_file, module_name, module_args):
loader = DataLoader()
variable_manager = VariableManager()
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=inventory_file)
variable_manager.set_inventory(inventory)
hosts = [x.name for x in inventory.get_hosts()]
play_source = {
"name": "Ansible Play",
"hosts": hosts,
"gather_facts": "no",
"tasks": [{
"action": {
"module": module_name,
"args": module_args
}
}]
}
logging.info(play_source)
play = Play().load(play_source,
variable_manager=variable_manager,
loader=loader)
Options = namedtuple('Options', ['connection', 'module_path', 'forks',
'become', 'become_method',
'become_user', 'check'])
options = Options(connection='local',
module_path='',
forks=100,
become=None,
become_method=None,
become_user=None,
check=False)
self.inventory = inventory
self.variable_manager = variable_manager
self.loader = loader
self.play = play
self.options = options
self.passwords = {"vault_pass": '******'}
def getAnsibleStaticHosts(hosts_dir):
static_hosts_dir = os.path.join(hosts_dir, 'host_static')
hosts_file = [
os.path.join(static_hosts_dir, f) for f in os.listdir(static_hosts_dir)
]
#hosts_file.append(os.path.join(hosts_dir, 'hosts'))
host_list = []
loader = DataLoader()
variable_manager = VariableManager()
for hf in hosts_file:
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=hf)
host_list.extend(inventory.get_hosts())
return host_list
class Inventory(object):
def __init__(self, inventory_path):
"""
The Inventory class provides methods to extract information from the
specified ansible inventory file.
:param inventory_path: The path to the inventory file
:type inventory_path: String
"""
from ansible.inventory import Inventory as AnsibleInventory
self._inventory = AnsibleInventory(host_list=inventory_path)
def get_ansible_inventory(self):
return self._inventory
def get_hosts(self, pattern='all'):
return self._inventory.get_hosts(pattern)
def get_groups(self):
return self._inventory.get_groups()
def playbook_list_hosts(pattern, host_list=C.DEFAULT_HOST_LIST):
loader = DataLoader()
variable_manager = VariableManager()
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=host_list)
return inventory.get_hosts(pattern)
while pending_results > 0:
debug("waiting for pending results (%d left)" % pending_results)
_process_pending_results()
time.sleep(0.01)
debug("starting")
cur_worker = 0
pending_results = 0
var_manager = VariableManager()
debug("loading inventory")
inventory = Inventory(host_list='/tmp/med_inventory', loader=loader, variable_manager=var_manager)
hosts = inventory.get_hosts()[:]
debug("done loading inventory")
play_context = PlayContext()
play_context.connection = 'local'
for i in range(NUM_TASKS):
#for j in range(NUM_HOSTS):
for h in hosts:
debug("queuing %s %d" % (h, i))
#h = Host(name="host%06d" % j)
t = Task().load(dict(name="task %d" % (i,), debug="msg='hello from %s, %d'" % (h,i)))
#t = Task().load(dict(name="task %d" % (i,), ping=""))
#task_vars = var_manager.get_vars(loader=loader, host=h, task=t)
task_vars = dict()
new_t = t.copy()
class AnsibleExecutor(ExecutorPrototype):
""" Executor implemented on top of ansible. """
__EXECUTOR_NAME__ = "ansible"
FORK = 10
RAW_ARG = "_RAW"
INVENTORY = "inventory"
PLAYBOOKS = "playbooks"
CMD_MODULE = "shell"
PING_MODULE = "ping"
FACTER_MODULE = "facter"
SERVICE_MODULE = "service"
RAW_MODULE = ('command', 'shell', 'script', 'raw')
INIT_PB = "_deploy.yml"
ROLE_VAR = "_role"
TARGET_VAR = "_targets"
DEFAULT_PARTIAL = "all"
def __init__(self,
hosts=[],
timeout=0,
concurrency=0,
workdir=os.getcwd(),
inventory=None,
playbooks=None,
sshkey=None):
""" Prepare ansible context. """
self._workdir = make_abs_path(workdir)
inventory = inventory if inventory else self.INVENTORY
playbooks = playbooks if playbooks else self.PLAYBOOKS
# check inventory dir or file exists
inventory = make_abs_path(inventory, self._workdir)
if not os.path.exists(inventory):
raise ExecutorPrepareError(
"{0}, bad inventory given".format(inventory))
self._inventory_path = inventory
# check playbooks dir and make sure init playbooks exists
playbooks = make_abs_path(playbooks, self._workdir)
if not os.path.isdir(playbooks):
raise ExecutorPrepareError(
"{0}, bad playbooks directory given".format(playbooks))
if not os.path.exists(os.path.join(playbooks, self.INIT_PB)):
raise ExecutorPrepareError(
"{0} not exists, init playbooks \"{0}\" not found".format(
self._INIT_PB))
self._playbooks_path = playbooks
# we don't need check sshkey here, assume the user have `~/.ssh` if no key given
if sshkey != None and not os.path.isabs(sshkey):
sshkey = make_abs_path(sshkey, self._workdir)
if not os.path.isfile(sshkey):
raise ExecutorPrepareError(
"{0}, bad sshkey file".format(sshkey))
self._sshkey = sshkey
LOG.info(
"init ansible executor with: "
"workdir <{0}>, inventory <{1}>, playbooks <{2}>, sshkey <{3}>".
format(self._workdir, inventory, playbooks, self._sshkey))
self._concurrency = concurrency if concurrency else self.FORK
self._prepare()
super(AnsibleExecutor, self).__init__(hosts, timeout)
def _prepare(self):
""" Prepare ansible internal data struts. """
self._loader = DataLoader()
self._varmanager = VariableManager()
self._inventory = Inventory(loader=self._loader,
variable_manager=self._varmanager,
host_list=self._inventory_path)
self._varmanager.set_inventory(self._inventory)
self._opts = AnsibleOpts(forks=self._concurrency,
private_key_file=self._sshkey,
deprecation_warnings=False)
def _reset_internal(self):
""" Reset ansible internal data struts by invoke `_prepare` again. """
self._prepare()
def _is_raw(self, module):
""" Ansible RAW module, see `_play_ds()` in `cli/adhoc.py` """
return module in self.RAW_MODULE
def _set_check_mode(self, check):
""" Ansible check mode, see `load_options_vars()` in `utils/vars.py` . """
self._varmanager.options_vars = dict(ansible_check_mode=check)
def _run_tasks(self, play, reaper):
""" Init TQM and run play. """
tqm = TaskQueueManager(inventory=self._inventory,
variable_manager=self._varmanager,
loader=self._loader,
options=self._opts,
passwords=None,
stdout_callback=reaper)
# with multiprocessing, the parent cannot handle exception riased
# by the child process.
# which means, the try/except in the `runner._async_deploy` cannot
# known what happened here, and cause the entire celery worker
# process stop working without exit.
# Solution:
# 1, handle ansible exception here (inside executor).
# 2, cannot raise other exception in `except` block, because of
# this piece of code may be run under other `fork()`.
# 3, because of <2>, we use `reaper` to tell outside something going wrong.
try:
tqm.run(play)
except AnsibleError:
reaper.reaper_exception(ExecutorPrepareError(str(excinst())))
finally:
tqm.cleanup()
reaper.done()
def _run_pbs(self, playbooks, reaper):
""" Init PBEX and run playbooks. """
pbex = PlaybookExecutor(playbooks=playbooks,
inventory=self._inventory,
variable_manager=self._varmanager,
loader=self._loader,
options=self._opts,
passwords=None)
pbex._tqm._stdout_callback = reaper
# Same raeson with `self._run_tasks`
try:
pbex.run()
except AnsibleError:
reaper.reaper_exception(ExecutorPrepareError(str(excinst())))
reaper.done()
def _execute_playbooks(self, playbooks, extra_vars=None, partial=None):
""" Execute ansible playbooks. """
if partial is None:
partial = self.DEFAULT_PARTIAL
if not isinstance(playbooks, (list, tuple)):
playbooks = [playbooks]
self._reset_internal()
self._set_check_mode(False)
self._opts.tags = partial
self._varmanager.extra_vars = extra_vars
collector = AnsibleReaper(self._hosts)
LOG.info(
"execute playbook <{0}> with extra_vars <{1}> and partial <{2}> on <{3}>"
.format(playbooks, extra_vars, partial, self._hosts))
worker = multiprocessing.Process(target=self._run_pbs,
args=(playbooks, collector))
worker.start()
return collector.reaper()
def target(self, pattern):
""" Match target by given pattern. """
return [h.get_name() for h in self._inventory.get_hosts(pattern)]
def execute(self, module, check_mode=False, **module_args):
""" Invoke ansible module with given args on remote host(s). """
# Handle raw module args
args = module_args.pop(self.RAW_ARG, None)
if args is None:
args = ""
# Handle module args
for opt, val in module_args.items():
args = "{0}={1} {2}".format(opt, val, args)
self._reset_internal()
self._set_check_mode(check_mode)
args = parse_kv(args.strip(), self._is_raw(module))
name = "execute {0} on {1}".format(module, self._hosts)
collector = AnsibleReaper(self._hosts)
play_ds = dict(
name=name,
hosts=self._hosts,
gather_facts="no",
tasks=[dict(name=name, action=dict(module=module, args=args))])
play = Play.load(play_ds,
variable_manager=self._varmanager,
loader=self._loader)
LOG.info(
"execute ansible module <{0}> with args <{1}> on <{2}>".format(
module, args, self._hosts))
worker = multiprocessing.Process(name="exec",
target=self._run_tasks,
args=(play, collector))
worker.start()
return collector.reaper()
def raw_execute(self, cmd):
""" Invoke ansible command module on remote host(s). """
raw = {self.RAW_ARG: cmd}
_handler = lambda host, result: {
host:
dict(status=result.get(EXE_STATUS_ATTR),
stdout=result.get(EXE_RETURN_ATTR).pop('stdout', ""),
stderr=result.get(EXE_RETURN_ATTR).pop('stderr', ""),
rtc=result.get(EXE_RETURN_ATTR).pop('rc', -1))
}
for _out in self.execute(self.CMD_MODULE, **raw):
yield _handler(*_out.popitem())
def ping(self):
""" Ping remote host(s). """
_handler = lambda host, result: {
host: {
EXE_STATUS_ATTR: result.get(EXE_STATUS_ATTR)
}
}
for _out in self.execute(self.PING_MODULE):
yield _handler(*_out.popitem())
def facter(self):
""" Gather information of remote host(s). """
_handler = lambda host, result: {
host: {
EXE_STATUS_ATTR:
result.get(EXE_STATUS_ATTR),
'facter':
dict([(attr, val)
for attr, val in result.pop(EXE_RETURN_ATTR).iteritems()
if attr not in
('_ansible_parsed', 'changed', '_ansible_no_log', 'cmd',
'failed', 'unreachable', 'rc', 'invocation', 'msg')])
}
}
for _out in self.execute(self.FACTER_MODULE):
yield _handler(*_out.popitem())
def service(self, name, start=True, restart=False, graceful=True):
""" Manipulate service on remote host(s). """
if restart:
state = "reloaded" if graceful else "restarted"
else:
state = "started" if start else "stopped"
_handler = lambda host, result: {
host: {
EXE_STATUS_ATTR: result.pop(EXE_STATUS_ATTR)
}
}
for _out in self.execute(self.SERVICE_MODULE, name=name, state=state):
yield _handler(*_out.popitem())
def deploy(self, roles, extra_vars=None, partial=None):
""" Deploy service/role/app on remote host(s). """
if extra_vars:
if not isinstance(extra_vars, dict):
raise ExecutorDeployError("Bad extra_vars for deploy")
else:
extra_vars = dict()
extra_vars[self.ROLE_VAR] = roles
extra_vars[self.TARGET_VAR] = self._hosts
playbook = os.path.join(self._playbooks_path, self.INIT_PB)
return self._execute_playbooks(playbook, extra_vars, partial)
class InventoryCLI(CLI):
''' used to display or dump the configured inventory as Ansible sees it '''
ARGUMENTS = {'host': 'The name of a host to match in the inventory, relevant when using --list',
'group': 'The name of a group in the inventory, relevant when using --graph', }
def __init__(self, args):
super(InventoryCLI, self).__init__(args)
self.vm = None
self.loader = None
self.inventory = None
self._new_api = True
def parse(self):
self.parser = CLI.base_parser(
usage='usage: %prog [options] [host|group]',
epilog='Show Ansible inventory information, by default it uses the inventory script JSON format',
inventory_opts=True,
vault_opts=True
)
# Actions
action_group = optparse.OptionGroup(self.parser, "Actions", "One of following must be used on invocation, ONLY ONE!")
action_group.add_option("--list", action="store_true", default=False, dest='list', help='Output all hosts info, works as inventory script')
action_group.add_option("--host", action="store", default=None, dest='host', help='Output specific host info, works as inventory script')
action_group.add_option("--graph", action="store_true", default=False, dest='graph',
help='create inventory graph, if supplying pattern it must be a valid group name')
self.parser.add_option_group(action_group)
# Options
self.parser.add_option("-y", "--yaml", action="store_true", default=False, dest='yaml',
help='Use YAML format instead of default JSON, ignored for --graph')
self.parser.add_option("--vars", action="store_true", default=False, dest='show_vars',
help='Add vars to graph display, ignored unless used with --graph')
super(InventoryCLI, self).parse()
display.verbosity = self.options.verbosity
self.validate_conflicts(vault_opts=True)
# there can be only one! and, at least, one!
used = 0
for opt in (self.options.list, self.options.host, self.options.graph):
if opt:
used += 1
if used == 0:
raise AnsibleOptionsError("No action selected, at least one of --host, --graph or --list needs to be specified.")
elif used > 1:
raise AnsibleOptionsError("Conflicting options used, only one of --host, --graph or --list can be used at the same time.")
# set host pattern to default if not supplied
if len(self.args) > 0:
self.options.pattern = self.args[0]
else:
self.options.pattern = 'all'
def run(self):
results = None
super(InventoryCLI, self).run()
# Initialize needed objects
if getattr(self, '_play_prereqs', False):
self.loader, self.inventory, self.vm = self._play_prereqs(self.options)
else:
# fallback to pre 2.4 way of initialzing
from ansible.vars import VariableManager
from ansible.inventory import Inventory
self._new_api = False
self.loader = DataLoader()
self.vm = VariableManager()
# use vault if needed
if self.options.vault_password_file:
vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=self.loader)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords()
else:
vault_pass = None
if vault_pass:
self.loader.set_vault_password(vault_pass)
# actually get inventory and vars
self.inventory = Inventory(loader=self.loader, variable_manager=self.vm, host_list=self.options.inventory)
self.vm.set_inventory(self.inventory)
if self.options.host:
hosts = self.inventory.get_hosts(self.options.host)
if len(hosts) != 1:
raise AnsibleOptionsError("You must pass a single valid host to --hosts parameter")
myvars = self._get_host_variables(host=hosts[0])
self._remove_internal(myvars)
# FIXME: should we template first?
results = self.dump(myvars)
elif self.options.graph:
results = self.inventory_graph()
elif self.options.list:
top = self._get_group('all')
if self.options.yaml:
results = self.yaml_inventory(top)
else:
results = self.json_inventory(top)
results = self.dump(results)
if results:
# FIXME: pager?
display.display(results)
exit(0)
exit(1)
def dump(self, stuff):
if self.options.yaml:
import yaml
from ansible.parsing.yaml.dumper import AnsibleDumper
results = yaml.dump(stuff, Dumper=AnsibleDumper, default_flow_style=False)
else:
import json
results = json.dumps(stuff, sort_keys=True, indent=4)
return results
def _get_host_variables(self, host):
if self._new_api:
hostvars = self.vm.get_vars(host=host)
else:
hostvars = self.vm.get_vars(self.loader, host=host)
return hostvars
def _get_group(self, gname):
if self._new_api:
group = self.inventory.groups.get(gname)
else:
group = self.inventory.get_group(gname)
return group
def _remove_internal(self, dump):
for internal in INTERNAL_VARS:
if internal in dump:
del dump[internal]
def _remove_empty(self, dump):
# remove empty keys
for x in ('hosts', 'vars', 'children'):
if x in dump and not dump[x]:
del dump[x]
def _show_vars(self, dump, depth):
result = []
self._remove_internal(dump)
if self.options.show_vars:
for (name, val) in sorted(dump.items()):
result.append(self._graph_name('{%s = %s}' % (name, val), depth + 1))
return result
def _graph_name(self, name, depth=0):
if depth:
name = " |" * (depth) + "--%s" % name
return name
def _graph_group(self, group, depth=0):
result = [self._graph_name('@%s:' % group.name, depth)]
depth = depth + 1
for kid in sorted(group.child_groups, key=attrgetter('name')):
result.extend(self._graph_group(kid, depth))
if group.name != 'all':
for host in sorted(group.hosts, key=attrgetter('name')):
result.append(self._graph_name(host.name, depth))
result.extend(self._show_vars(host.get_vars(), depth))
result.extend(self._show_vars(group.get_vars(), depth))
return result
def inventory_graph(self):
start_at = self._get_group(self.options.pattern)
if start_at:
return '\n'.join(self._graph_group(start_at))
else:
raise AnsibleOptionsError("Pattern must be valid group name when using --graph")
def json_inventory(self, top):
def format_group(group):
results = {}
results[group.name] = {}
if group.name != 'all':
results[group.name]['hosts'] = [h.name for h in sorted(group.hosts, key=attrgetter('name'))]
results[group.name]['vars'] = group.get_vars()
results[group.name]['children'] = []
for subgroup in sorted(group.child_groups, key=attrgetter('name')):
results[group.name]['children'].append(subgroup.name)
results.update(format_group(subgroup))
self._remove_empty(results[group.name])
return results
results = format_group(top)
# populate meta
results['_meta'] = {'hostvars': {}}
hosts = self.inventory.get_hosts()
for host in hosts:
results['_meta']['hostvars'][host.name] = self._get_host_variables(host=host)
self._remove_internal(results['_meta']['hostvars'][host.name])
return results
def yaml_inventory(self, top):
seen = []
def format_group(group):
results = {}
# initialize group + vars
results[group.name] = {}
results[group.name]['vars'] = group.get_vars()
# subgroups
results[group.name]['children'] = {}
for subgroup in sorted(group.child_groups, key=attrgetter('name')):
if subgroup.name != 'all':
results[group.name]['children'].update(format_group(subgroup))
# hosts for group
results[group.name]['hosts'] = {}
if group.name != 'all':
for h in sorted(group.hosts, key=attrgetter('name')):
myvars = {}
if h.name not in seen: # avoid defining host vars more than once
seen.append(h.name)
myvars = self._get_host_variables(host=h)
self._remove_internal(myvars)
results[group.name]['hosts'][h.name] = myvars
self._remove_empty(results[group.name])
return results
return format_group(top)
def test_get_hosts(self):
inventory = Inventory('127.0.0.1,192.168.1.1')
hosts = inventory.get_hosts('!10.0.0.1')
hosts_all = inventory.get_hosts('all')
self.assertEqual(sorted(hosts), sorted(hosts_all))
class InventoryCLI(CLI):
''' used to display or dump the configured inventory as Ansible sees it '''
ARGUMENTS = {
'host':
'The name of a host to match in the inventory, relevant when using --list',
'group':
'The name of a group in the inventory, relevant when using --graph',
}
def __init__(self, args):
super(InventoryCLI, self).__init__(args)
self.vm = None
self.loader = None
self.inventory = None
self._new_api = True
def parse(self):
self.parser = CLI.base_parser(
usage='usage: %prog [options] [host|group]',
epilog=
'Show Ansible inventory information, by default it uses the inventory script JSON format',
inventory_opts=True,
vault_opts=True,
basedir_opts=True,
)
# remove unused default options
self.parser.remove_option('--limit')
self.parser.remove_option('--list-hosts')
# Actions
action_group = optparse.OptionGroup(
self.parser, "Actions",
"One of following must be used on invocation, ONLY ONE!")
action_group.add_option(
"--list",
action="store_true",
default=False,
dest='list',
help='Output all hosts info, works as inventory script')
action_group.add_option(
"--host",
action="store",
default=None,
dest='host',
help='Output specific host info, works as inventory script')
action_group.add_option(
"--graph",
action="store_true",
default=False,
dest='graph',
help=
'create inventory graph, if supplying pattern it must be a valid group name'
)
self.parser.add_option_group(action_group)
# graph
self.parser.add_option(
"-y",
"--yaml",
action="store_true",
default=False,
dest='yaml',
help='Use YAML format instead of default JSON, ignored for --graph'
)
self.parser.add_option(
"--vars",
action="store_true",
default=False,
dest='show_vars',
help='Add vars to graph display, ignored unless used with --graph')
# list
self.parser.add_option(
"--export",
action="store_true",
default=C.INVENTORY_EXPORT,
dest='export',
help=
"When doing an --list, represent in a way that is optimized for export,"
"not as an accurate representation of how Ansible has processed it"
)
# self.parser.add_option("--ignore-vars-plugins", action="store_true", default=False, dest='ignore_vars_plugins',
# help="When doing an --list, skip vars data from vars plugins, by default, this would include group_vars/ and host_vars/")
super(InventoryCLI, self).parse()
display.verbosity = self.options.verbosity
self.validate_conflicts(vault_opts=True)
# there can be only one! and, at least, one!
used = 0
for opt in (self.options.list, self.options.host, self.options.graph):
if opt:
used += 1
if used == 0:
raise AnsibleOptionsError(
"No action selected, at least one of --host, --graph or --list needs to be specified."
)
elif used > 1:
raise AnsibleOptionsError(
"Conflicting options used, only one of --host, --graph or --list can be used at the same time."
)
# set host pattern to default if not supplied
if len(self.args) > 0:
self.options.pattern = self.args[0]
else:
self.options.pattern = 'all'
def run(self):
results = None
super(InventoryCLI, self).run()
# Initialize needed objects
if getattr(self, '_play_prereqs', False):
self.loader, self.inventory, self.vm = self._play_prereqs(
self.options)
else:
# fallback to pre 2.4 way of initialzing
from ansible.vars import VariableManager
from ansible.inventory import Inventory
self._new_api = False
self.loader = DataLoader()
self.vm = VariableManager()
# use vault if needed
if self.options.vault_password_file:
vault_pass = CLI.read_vault_password_file(
self.options.vault_password_file, loader=self.loader)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords()
else:
vault_pass = None
if vault_pass:
self.loader.set_vault_password(vault_pass)
# actually get inventory and vars
self.inventory = Inventory(loader=self.loader,
variable_manager=self.vm,
host_list=self.options.inventory)
self.vm.set_inventory(self.inventory)
if self.options.host:
hosts = self.inventory.get_hosts(self.options.host)
if len(hosts) != 1:
raise AnsibleOptionsError(
"You must pass a single valid host to --hosts parameter")
myvars = self._get_host_variables(host=hosts[0])
self._remove_internal(myvars)
# FIXME: should we template first?
results = self.dump(myvars)
elif self.options.graph:
results = self.inventory_graph()
elif self.options.list:
top = self._get_group('all')
if self.options.yaml:
results = self.yaml_inventory(top)
else:
results = self.json_inventory(top)
results = self.dump(results)
if results:
# FIXME: pager?
display.display(results)
exit(0)
exit(1)
def dump(self, stuff):
if self.options.yaml:
import yaml
from ansible.parsing.yaml.dumper import AnsibleDumper
results = yaml.dump(stuff,
Dumper=AnsibleDumper,
default_flow_style=False)
else:
import json
from ansible.parsing.ajson import AnsibleJSONEncoder
results = json.dumps(stuff,
cls=AnsibleJSONEncoder,
sort_keys=True,
indent=4)
return results
# FIXME: refactor to use same for VM
def get_plugin_vars(self, path, entity):
data = {}
def _get_plugin_vars(plugin, path, entities):
data = {}
try:
data = plugin.get_vars(self.loader, path, entity)
except AttributeError:
try:
if isinstance(entity, Host):
data = combine_vars(data,
plugin.get_host_vars(entity.name))
else:
data = combine_vars(data,
plugin.get_group_vars(entity.name))
except AttributeError:
if hasattr(plugin, 'run'):
raise AnsibleError(
"Cannot use v1 type vars plugin %s from %s" %
(plugin._load_name, plugin._original_path))
else:
raise AnsibleError(
"Invalid vars plugin %s from %s" %
(plugin._load_name, plugin._original_path))
return data
for plugin in vars_loader.all():
data = combine_vars(data, _get_plugin_vars(plugin, path, entity))
return data
def _get_group_variables(self, group):
# get info from inventory source
res = group.get_vars()
# FIXME: add switch to skip vars plugins, add vars plugin info
for inventory_dir in self.inventory._sources:
res = combine_vars(res, self.get_plugin_vars(inventory_dir, group))
if group.priority != 1:
res['ansible_group_priority'] = group.priority
return res
def _get_host_variables(self, host):
if self.options.export:
hostvars = host.get_vars()
# FIXME: add switch to skip vars plugins
# add vars plugin info
for inventory_dir in self.inventory._sources:
hostvars = combine_vars(
hostvars, self.get_plugin_vars(inventory_dir, host))
else:
if self._new_api:
hostvars = self.vm.get_vars(host=host, include_hostvars=False)
else:
hostvars = self.vm.get_vars(self.loader,
host=host,
include_hostvars=False)
return hostvars
def _get_group(self, gname):
if self._new_api:
group = self.inventory.groups.get(gname)
else:
group = self.inventory.get_group(gname)
return group
def _remove_internal(self, dump):
for internal in INTERNAL_VARS:
if internal in dump:
del dump[internal]
def _remove_empty(self, dump):
# remove empty keys
for x in ('hosts', 'vars', 'children'):
if x in dump and not dump[x]:
del dump[x]
def _show_vars(self, dump, depth):
result = []
self._remove_internal(dump)
if self.options.show_vars:
for (name, val) in sorted(dump.items()):
result.append(
self._graph_name('{%s = %s}' % (name, val), depth))
return result
def _graph_name(self, name, depth=0):
if depth:
name = " |" * (depth) + "--%s" % name
return name
def _graph_group(self, group, depth=0):
result = [self._graph_name('@%s:' % group.name, depth)]
depth = depth + 1
for kid in sorted(group.child_groups, key=attrgetter('name')):
result.extend(self._graph_group(kid, depth))
if group.name != 'all':
for host in sorted(group.hosts, key=attrgetter('name')):
result.append(self._graph_name(host.name, depth))
result.extend(self._show_vars(host.get_vars(), depth + 1))
result.extend(self._show_vars(self._get_group_variables(group), depth))
return result
def inventory_graph(self):
start_at = self._get_group(self.options.pattern)
if start_at:
return '\n'.join(self._graph_group(start_at))
else:
raise AnsibleOptionsError(
"Pattern must be valid group name when using --graph")
def json_inventory(self, top):
def format_group(group):
results = {}
results[group.name] = {}
if group.name != 'all':
results[group.name]['hosts'] = [
h.name for h in sorted(group.hosts, key=attrgetter('name'))
]
results[group.name]['children'] = []
for subgroup in sorted(group.child_groups, key=attrgetter('name')):
results[group.name]['children'].append(subgroup.name)
results.update(format_group(subgroup))
if self.options.export:
results[group.name]['vars'] = self._get_group_variables(group)
self._remove_empty(results[group.name])
return results
results = format_group(top)
# populate meta
results['_meta'] = {'hostvars': {}}
hosts = self.inventory.get_hosts()
for host in hosts:
hvars = self._get_host_variables(host)
if hvars:
self._remove_internal(hvars)
results['_meta']['hostvars'][host.name] = hvars
return results
def yaml_inventory(self, top):
seen = []
def format_group(group):
results = {}
# initialize group + vars
results[group.name] = {}
# subgroups
results[group.name]['children'] = {}
for subgroup in sorted(group.child_groups, key=attrgetter('name')):
if subgroup.name != 'all':
results[group.name]['children'].update(
format_group(subgroup))
# hosts for group
results[group.name]['hosts'] = {}
if group.name != 'all':
for h in sorted(group.hosts, key=attrgetter('name')):
myvars = {}
if h.name not in seen: # avoid defining host vars more than once
seen.append(h.name)
myvars = self._get_host_variables(host=h)
self._remove_internal(myvars)
results[group.name]['hosts'][h.name] = myvars
if self.options.export:
gvars = self._get_group_variables(group)
if gvars:
results[group.name]['vars'] = gvars
self._remove_empty(results[group.name])
return results
return format_group(top)
def run(self):
super(PlaybookCLI, self).run()
# Note: slightly wrong, this is written so that implicit localhost
# Manage passwords
sshpass = None
becomepass = None
vault_pass = None
passwords = {}
# don't deal with privilege escalation or passwords when we don't need to
if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax:
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }
loader = DataLoader()
if self.options.vault_password_file:
# read vault_pass from a file
vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
loader.set_vault_password(vault_pass)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords()[0]
loader.set_vault_password(vault_pass)
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
for playbook in self.args:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" % playbook)
if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError("the playbook: %s does not appear to be a file" % playbook)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)
variable_manager.options_vars = load_options_vars(self.options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
if len(inventory.list_hosts()) == 0:
# Empty inventory
display.warning("provided hosts list is empty, only localhost is available")
no_hosts = True
inventory.subset(self.options.subset)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
raise AnsibleError("Specified --limit does not match any hosts")
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(playbooks=self.args, inventory=inventory, variable_manager=variable_manager, loader=loader, options=self.options, passwords=passwords)
results = pbex.run()
if isinstance(results, list):
for p in results:
display.display('\nplaybook: %s' % p['playbook'])
for idx, play in enumerate(p['plays']):
msg = "\n play #%d (%s): %s" % (idx + 1, ','.join(play.hosts), play.name)
mytags = set(play.tags)
msg += '\tTAGS: [%s]' % (','.join(mytags))
if self.options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
display.display(msg)
all_tags = set()
if self.options.listtags or self.options.listtasks:
taskmsg = ''
if self.options.listtasks:
taskmsg = ' tasks:\n'
def _process_block(b):
taskmsg = ''
for task in b.block:
if isinstance(task, Block):
taskmsg += _process_block(task)
else:
if task.action == 'meta':
continue
all_tags.update(task.tags)
if self.options.listtasks:
cur_tags = list(mytags.union(set(task.tags)))
cur_tags.sort()
if task.name:
taskmsg += " %s" % task.get_name()
else:
taskmsg += " %s" % task.action
taskmsg += "\tTAGS: [%s]\n" % ', '.join(cur_tags)
return taskmsg
all_vars = variable_manager.get_vars(loader=loader, play=play)
play_context = PlayContext(play=play, options=self.options)
for block in play.compile():
block = block.filter_tagged_tasks(play_context, all_vars)
if not block.has_tasks():
continue
taskmsg += _process_block(block)
if self.options.listtags:
cur_tags = list(mytags.union(all_tags))
cur_tags.sort()
taskmsg += " TASK TAGS: [%s]\n" % ', '.join(cur_tags)
display.display(taskmsg)
return 0
else:
return results
def run(self):
super(PlaybookCLI, self).run()
# Note: slightly wrong, this is written so that implicit localhost
# Manage passwords
sshpass = None
becomepass = None
vault_pass = None
passwords = {}
# don't deal with privilege escalation or passwords when we don't need to
if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax:
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = { 'conn_pass': sshpass, 'become_pass': becomepass }
if self.options.vault_password_file:
# read vault_pass from a file
vault_pass = CLI.read_vault_password_file(self.options.vault_password_file)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0]
loader = DataLoader(vault_password=vault_pass)
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
for playbook in self.args:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" % playbook)
if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError("the playbook: %s does not appear to be a file" % playbook)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
if len(inventory.list_hosts()) == 0:
# Empty inventory
self.display.warning("provided hosts list is empty, only localhost is available")
no_hosts = True
inventory.subset(self.options.subset)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
raise AnsibleError("Specified --limit does not match any hosts")
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(playbooks=self.args, inventory=inventory, variable_manager=variable_manager, loader=loader, display=self.display, options=self.options, passwords=passwords)
results = pbex.run()
if isinstance(results, list):
for p in results:
self.display.display('\nplaybook: %s' % p['playbook'])
i = 1
for play in p['plays']:
if play.name:
playname = play.name
else:
playname = '#' + str(i)
msg = "\n PLAY: %s" % (playname)
mytags = set()
if self.options.listtags and play.tags:
mytags = mytags.union(set(play.tags))
msg += ' TAGS: [%s]' % (','.join(mytags))
if self.options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
self.display.display(msg)
if self.options.listtags or self.options.listtasks:
taskmsg = ' tasks:'
for block in play.compile():
if not block.has_tasks():
continue
j = 1
for task in block.block:
taskmsg += "\n %s" % task
if self.options.listtags and task.tags:
taskmsg += " TAGS: [%s]" % ','.join(mytags.union(set(task.tags)))
j = j + 1
self.display.display(taskmsg)
i = i + 1
return 0
else:
return results
def listfunction(llist):
"""This function does all the work and lists groups or hosts"""
llist = llist
variable_manager = VariableManager()
loader = DataLoader()
if not os.path.isfile(inventory_file):
print "%s is not a file - halting. Consider using the '--inventory $path/to/ansible_inventory file' parameter" % inventory_file
sys.exit(1)
else:
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=inventory_file)
groups = inventory.groups
for group in groups:
dogroup = inventory.get_group(group)
if dogroup.child_groups:
list_of_hosts = dogroup.child_groups
# we need to quote the items in the list because JSON
list_of_hosts2 = ','.join("'{0}'".format(x) for x in list_of_hosts)
list_of_hosts3 = list_of_hosts2.replace('["', '[')
list_of_hosts4 = list_of_hosts3.replace('"]', ']')
# the three lists
dict_groups2[group] = list_of_hosts # used with --single
dict_groups[group] = [list_of_hosts4] # used with --list and --group
dict_hosts[group] = inventory.list_hosts(group) # used with --single
## Nested child groups
# To make this work with optionally nested groups (example up top) we need to after the group is populated, look in dict_groups if there are any double-nested groups.
if chosen_group and single:
for group in dict_groups2[chosen_group]:
try:
dict_groups2[str(group)]
child_child_groups.append(str(group))
except KeyError:
# this group does not have a child group!
continue
# here we loop through child_child groups and list the hosts of those groups and then add the last in each group to the dict_single_hosts
if child_child_groups != []:
for child in child_child_groups:
for child_child in dict_groups2[child]:
child_host = inventory.get_hosts(str(child_child))[-1]
dict_single_hosts[child_child] = [child_host]
## End of this child of mine
# make a dict that only has one host per child group
if chosen_group:
for host in dict_hosts[chosen_group]:
groups_for_host = inventory.groups_for_host(str(host))
# if debug: print "groups_for_host: %s" % groups_for_host
for group in groups_for_host:
if group in dict_groups2[chosen_group]:
#this host is in one of the child groups of the chosen_group
if len(dict_single_hosts[chosen_group]) == 0:
dict_single_hosts[group] = [host]
# here we populate dict_single_hosts so that the chosen_group key only has a list of hosts that are in separate child groups
for group in dict_single_hosts:
if chosen_group == group:
continue
if len(dict_single_hosts[chosen_group]) < (len(dict_single_hosts) - 1):
# -1 because the chosen_group is also in the same dict
for host in dict_single_hosts[group]:
# and we first check if it's already in there, that might have been added by the child_child hosts
if host not in dict_single_hosts[chosen_group]:
dict_single_hosts[chosen_group].append(host)
# here we quote the entries in dict_of_single_hosts (because JSON)
if single:
list_of_single_hosts = dict_single_hosts[chosen_group]
list_of_single_hosts2 = ','.join("'{0}'".format(x) for x in list_of_single_hosts)
dict_single_hosts[chosen_group] = []
dict_single_hosts[chosen_group] = [list_of_single_hosts2]
## ########
# Some arguments checking - this could probably be done with argparse settings
if chosen_group:
if single:
return dict_single_hosts[chosen_group]
else:
return dict_groups[chosen_group]
else:
return dict_groups
def run(self):
super(PlaybookCLI, self).run()
# Note: slightly wrong, this is written so that implicit localhost
# Manage passwords
sshpass = None
becomepass = None
vault_pass = None
passwords = {}
# don't deal with privilege escalation or passwords when we don't need to
if (
not self.options.listhosts
and not self.options.listtasks
and not self.options.listtags
and not self.options.syntax
):
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = {"conn_pass": sshpass, "become_pass": becomepass}
loader = DataLoader()
if self.options.vault_password_file:
# read vault_pass from a file
vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader)
loader.set_vault_password(vault_pass)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0]
loader.set_vault_password(vault_pass)
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
for playbook in self.args:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" % playbook)
if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError("the playbook: %s does not appear to be a file" % playbook)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
if len(inventory.list_hosts()) == 0:
# Empty inventory
self.display.warning("provided hosts list is empty, only localhost is available")
no_hosts = True
inventory.subset(self.options.subset)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
raise AnsibleError("Specified --limit does not match any hosts")
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(
playbooks=self.args,
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
display=self.display,
options=self.options,
passwords=passwords,
)
results = pbex.run()
if isinstance(results, list):
for p in results:
self.display.display("\nplaybook: %s" % p["playbook"])
i = 1
for play in p["plays"]:
if play.name:
playname = play.name
else:
playname = "#" + str(i)
msg = "\n PLAY: %s" % (playname)
mytags = set()
if self.options.listtags and play.tags:
mytags = mytags.union(set(play.tags))
msg += " TAGS: [%s]" % (",".join(mytags))
if self.options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
self.display.display(msg)
if self.options.listtags or self.options.listtasks:
taskmsg = " tasks:"
for block in play.compile():
if not block.has_tasks():
continue
j = 1
for task in block.block:
taskmsg += "\n %s" % task
if self.options.listtags and task.tags:
taskmsg += " TAGS: [%s]" % ",".join(mytags.union(set(task.tags)))
j = j + 1
self.display.display(taskmsg)
i = i + 1
return 0
else:
return results
def test_get_hosts(self):
inventory = Inventory("127.0.0.1,192.168.1.1")
hosts = inventory.get_hosts("!10.0.0.1")
hosts_all = inventory.get_hosts("all")
self.assertEqual(sorted(hosts), sorted(hosts_all))
def main(argv):
#parse args
phase = 0
if len(argv)!=2:
exit(1)
if argv[1]=="clean":
phase = 0
elif argv[1] == "inventory":
phase = 1
elif argv[1] == "deploy":
phase = 2
with open("keys/rootkey.csv","r") as keyfile:
lines = keyfile.readlines()
aws_access_key = lines[0].split('=')[1].strip(' ').rstrip()
#print aws_access_key
aws_secret_key = lines[1].split('=')[1].strip(' ').rstrip()
#print aws_secret_key
d_token = lines[2].split('=')[1].strip(' ').rstrip()
#print d_token
os.environ['AWS_ACCESS_KEY_ID']= aws_access_key
os.environ['AWS_SECRET_ACCESS_KEY']= aws_secret_key
if phase == 0 or phase == 1:
d = deployment(d_token)
print "Clean up stale reservations...*****************\n"
d.destroy_aws_instance()
d.destroy_digital_instance()
if phase == 1:
print "\nCreating digitalocean droplet...**************"
d.create_digital_instance()
print "\nCreating AWS EC2 instance...******************"
d.create_aws_instance()
print "\nCheck Droplet status...***********************"
dropletIp = d.get_digital_reservation()
while dropletIp == None:
print "Droplet not ready, will retry after 30 sec"
time.sleep(30)
dropletIp = d.get_digital_reservation()
print "Droplet IP ="+dropletIp
print "\nCheck AWS instance status...******************"
aws_ip = d.get_aws_reservation()
while aws_ip == None:
print "AWS Instance not ready, retry after 30 sec"
time.sleep(30)
aws_ip = d.get_aws_reservation()
print "AWS instance IP =" + aws_ip
digital_inv = "droplet ansible_ssh_host="+dropletIp+" ansible_ssh_user=root ansible_ssh_private_key_file=./keys/hw1.key\n"
aws_inv = "aws ansible_ssh_host="+aws_ip+" ansible_ssh_user=ubuntu ansible_ssh_private_key_file=./keys/aws_hw1.key"
print "\nWriting Inventory...**************************"
with open("inventory","w") as f:
f.write(digital_inv)
f.write(aws_inv)
if phase == 2:
os.environ['ANSIBLE_HOST_KEY_CHECKING']="false"
utils.VERBOSITY = 0
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
stats = callbacks.AggregateStats()
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
inventory = Inventory('inventory')
print "\nRun Ansible PlayBook...**********************"
pb = PlayBook(playbook='server_play.yml',
inventory=inventory,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats
)
pb.run()
print "\nOpening web browser...***********************"
for host in inventory.get_hosts():
print "Opening nginx page on ", host.name, host.vars['ansible_ssh_host']
webbrowser.open_new("http://"+host.vars['ansible_ssh_host'])
time.sleep(1)
def default(self):
verbose = self.app.pargs.verbose
init(autoreset=True)
if len(self.app.pargs.extra_arguments) == 0:
self.app.args.print_help()
print(Fore.RED + 'Error! You must specify a playbook file to run')
sys.exit(1)
if str(self.app.pargs.become_method).upper() not in [
'SUDO', 'SU', 'PBRUN', 'PFEXEC', 'DOAS', 'DZDO', 'KSU', 'RUNAS'
]:
print(
Fore.RED +
'Error! Become method must be one of sudo, su, pbrun, pfexec, doas, dzdo, ksu or runas.'
)
sys.exit(1)
#if len(self.app.pargs.extra_arguments) > 0:
playbook_path = self.app.pargs.extra_arguments[0] if len(
self.app.pargs.extra_arguments) > 0 else "site.yml"
if not os.path.isfile(playbook_path):
print(Fore.RED + 'Error! The playbook file does not exist')
sys.exit(1)
inventory_path = self.app.pargs.inventory
if not os.path.isfile(inventory_path):
print(Fore.RED + 'Error! The inventory file does not exist.')
sys.exit(1)
# Most of the code from here down is straight copy & paste from ansible source code,
# with a few tweaks/hacks to clean up variables that we don't want to emit in the generated
# YAML.
loader = DataLoader()
pb_cli = PlaybookCLI(sys.argv[1:])
pb_cli.parse()
# !!!!! WARNING: THESE WILL BE INCLUDED IN THE GENERATED YAML FILE !!!!!
(ssh_pass, become_pass) = pb_cli.ask_passwords()
passwords = {'conn_pass': ssh_pass, 'become_pass': become_pass}
vault_pass = None
if self.app.pargs.ask_vault_pass:
vault_pass = pb_cli.ask_vault_passwords()
else:
vault_pass = pb_cli.read_vault_password_file(
self.app.pargs.vault_password_file, loader)
if vault_pass is not None:
loader.set_vault_password(vault_pass)
# create the inventory, and filter it based on the subset specified (if any)
host_list = self.app.pargs.inventory if self.app.pargs.inventory else constants.DEFAULT_HOST_LIST
# FIXME: This should parse any arguments provided via -e or --extra-vars. Currently it seems
# to parse the argument value wrongly and returns
# '_raw_params': '<last character in variable>'. This prevents the ability to use
# --extra-vars when executing plays.
variable_manager = VariableManager()
options = Options(
verbosity=self.app.pargs.verbose,
inventory=self.app.pargs.inventory,
subset=self.app.pargs.limit,
extra_vars=[self.app.pargs.extra_vars],
forks=self.app.pargs.forks,
ask_vault_pass=self.app.pargs.ask_vault_pass,
vault_password_files=self.app.pargs.vault_password_file,
tags=self.app.pargs.tags,
skip_tags=self.app.pargs.skip_tags,
become=self.app.pargs.become,
become_method=self.app.pargs.become_method,
become_user=self.app.pargs.become_user,
become_ask_pass=self.app.pargs.ask_become_pass,
ask_pass=self.app.pargs.ask_pass,
private_key_file=self.app.pargs.private_key,
remote_user=self.app.pargs.user,
connection=self.app.pargs.connection,
timeout=self.app.pargs.timeout,
ssh_common_args=self.app.pargs.ssh_common_args,
sftp_extra_args=self.app.pargs.sftp_extra_args,
ssh_extra_args=self.app.pargs.ssh_extra_args,
scp_extra_args=self.app.pargs.scp_extra_args,
flush_cache=self.app.pargs.flush_cache,
module_path=self.app.pargs.module_path)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=host_list)
variable_manager.extra_vars = load_extra_vars(loader=loader,
options=options)
#variable_manager.options_vars = load_options_vars(options)
if self.app.pargs.flush_cache:
inventory.refresh_inventory()
no_hosts = False
if len(inventory.list_hosts()) == 0:
print(
Fore.YELLOW +
"Warning! Provided hosts list is empty, only localhost is available."
)
no_hosts = True
# FIXME: Limit is currently ignored.
inventory.subset(self.app.pargs.limit)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
print(Fore.RED +
"Error! Specified --limit does not match any hosts.")
sys.exit(1)
play_data = loader.load_from_file(playbook_path)
host_list = inventory.get_hosts(
pattern=self.app.pargs.limit
) if self.app.pargs.limit else inventory.get_hosts()
playbook = Playbook(loader=loader).load(
playbook_path, variable_manager=variable_manager, loader=loader)
pbex = PlaybookExecutor(playbooks=playbook.get_plays(),
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=passwords)
# Ansible variables matching these strings will be excluded from the generated yaml content.
ignored_elements = [
'ansible_playbook_python', 'ansible_version', 'group_names',
'inventory_hostname', 'inventory_hostname_short', 'omit',
'playbook_dir', 'role_names'
]
json_data = {}
for host in host_list:
host_vars = host.get_vars()
host_name = host_vars[
'ansible_host'] if 'ansible_host' in host_vars.keys(
) else host.get_name()
for play in playbook.get_plays():
loader.set_vault_password(vault_pass)
all_vars = variable_manager.get_vars(loader=loader,
play=play,
host=host)
json_data[host_name] = all_vars['vars']
json_data[host_name]['ansible_groups'] = all_vars[
'group_names']
json_data[host_name]['ansible_roles'] = all_vars['role_names']
for elem in ignored_elements:
del json_data['{}'.format(host)][elem]
json_data[host_name][
'ansible_become_user'] = self.app.pargs.become_user
if passwords['become_pass'] is not None:
json_data[host_name]['ansible_become_pass'] = passwords[
'become_pass']
json_data[host_name]['ansible_become'] = self.app.pargs.become
json_data[host_name][
'ansible_become_method'] = self.app.pargs.become_method
if self.app.pargs.ssh_extra_args:
json_data[host_name][
'ansible_ssh_extra_args'] = self.app.pargs.ssh_extra_args
if self.app.pargs.scp_extra_args:
json_data[host_name][
'ansible_scp_extra_args'] = self.app.pargs.scp_extra_args
if self.app.pargs.sftp_extra_args:
json_data[host_name][
'ansible_sftp_extra_args'] = self.app.pargs.sftp_extra_args
if self.app.pargs.ssh_common_args:
json_data[host_name][
'ansible_ssh_common_args'] = self.app.pargs.ssh_common_args
json_data[host_name][
'ansible_connection_timeout'] = self.app.pargs.timeout
json_data[host_name][
'ansible_connection_method'] = self.app.pargs.connection
if self.app.pargs.private_key:
json_data[host_name][
'ansible_private_key'] = self.app.pargs.private_key
json_data[host_name][
'ansible_ask_pass'] = self.app.pargs.ask_pass
if self.app.pargs.limit:
json_data[host_name][
'ansible_limit'] = self.app.pargs.limit
# FIXME: Extra vars needs to be processed by Ansible instead of dumping them
# here as a "dumb" string.
if self.app.pargs.extra_vars:
json_data[host_name][
'ansible_extra_vars'] = self.app.pargs.extra_vars
for key, value in json_data[host_name].iteritems():
if type(value) is AnsibleVaultEncryptedUnicode:
json_data[host_name][key] = str(value)
# Convert the processed python dictionary to a valid json string
json_obj = json.dumps(json_data)
# Take the json string, and convert it to a valid yaml string
yml = YAML(typ="safe", pure=True)
yml.default_flow_style = False
yml_obj = yml.load(json_obj)
if self.app.pargs.output is None:
yml.dump(yml_obj, sys.stdout)
else:
with open(self.app.pargs.output, 'w') as outfile:
yml.dump(yml_obj, outfile)
print(Fore.GREEN +
"Ansible variable dictionary written to {}".format(
self.app.pargs.output))
sys.exit(1)
class InventoryCLI(CLI):
''' used to display or dump the configured inventory as Ansible sees it '''
ARGUMENTS = {
'host':
'The name of a host to match in the inventory, relevant when using --list',
'group':
'The name of a group in the inventory, relevant when using --graph',
}
def __init__(self, args):
super(InventoryCLI, self).__init__(args)
self.args = args
self.vm = None
self.loader = None
self.inventory = None
self._new_api = True
def parse(self):
self.parser = CLI.base_parser(
usage='usage: %prog [options] [host|group]',
epilog=
'Show Ansible inventory information, by default it uses the inventory script JSON format',
inventory_opts=True,
vault_opts=True)
self.parser.add_option(
"--optimize",
action="store_true",
default=False,
dest='optimize',
help='Output variables on the group or host where they are defined'
)
# Actions
action_group = optparse.OptionGroup(
self.parser, "Actions",
"One of following must be used on invocation, ONLY ONE!")
action_group.add_option(
"--list",
action="store_true",
default=False,
dest='list',
help='Output all hosts info, works as inventory script')
action_group.add_option(
"--host",
action="store",
default=None,
dest='host',
help='Output specific host info, works as inventory script')
action_group.add_option(
"--graph",
action="store_true",
default=False,
dest='graph',
help=
'create inventory graph, if supplying pattern it must be a valid group name'
)
self.parser.add_option_group(action_group)
# Options
self.parser.add_option(
"-y",
"--yaml",
action="store_true",
default=False,
dest='yaml',
help='Use YAML format instead of default JSON, ignored for --graph'
)
self.parser.add_option(
"--vars",
action="store_true",
default=False,
dest='show_vars',
help='Add vars to graph display, ignored unless used with --graph')
try:
super(InventoryCLI, self).parse()
except Exception as e:
if 'Need to implement!' not in e.args[0]:
raise
# --- Start of 2.3+ super(InventoryCLI, self).parse() ---
self.options, self.args = self.parser.parse_args(self.args[1:])
# --- End of 2.3+ super(InventoryCLI, self).parse() ---
display.verbosity = self.options.verbosity
self.validate_conflicts(vault_opts=True)
# there can be only one! and, at least, one!
used = 0
for opt in (self.options.list, self.options.host, self.options.graph):
if opt:
used += 1
if used == 0:
raise AnsibleOptionsError(
"No action selected, at least one of --host, --graph or --list needs to be specified."
)
elif used > 1:
raise AnsibleOptionsError(
"Conflicting options used, only one of --host, --graph or --list can be used at the same time."
)
# set host pattern to default if not supplied
if len(self.args) > 0:
self.options.pattern = self.args[0]
else:
self.options.pattern = 'all'
def run(self):
results = None
super(InventoryCLI, self).run()
# Initialize needed objects
if getattr(self, '_play_prereqs', False):
self.loader, self.inventory, self.vm = self._play_prereqs(
self.options)
else:
# fallback to pre 2.4 way of initialzing
from ansible.vars import VariableManager
from ansible.inventory import Inventory
self._new_api = False
self.loader = DataLoader()
self.vm = VariableManager()
# use vault if needed
if self.options.vault_password_file:
vault_pass = CLI.read_vault_password_file(
self.options.vault_password_file, loader=self.loader)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords()
else:
vault_pass = None
if vault_pass:
self.loader.set_vault_password(vault_pass)
# actually get inventory and vars
self.inventory = Inventory(loader=self.loader,
variable_manager=self.vm,
host_list=self.options.inventory)
self.vm.set_inventory(self.inventory)
if self.options.host:
hosts = self.inventory.get_hosts(self.options.host)
if len(hosts) != 1:
raise AnsibleOptionsError(
"You must pass a single valid host to --hosts parameter")
myvars = self._get_host_variables(host=hosts[0])
self._remove_internal(myvars)
# FIXME: should we template first?
results = self.dump(myvars)
elif self.options.graph:
results = self.inventory_graph()
elif self.options.list:
top = self._get_group('all')
if self.options.yaml:
results = self.yaml_inventory(top)
else:
results = self.json_inventory(top)
results = self.dump(results)
if results:
# FIXME: pager?
display.display(results)
exit(0)
exit(1)
def dump(self, stuff):
if self.options.yaml:
import yaml
from ansible.parsing.yaml.dumper import AnsibleDumper
results = yaml.dump(stuff,
Dumper=AnsibleDumper,
default_flow_style=False)
else:
import json
results = json.dumps(stuff, sort_keys=True, indent=4)
return results
def _get_host_variables(self, host):
if self._new_api:
hostvars = self.vm.get_vars(host=host)
else:
hostvars = self.vm.get_vars(self.loader, host=host)
return hostvars
def _get_group(self, gname):
if self._new_api:
group = self.inventory.groups.get(gname)
else:
group = self.inventory.get_group(gname)
return group
def _remove_internal(self, dump):
for internal in INTERNAL_VARS:
if internal in dump:
del dump[internal]
def _remove_empty(self, dump):
# remove empty keys
for x in ('hosts', 'vars', 'children'):
if x in dump and not dump[x]:
del dump[x]
def _show_vars(self, dump, depth):
result = []
self._remove_internal(dump)
if self.options.show_vars:
for (name, val) in sorted(dump.items()):
result.append(
self._graph_name('{%s = %s}' % (name, val), depth + 1))
return result
def _graph_name(self, name, depth=0):
if depth:
name = " |" * (depth) + "--%s" % name
return name
def _graph_group(self, group, depth=0):
result = [self._graph_name('@%s:' % group.name, depth)]
depth = depth + 1
for kid in sorted(group.child_groups, key=attrgetter('name')):
result.extend(self._graph_group(kid, depth))
if group.name != 'all':
for host in sorted(group.hosts, key=attrgetter('name')):
result.append(self._graph_name(host.name, depth))
result.extend(self._show_vars(host.get_vars(), depth))
result.extend(self._show_vars(group.get_vars(), depth))
return result
def inventory_graph(self):
start_at = self._get_group(self.options.pattern)
if start_at:
return '\n'.join(self._graph_group(start_at))
else:
raise AnsibleOptionsError(
"Pattern must be valid group name when using --graph")
def json_inventory(self, top):
def format_group(group):
results = {}
results[group.name] = {}
if group.name != 'all':
results[group.name]['hosts'] = [
h.name for h in sorted(group.hosts, key=attrgetter('name'))
]
results[group.name]['vars'] = group.get_vars()
results[group.name]['children'] = []
for subgroup in sorted(group.child_groups, key=attrgetter('name')):
results[group.name]['children'].append(subgroup.name)
results.update(format_group(subgroup))
self._remove_empty(results[group.name])
return results
results = format_group(top)
# populate meta
results['_meta'] = {'hostvars': {}}
hosts = self.inventory.get_hosts()
for host in hosts:
results['_meta']['hostvars'][host.name] = self._get_host_variables(
host=host)
self._remove_internal(results['_meta']['hostvars'][host.name])
return results
def yaml_inventory(self, top):
seen = []
def format_group(group):
results = {}
# initialize group + vars
results[group.name] = {}
results[group.name]['vars'] = group.get_vars()
# subgroups
results[group.name]['children'] = {}
for subgroup in sorted(group.child_groups, key=attrgetter('name')):
if subgroup.name != 'all':
results[group.name]['children'].update(
format_group(subgroup))
# hosts for group
results[group.name]['hosts'] = {}
if group.name != 'all':
for h in sorted(group.hosts, key=attrgetter('name')):
myvars = {}
if h.name not in seen: # avoid defining host vars more than once
seen.append(h.name)
myvars = self._get_host_variables(host=h)
self._remove_internal(myvars)
results[group.name]['hosts'][h.name] = myvars
self._remove_empty(results[group.name])
return results
return format_group(top)
def run_playbook(playbook, form_data, runner):
# Mark play as started
runner.pid = os.getpid()
runner.status = 'started'
runner.save()
# Create ansible default objects
variable_manager = VariableManager()
loader = DataLoader()
inventory = Inventory(loader=loader, variable_manager=variable_manager)
variable_manager.set_inventory(inventory)
# Set inventory subset if available:
if 'subset' in form_data:
inventory.subset(form_data['subset'])
# Add host list to runner object
host_list = inventory.get_hosts(pattern=runner.hosts)
# Create password dictionary
passwords = {'conn_pass': form_data['remote_pass'], 'become_pass': form_data['become_pass']}
# Set sudo user
become_user = c.DEFAULT_BECOME_USER
if 'sudo_user' in form_data:
become_user = form_data['sudo_user']
# Create ansible options tuple
options = AnsibleOptions(connection='paramiko',
module_path=c.DEFAULT_MODULE_PATH,
forks=c.DEFAULT_FORKS,
remote_user=form_data['username'],
private_key_file=form_data['rsa_key'],
ssh_common_args=None,
ssh_extra_args=None,
sftp_extra_args=None,
scp_extra_args=None,
become=form_data['become'],
become_method=c.DEFAULT_BECOME_METHOD,
become_user=become_user,
verbosity=None,
check=form_data['check'],
tags=form_data['tags'],
skip_tags=None)
# Create ansible play object
play = Play().load(playbook, variable_manager=variable_manager, loader=loader)
# Execute play
tqm = None
if True:
# try:
tqm = TaskQueueManager(inventory=inventory,
variable_manager=variable_manager,
passwords=passwords,
loader=loader,
options=options,
# stdout_callback=TestCallback()
stdout_callback=AdHocCallback(runner, host_list))
tqm.run(play)
# finally:
if tqm is not None:
tqm.cleanup()
runner.status = 'finished'
else:
runner.status = 'failed'
runner.message = 'tqm object in not None'
runner.save()
debug("waiting for pending results (%d left)" % pending_results)
_process_pending_results()
time.sleep(0.01)
debug("starting")
cur_worker = 0
pending_results = 0
var_manager = VariableManager()
debug("loading inventory")
inventory = Inventory(host_list='/tmp/med_inventory',
loader=loader,
variable_manager=var_manager)
hosts = inventory.get_hosts()[:]
debug("done loading inventory")
ci = ConnectionInformation()
ci.connection = 'local'
for i in range(NUM_TASKS):
#for j in range(NUM_HOSTS):
for h in hosts:
debug("queuing %s %d" % (h, i))
#h = Host(name="host%06d" % j)
t = Task().load(
dict(name="task %d" % (i, ),
debug="msg='hello from %s, %d'" % (h, i)))
#t = Task().load(dict(name="task %d" % (i,), ping=""))
#task_vars = var_manager.get_vars(loader=loader, host=h, task=t)
def run_ansible_on_host(host, logger, driver_config):
from ansible.plugins.callback import CallbackBase
# A rough logger that logs dict messages to standard logger
class ResultCallback(CallbackBase):
def __init__(self):
super(ResultCallback, self).__init__()
def v2_runner_on_ok(self, result, **kwargs):
self.log('ok :' + str(result._result))
def v2_runner_on_failed(self, result, **kwargs):
warnings = result._result['warnings']
error = result._result['stderr']
if warnings:
self.log('warning : ' + str(result._result))
elif error:
self.log('error : ' + str(result._result), info=True)
def v2_runner_on_skipped(self, result, **kwargs):
self.log('skipped : ' + str(result._result))
def v2_runner_on_unreachable(self, result, **kwargs):
self.log('unreachable : ' + str(result._result), info=True)
def v2_playbook_on_no_hosts_matched(self, *args, **kwargs):
self.log('no hosts matched!')
def v2_playbook_on_no_hosts_remaining(self, *args, **kwargs):
self.log('NO MORE HOSTS LEFT')
def v2_playbook_on_task_start(self, task, **kwargs):
self.log('starting task: ' + str(task))
def v2_playbook_on_start(self, playbook, **kwargs):
self.log('starting playbook' + str(playbook), info=True)
def v2_playbook_on_play_start(self, play, **kwargs):
self.log('starting play' + str(play), info=True)
def v2_playbook_on_stats(self, stats, info=True, **kwargs):
self.log('STATS FOR PLAY')
hosts = sorted(stats.processed.keys())
hosts.extend(stats.failures.keys())
hosts.extend(stats.dark.keys())
hosts.extend(stats.changed.keys())
hosts.extend(stats.skipped.keys())
hosts.extend(stats.ok.keys())
for h in hosts:
t = stats.summarize(h)
self.log(str(t))
def log(self, param, info=False):
if not info:
logger.debug(str(param))
else:
logger.info(str(param))
from ansible.parsing.dataloader import DataLoader
from ansible.inventory import Inventory, Group, Host
from ansible.executor import playbook_executor
from ansible.vars import VariableManager
from collections import namedtuple
Options = namedtuple(
'Options',
[
'connection',
'module_path',
'forks',
'become',
'become_method',
'become_user',
'check',
'ansible_user',
'listhosts',
'listtasks',
'listtags',
'syntax',
'ssh_private_key_file',
'host_key_checking'
]
)
options = Options(
connection='ssh',
become=True,
become_method='sudo',
become_user='******',
check=False,
module_path=None,
forks=100,
ansible_user='******',
listhosts=False,
listtasks=False,
listtags=False,
syntax=False,
ssh_private_key_file=PEBBLES_SSH_KEY_LOCATION,
host_key_checking=False
)
variable_manager = VariableManager()
loader = DataLoader()
a_host = Host(name=host['private_ip'])
a_host.set_variable('ansible_host', host['private_ip'])
a_group = Group(name='notebook_host')
a_group.add_host(a_host)
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=[host['private_ip']])
inventory.add_group(a_group)
variable_manager.set_inventory(inventory)
logger.debug('HOST:')
logger.debug(a_host.serialize() )
logger.debug('HOSTs from inventory:')
# for some reason setting these before adding the host to inventory didn't work so well
# ToDo: read up on variable_manager and figure out a more elegant way to set the variables
for h_ in inventory.get_hosts():
h_.set_variable('ansible_user', 'cloud-user')
h_.set_variable('ansible_ssh_common_args', '-o StrictHostKeyChecking=no')
h_.set_variable('ansible_ssh_private_key_file', '/home/pebbles/.ssh/id_rsa')
extra_vars = dict()
extra_vars['ansible_ssh_extra_args'] = '-o StrictHostKeyChecking=no'
logger.debug('Setting driver config....')
if 'DD_HOST_DATA_VOLUME_DEVICE' in driver_config:
extra_vars['notebook_host_block_dev_path'] = driver_config['DD_HOST_DATA_VOLUME_DEVICE']
variable_manager.extra_vars = extra_vars
pb_executor = playbook_executor.PlaybookExecutor(
playbooks=['/webapps/pebbles/source/ansible/notebook_playbook.yml'],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None
)
rescb = ResultCallback()
pb_executor._tqm._stdout_callback = rescb
logger.info('_prepare_host(): running ansible')
logger.info('_prepare_host(): inventory hosts')
for h_ in inventory.get_hosts():
logger.info(h_.serialize())
logger.info(h_.get_vars())
pb_executor.run()
stats = pb_executor._tqm._stats
run_success = True
hosts_list = sorted(stats.processed.keys())
if len(hosts_list) == 0:
logger.debug('no hosts handled')
for h in hosts_list:
t = stats.summarize(h)
logger.debug(t)
logger.debug(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
if run_success:
logger.debug('_prepare_host(): run successfull')
else:
logger.debug('_prepare_host(): run failed')
if getattr(pb_executor, '_unreachable_hosts', False):
logger.debug('UNREACHABLE HOSTS ' + str(pb_executor._unreachable_hosts))
if getattr(pb_executor, '_failed_hosts', False):
logger.debug('FAILED_HOSTS ' + str(pb_executor._failed_hosts))
raise RuntimeError('run_ansible_on_host(%s) failed' % host['id'])
logger.debug('_prepare_host(): done running ansible')
def main(argv):
#parse args
phase = 0
if len(argv) != 2:
exit(1)
if argv[1] == "clean":
phase = 0
elif argv[1] == "inventory":
phase = 1
elif argv[1] == "deploy":
phase = 2
with open("keys/rootkey.csv", "r") as keyfile:
lines = keyfile.readlines()
aws_access_key = lines[0].split('=')[1].strip(' ').rstrip()
#print aws_access_key
aws_secret_key = lines[1].split('=')[1].strip(' ').rstrip()
#print aws_secret_key
d_token = lines[2].split('=')[1].strip(' ').rstrip()
#print d_token
os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key
os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_key
if phase == 0 or phase == 1:
d = deployment(d_token)
print "Clean up stale reservations...*****************\n"
d.destroy_aws_instance()
d.destroy_digital_instance()
if phase == 1:
print "\nCreating digitalocean droplet...**************"
d.create_digital_instance()
print "\nCreating AWS EC2 instance...******************"
d.create_aws_instance()
print "\nCheck Droplet status...***********************"
dropletIp = d.get_digital_reservation()
while dropletIp == None:
print "Droplet not ready, will retry after 30 sec"
time.sleep(30)
dropletIp = d.get_digital_reservation()
print "Droplet IP =" + dropletIp
print "\nCheck AWS instance status...******************"
aws_ip = d.get_aws_reservation()
while aws_ip == None:
print "AWS Instance not ready, retry after 30 sec"
time.sleep(30)
aws_ip = d.get_aws_reservation()
print "AWS instance IP =" + aws_ip
digital_inv = "droplet ansible_ssh_host=" + dropletIp + " ansible_ssh_user=root ansible_ssh_private_key_file=./keys/hw1.key\n"
aws_inv = "aws ansible_ssh_host=" + aws_ip + " ansible_ssh_user=ubuntu ansible_ssh_private_key_file=./keys/aws_hw1.key"
print "\nWriting Inventory...**************************"
with open("inventory", "w") as f:
f.write(digital_inv)
f.write(aws_inv)
if phase == 2:
os.environ['ANSIBLE_HOST_KEY_CHECKING'] = "false"
utils.VERBOSITY = 0
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
stats = callbacks.AggregateStats()
runner_cb = callbacks.PlaybookRunnerCallbacks(stats,
verbose=utils.VERBOSITY)
inventory = Inventory('inventory')
print "\nRun Ansible PlayBook...**********************"
pb = PlayBook(playbook='server_play.yml',
inventory=inventory,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats)
pb.run()
print "\nOpening web browser...***********************"
for host in inventory.get_hosts():
print "Opening nginx page on ", host.name, host.vars[
'ansible_ssh_host']
webbrowser.open_new("http://" + host.vars['ansible_ssh_host'])
time.sleep(1)
def run(self):
# Note: slightly wrong, this is written so that implicit localhost
# Manage passwords
sshpass = None
becomepass = None
b_vault_pass = None
passwords = {}
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
for playbook in self.playbooks:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" %
playbook)
if not (os.path.isfile(playbook)
or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError(
"the playbook: %s does not appear to be a file" % playbook)
# don't deal with privilege escalation or passwords when we don't need to
if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax:
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
loader = DataLoader()
if self.options.vault_password_file:
# read vault_pass from a file
b_vault_pass = CLI.read_vault_password_file(
self.options.vault_password_file, loader=loader)
loader.set_vault_password(b_vault_pass)
elif self.options.ask_vault_pass:
b_vault_pass = self.ask_vault_passwords()
loader.set_vault_password(b_vault_pass)
elif 'VAULT_PASS' in os.environ:
loader.set_vault_password(os.environ['VAULT_PASS'])
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
# Subspace injection
option_extra_vars = load_extra_vars(loader=loader,
options=self.options)
option_extra_vars.update(self.extra_vars)
variable_manager.extra_vars = option_extra_vars
# End Subspace injection
variable_manager.options_vars = load_options_vars(self.options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=self.options.inventory)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
if len(inventory.list_hosts()) == 0:
# Empty inventory
display.warning(
"provided hosts list is empty, only localhost is available")
no_hosts = True
inventory.subset(self.options.subset)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
raise AnsibleError("Specified --limit does not match any hosts")
# flush fact cache if requested
if self.options.flush_cache:
self._flush_cache(inventory, variable_manager)
hosts = inventory.get_hosts()
if self.options.subset and not hosts:
raise NoValidHosts(
"The limit <%s> is not included in the inventory: %s" %
(self.options.subset, inventory.host_list))
# create the playbook executor, which manages running the plays via a task queue manager
# Subspace injection
pbex = PlaybookExecutor(playbooks=self.playbooks,
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=self.options,
passwords=passwords)
playbook_map = self._get_playbook_map()
pbex._tqm._stats = SubspaceAggregateStats(playbook_map)
pbex._tqm.load_callbacks()
pbex._tqm.send_callback(
'start_logging',
logger=self.options.logger,
username=self.extra_vars.get('ATMOUSERNAME', "No-User"),
)
for host in inventory._subset:
variables = inventory.get_vars(host)
self.options.logger.info("Vars found for hostname %s: %s" %
(host, variables))
# End Subspace injection
results = pbex.run()
# Subspace injection
stats = pbex._tqm._stats
self.stats = stats
# Nonpersistent fact cache stores 'register' variables. We would like
# to get access to stdout/stderr for specific commands and relay
# some of that information back to the end user.
self.results = dict(pbex._variable_manager._nonpersistent_fact_cache)
# End Subspace injection
if isinstance(results, list):
for p in results:
display.display('\nplaybook: %s' % p['playbook'])
for idx, play in enumerate(p['plays']):
if play._included_path is not None:
loader.set_basedir(play._included_path)
else:
pb_dir = os.path.realpath(
os.path.dirname(p['playbook']))
loader.set_basedir(pb_dir)
msg = "\n play #%d (%s): %s" % (idx + 1, ','.join(
play.hosts), play.name)
mytags = set(play.tags)
msg += '\tTAGS: [%s]' % (','.join(mytags))
if self.options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (
play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
display.display(msg)
all_tags = set()
if self.options.listtags or self.options.listtasks:
taskmsg = ''
if self.options.listtasks:
taskmsg = ' tasks:\n'
def _process_block(b):
taskmsg = ''
for task in b.block:
if isinstance(task, Block):
taskmsg += _process_block(task)
else:
if task.action == 'meta':
continue
all_tags.update(task.tags)
if self.options.listtasks:
cur_tags = list(
mytags.union(set(task.tags)))
cur_tags.sort()
if task.name:
taskmsg += " %s" % task.get_name(
)
else:
taskmsg += " %s" % task.action
taskmsg += "\tTAGS: [%s]\n" % ', '.join(
cur_tags)
return taskmsg
all_vars = variable_manager.get_vars(loader=loader,
play=play)
play_context = PlayContext(play=play,
options=self.options)
for block in play.compile():
block = block.filter_tagged_tasks(
play_context, all_vars)
if not block.has_tasks():
continue
taskmsg += _process_block(block)
if self.options.listtags:
cur_tags = list(mytags.union(all_tags))
cur_tags.sort()
taskmsg += " TASK TAGS: [%s]\n" % ', '.join(
cur_tags)
display.display(taskmsg)
return 0
else:
return results
def list_tags():
# ansible playbook specific opts
# parser.add_argument('--list-tasks', dest='listtasks', action='store_true',
# help="list all tasks that would be executed")
# parser.add_argument('--list-tags', dest='listtags', action='store_true',
# help="list all available tags")
# parser.add_argument('--step', dest='step', action='store_true',
# help="one-step-at-a-time: confirm each task before running")
# parser.add_argument('--start-at-task', dest='start_at_task',
# help="start the playbook at the task matching this name")
#
# parser.add_argument('--inventory_file', help='Inventory Filename', required=True)
# parser.add_argument('--playbook_path', help='Inventory Filename', required=True)
parser = CLI.base_parser(
usage="%prog playbook.yml",
connect_opts=True,
meta_opts=True,
runas_opts=True,
subset_opts=True,
check_opts=True,
inventory_opts=True,
runtask_opts=True,
vault_opts=True,
fork_opts=True,
module_opts=True,
)
# ansible playbook specific opts
parser.add_option('--list-tasks',
dest='listtasks',
action='store_true',
help="list all tasks that would be executed")
parser.add_option('--list-tags',
dest='listtags',
action='store_true',
help="list all available tags")
parser.add_option(
'--step',
dest='step',
action='store_true',
help="one-step-at-a-time: confirm each task before running")
parser.add_option('--start-at-task',
dest='start_at_task',
help="start the playbook at the task matching this name")
options, args = parser.parse_args(args[1:])
# options = parser.parse_args()
inventory_path = options.inventory_file
playbook_path = options.playbook_path
if not os.path.exists(playbook_path):
print('[ERROR] The playbook does not exist')
sys.exit(1)
loader = DataLoader()
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
# for playbook in playbooks:
# if not os.path.exists(playbook):
# raise Exception("the playbook: %s could not be found" % playbook)
# if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
# raise Exception("the playbook: %s does not appear to be a file" % playbook)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
# variable_manager.extra_vars = load_extra_vars(loader=loader, options=options)
# variable_manager.options_vars = load_options_vars(options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=inventory_path)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
if len(inventory.list_hosts()) == 0:
# Empty inventory
print("provided hosts list is empty, only localhost is available")
no_hosts = True
# inventory.subset(options.subset)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
raise Exception("Specified --limit does not match any hosts")
# flush fact cache if requested
# if options.flush_cache:
# for host in inventory.list_hosts():
# variable_manager.clear_facts(host)
passwords = {}
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(playbooks=[playbook_path],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=passwords)
results = pbex.run()
if isinstance(results, list):
for p in results:
print('\nplaybook: %s' % p['playbook'])
for idx, play in enumerate(p['plays']):
msg = "\n play #%d (%s): %s" % (idx + 1, ','.join(
play.hosts), play.name)
mytags = set(play.tags)
msg += '\tTAGS: [%s]' % (','.join(mytags))
if options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (
play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
print(msg)
all_tags = set()
if options.listtags or options.listtasks:
taskmsg = ''
if options.listtasks:
taskmsg = ' tasks:\n'
def _process_block(b):
taskmsg = ''
for task in b.block:
if isinstance(task, Block):
taskmsg += _process_block(task)
else:
if task.action == 'meta':
continue
all_tags.update(task.tags)
if options.listtasks:
cur_tags = list(
mytags.union(set(task.tags)))
cur_tags.sort()
if task.name:
taskmsg += " %s" % task.get_name()
else:
taskmsg += " %s" % task.action
taskmsg += "\tTAGS: [%s]\n" % ', '.join(
cur_tags)
return taskmsg
all_vars = variable_manager.get_vars(loader=loader,
play=play)
play_context = PlayContext(play=play, options=options)
for block in play.compile():
block = block.filter_tagged_tasks(
play_context, all_vars)
if not block.has_tasks():
continue
taskmsg += _process_block(block)
if options.listtags:
cur_tags = list(mytags.union(all_tags))
cur_tags.sort()
taskmsg += " TASK TAGS: [%s]\n" % ', '.join(
cur_tags)
print(taskmsg)
class AnsibleInventoryManager():
"""Class to handle the ansible inventory file.
This class will allow you to add groups/hosts and remove them from the inventory file"""
#FROM http://www.ansibleworks.com/docs/patterns.html
ALLOWED_VARIABLES = ['ansible_ssh_host',
'ansible_ssh_port',
'ansible_ssh_user',
'ansible_ssh_pass',
'ansible_connection',
'ansible_ssh_private_key_file',
'ansible_syslog_facility',
'ansible_python_interpreter',
]
# NOTE:Works for anything such as ruby or perl and works just like
# ansible_python_interpreter.
# This replaces shebang of modules which will run on that host.
def __init__(self, inventory_file="/etc/ansible/hosts"):
self.__inventory_file = inventory_file
self.__dirty = False
if not os.path.exists(self.__inventory_file):
raise AnsibleInventoryManagerFileNotFound("File: %s Not found or not accessible" % self.__inventory_file)
self.__inventory = Inventory(inventory_file)
def get_hosts(self):
"""return the list of hosts
Returns a list host ips"""
host_list = [host.name for host in self.__inventory.get_hosts()]
return host_list
def get_groups(self):
"""return the groups
Returns a list of objects: ansible.inventory.group.Group"""
return self.__inventory.get_groups()
def get_groups_for_host(self,host):
"""return the groups list where the given host appears"""
return self.__inventory.groups_for_host(host)
def get_group(self, groupname):
"""Returns the given group"""
return self.__inventory.get_group(groupname)
def delete_host(self,host_ip,group=""):
"""Removes a host from a given group
if group is empty, removes the host from all the groups
"""
self.__dirty = True
groups = []
if group == "":
groups = [group.name for group in self.get_groups_for_host(host_ip)]
else:
groups.append(group)
for group in groups:
grp = self.__inventory.get_group(group)
new_host_list = [host for host in grp.get_hosts() if host.name != host_ip]
grp.hosts = new_host_list
def add_host(self, host_ip, add_to_root=True, group_list=[], var_list={}):
"""Add a host ip to the ansible host file
This is a simple function to add hosts to
add_to_root = Adds the host to the root group (unnamed)
groups_list: List of groupnames where the host should appears.
var_list: Variable list. see allowed_variables."""
#root group in unnamed, but in the inventory object
# is the 'ungrouped' group
self.__dirty = True
new_host = Host(host_ip)
for key,value in var_list.iteritems():
if self.is_allowed_variable(key):
new_host.set_variable(key,value)
if add_to_root:
if 'ungrouped' not in group_list:
group_list.append('ungrouped')
#Check groups. The ansible inventory should contain each of the groups.
for group in group_list:
if not self.__inventory.get_group(group):
new_group= Group(group)
self.__inventory.add_group(new_group)
grp = self.__inventory.get_group(group)
host_names = [host.name for host in grp.get_hosts()]
if new_host.name not in host_names:
grp.add_host(new_host)
def is_dirty(self):
return self.__dirty
def is_allowed_variable(self, variable):
"""Checks if the given variable is an allowed variable"""
if variable in self.ALLOWED_VARIABLES:
return True
elif re.match("ansible_(.+)_interpreter", variable):
return True
return False
def save_inventory(self, backup_file=""):
"""Saves the inventory file. If a backup_file is given,
a backup will be done before re-write the file"""
try:
if backup_file != "":
copyfile(self.__inventory_file, backup_file)
data = ""
for group in self.__inventory.get_groups():
ingroup = False
if group.name == "all":
continue
elif group.name != "ungrouped":
data += "[%s]\n" % group.name
ingroup = True
strvars = ""
for host in group.get_hosts():
for key, value in host.get_variables().iteritems():
if key in AnsibleInventoryManager.ALLOWED_VARIABLES:
strvars += "%s=%s " % (key, value)
if ingroup:
data += "\t%s\t%s\n" % (host.name, strvars)
else:
data += "%s\t%s\n" % (host.name, strvars)
ansiblehostfile = open(self.__inventory_file, "w")
ansiblehostfile.write(data)
ansiblehostfile.close()
except Exception, e:
error("Error doing the %s backup: %s" % (self.__inventory_file, str(e)))
def test_get_hosts(self):
inventory = Inventory('127.0.0.1,192.168.1.1')
hosts = inventory.get_hosts('!10.0.0.1')
hosts_all = inventory.get_hosts('all')
self.assertEqual(sorted(hosts), sorted(hosts_all))
def run_ansible_on_host(host, logger, config):
from ansible.plugins.callback import CallbackBase
# A rough logger that logs dict messages to standard logger
class ResultCallback(CallbackBase):
def __init__(self):
super(ResultCallback, self).__init__()
def v2_runner_on_ok(self, result, **kwargs):
self.log('ok :' + str(result._result), info=True)
def v2_runner_on_failed(self, result, **kwargs):
self.log(result._result, info=True)
def v2_runner_on_skipped(self, result, **kwargs):
self.log(result._result, info=True)
def v2_runner_on_unreachable(self, result, **kwargs):
self.log(result._result, info=True)
def v2_playbook_on_no_hosts_matched(self, *args, **kwargs):
self.log('no hosts matched!')
def v2_playbook_on_no_hosts_remaining(self, *args, **kwargs):
self.log('NO MORE HOSTS LEFT')
def v2_playbook_on_task_start(self, task, **kwargs):
self.log('starting task: ' + str(task))
def v2_playbook_on_start(self, playbook, **kwargs):
self.log('starting playbook' + str(playbook), info=True)
def v2_playbook_on_play_start(self, play, **kwargs):
self.log('starting play' + str(play), info=True)
def v2_playbook_on_stats(self, stats, info=True, **kwargs):
self.log('STATS FOR PLAY')
hosts = sorted(stats.processed.keys())
hosts.extend(stats.failures.keys())
hosts.extend(stats.dark.keys())
hosts.extend(stats.changed.keys())
hosts.extend(stats.skipped.keys())
hosts.extend(stats.ok.keys())
for h in hosts:
t = stats.summarize(h)
self.log(str(t))
def log(self, param, info=False):
if not info:
logger.debug(str(param))
else:
logger.info(str(param))
from ansible.parsing.dataloader import DataLoader
from ansible.inventory import Inventory, Group, Host
from ansible.executor import playbook_executor
from ansible.vars import VariableManager
from collections import namedtuple
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'become', 'become_method',
'become_user', 'check', 'ansible_user', 'listhosts', 'listtasks',
'listtags', 'syntax', 'ssh_private_key_file', 'host_key_checking'
])
options = Options(connection='ssh',
become=True,
become_method='sudo',
become_user='******',
check=False,
module_path=None,
forks=100,
ansible_user='******',
listhosts=False,
listtasks=False,
listtags=False,
syntax=False,
ssh_private_key_file=PEBBLES_SSH_KEY_LOCATION,
host_key_checking=False)
variable_manager = VariableManager()
loader = DataLoader()
a_host = Host(name=host['private_ip'])
a_host.set_variable('ansible_host', host['private_ip'])
a_group = Group(name='notebook_host')
a_group.add_host(a_host)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[host['private_ip']])
inventory.add_group(a_group)
variable_manager.set_inventory(inventory)
logger.debug('HOST:')
logger.debug(a_host.serialize())
logger.debug('HOSTs from inventory:')
# for some reason setting these before adding the host to inventory didn't work so well
# ToDo: read up on variable_manager and figure out a more elegant way to set the variables
for h_ in inventory.get_hosts():
h_.set_variable('ansible_user', 'cloud-user')
h_.set_variable('ansible_ssh_common_args',
'-o StrictHostKeyChecking=no')
h_.set_variable('ansible_ssh_private_key_file',
'/home/pebbles/.ssh/id_rsa')
extra_vars = dict()
extra_vars['ansible_ssh_extra_args'] = '-o StrictHostKeyChecking=no'
if 'DD_HOST_DATA_VOLUME_DEVICE' in config:
extra_vars['notebook_host_block_dev_path'] = config[
'DD_HOST_DATA_VOLUME_DEVICE']
variable_manager.extra_vars = extra_vars
pb_executor = playbook_executor.PlaybookExecutor(
playbooks=[
'/webapps/pebbles/source/ansible/notebook_playbook.yml'
],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None)
rescb = ResultCallback()
pb_executor._tqm._stdout_callback = rescb
logger.info('_prepare_host(): running ansible')
logger.info('_prepare_host(): inventory hosts')
for h_ in inventory.get_hosts():
logger.info(h_.serialize())
logger.info(h_.get_vars())
pb_executor.run()
stats = pb_executor._tqm._stats
run_success = True
hosts_list = sorted(stats.processed.keys())
if len(hosts_list) == 0:
logger.debug('no hosts handled')
for h in hosts_list:
t = stats.summarize(h)
logger.debug(t)
logger.debug(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
if run_success:
logger.debug('_prepare_host(): run successfull')
else:
logger.debug('_prepare_host(): run failed')
if getattr(pb_executor, '_unreachable_hosts', False):
logger.debug('UNREACHABLE HOSTS ' +
str(pb_executor._unreachable_hosts))
if getattr(pb_executor, '_failed_hosts', False):
logger.debug('FAILED_HOSTS ' + str(pb_executor._failed_hosts))
raise RuntimeError('run_ansible_on_host(%s) failed' % host['id'])
logger.debug('_prepare_host(): done running ansible')
def run(self):
super(PlaybookCLI, self).run()
# Note: slightly wrong, this is written so that implicit localhost
# Manage passwords
sshpass = None
becomepass = None
vault_pass = None
passwords = {}
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
for playbook in self.args:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" %
playbook)
if not (os.path.isfile(playbook)
or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError(
"the playbook: %s does not appear to be a file" % playbook)
# don't deal with privilege escalation or passwords when we don't need to
if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax:
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
loader = DataLoader()
if self.options.vault_password_file:
# read vault_pass from a file
vault_pass = CLI.read_vault_password_file(
self.options.vault_password_file, loader=loader)
loader.set_vault_password(vault_pass)
elif self.options.ask_vault_pass:
vault_pass = self.ask_vault_passwords()[0]
loader.set_vault_password(vault_pass)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
variable_manager.extra_vars = load_extra_vars(loader=loader,
options=self.options)
variable_manager.options_vars = load_options_vars(self.options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=self.options.inventory)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
if len(inventory.list_hosts()) == 0:
# Empty inventory
display.warning(
"provided hosts list is empty, only localhost is available")
no_hosts = True
inventory.subset(self.options.subset)
if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
raise AnsibleError("Specified --limit does not match any hosts")
# flush fact cache if requested
if self.options.flush_cache:
for host in inventory.list_hosts():
variable_manager.clear_facts(host)
# create the playbook executor, which manages running the plays via a task queue manager
pbex = PlaybookExecutor(playbooks=self.args,
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=self.options,
passwords=passwords)
results = pbex.run()
if isinstance(results, list):
for p in results:
display.display('\nplaybook: %s' % p['playbook'])
for idx, play in enumerate(p['plays']):
msg = "\n play #%d (%s): %s" % (idx + 1, ','.join(
play.hosts), play.name)
mytags = set(play.tags)
msg += '\tTAGS: [%s]' % (','.join(mytags))
if self.options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (
play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
display.display(msg)
all_tags = set()
if self.options.listtags or self.options.listtasks:
taskmsg = ''
if self.options.listtasks:
taskmsg = ' tasks:\n'
def _process_block(b):
taskmsg = ''
for task in b.block:
if isinstance(task, Block):
taskmsg += _process_block(task)
else:
if task.action == 'meta':
continue
all_tags.update(task.tags)
if self.options.listtasks:
cur_tags = list(
mytags.union(set(task.tags)))
cur_tags.sort()
if task.name:
taskmsg += " %s" % task.get_name(
)
else:
taskmsg += " %s" % task.action
taskmsg += "\tTAGS: [%s]\n" % ', '.join(
cur_tags)
return taskmsg
all_vars = variable_manager.get_vars(loader=loader,
play=play)
play_context = PlayContext(play=play,
options=self.options)
for block in play.compile():
block = block.filter_tagged_tasks(
play_context, all_vars)
if not block.has_tasks():
continue
taskmsg += _process_block(block)
if self.options.listtags:
cur_tags = list(mytags.union(all_tags))
cur_tags.sort()
taskmsg += " TASK TAGS: [%s]\n" % ', '.join(
cur_tags)
display.display(taskmsg)
return 0
else:
return results
def populateModelFromInventory(model, inventory):
if HOSTS not in model:
model[HOSTS] = []
if HOST_GROUPS not in model:
model[HOST_GROUPS] = []
loader = DataLoader()
if inventoryHandler == PRE24:
if VAULT_PASSWORD_FILE in inventory:
vpf = inventory[VAULT_PASSWORD_FILE]
if not os.path.exists(vpf):
misc.ERROR(
"Ansible vault password file '{0}' does not exists!".
format(vpf))
with open(vpf) as f:
content = f.readlines()
if len(content) == 0 or len(content[0].strip()) == 0:
misc.ERROR(
"Invalid Ansible vault password file '{0}' content!".
format(vpf))
loader.set_vault_password(content[0].strip())
elif ASK_VAULT_PASSWORD in inventory and inventory[ASK_VAULT_PASSWORD]:
prompt = "Password for Ansible inventory{0}: ".format(
" '" + inventory[NAME] + "'" if NAME in inventory else "")
content = getpass.getpass(prompt)
loader.set_vault_password(content.strip())
variable_manager = VariableManager()
try:
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=inventory[FILE])
except Exception as e:
misc.ERROR(str(e))
variable_manager.set_inventory(inventory)
hosts = inventory.get_hosts()
else:
if VAULT_PASSWORD_FILE in inventory:
from ansible.cli import CLI
vpf = inventory[VAULT_PASSWORD_FILE]
if not os.path.exists(vpf):
misc.ERROR(
"Ansible vault password file '{0}' does not exists!".
format(vpf))
CLI.setup_vault_secrets(loader=loader,
vault_ids=[],
vault_password_files=[vpf],
ask_vault_pass=False,
create_new_password=False)
elif ASK_VAULT_PASSWORD in inventory and inventory[ASK_VAULT_PASSWORD]:
from ansible.cli import CLI
CLI.setup_vault_secrets(loader=loader,
vault_ids=[],
vault_password_files=[],
ask_vault_pass=True,
create_new_password=False)
inventory = InventoryManager(loader=loader, sources=inventory[FILE])
variable_manager = VariableManager(loader=loader, inventory=inventory)
hosts = inventory.get_hosts()
existingHosts = set(map(lambda x: x[NAME], model[HOSTS]))
#print(existingHosts)
for host in hosts:
if not host.name in existingHosts:
h = {}
h[NAME] = host.name.encode('utf8')
for key in host.vars:
if key.startswith('ansible_'):
key2 = key[len('ansible_'):].encode('utf8')
#print "key2:{}".format(key2)
if not key2.startswith("ssh_"):
key2 = "ssh_" + key2
#print "key2b:{}".format(key2)
if isinstance(host.vars[key], basestring):
h[key2] = host.vars[key].encode('utf8')
else:
h[key2] = host.vars[key]
# ssh_private_key_file may be relative to source inventory file. Set it absolute
if key2 == SSH_PRIVATE_FILE_FILE:
p = h[key2]
if not os.path.isabs(p):
h[key2] = os.path.normpath(
os.path.join(os.path.dirname(inventory[FILE]),
p))
model[HOSTS].append(h)
existingHostGroups = set(map(lambda x: x[NAME], model[HOST_GROUPS]))
#print(existingHostGroups)
if inventoryHandler == PRE24:
hvars = variable_manager.get_vars(loader=loader,
host=hosts[0],
include_hostvars=False)
else:
hvars = variable_manager.get_vars(host=hosts[0],
include_hostvars=False)
groups = hvars[GROUPS]
for grpName in groups:
if grpName != 'all' and grpName != 'ungrouped' and (
not grpName in existingHostGroups):
grp = {}
grp[NAME] = grpName.encode('utf8')
l = []
for h in groups[grpName]:
l.append(h.encode('utf8'))
grp[HOSTS] = l
model[HOST_GROUPS].append(grp)