def build_inventory(loader, variable_manager, group_names, playbook_basedir):
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['localhost'])
# because we just pass in a "host list" which isn't a real inventory file,
# we explicitly have to add all of the desired groups to the inventory. By
# default an "all" group is created whenever a new inventory is created
for group_name in group_names:
if not inventory.get_group(group_name):
inventory.add_group(Group(group_name))
# because we are explicitly adding groups, we also need to make sure that a
# playbook basedir is set so that `group_vars` can be loaded from the
# correct directory.
inventory.set_playbook_basedir(playbook_basedir)
# for each group specified, ensure that the inventory's host (localhost) is
# explicitly in the group.
for group_name in group_names:
group = inventory.get_group(group_name)
if group.get_hosts():
continue
for host in inventory.get_hosts():
group.add_host(host)
return inventory
def build_inventory(loader, variable_manager, group_names, playbook_basedir):
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['localhost'])
# because we just pass in a "host list" which isn't a real inventory file,
# we explicitly have to add all of the desired groups to the inventory. By
# default an "all" group is created whenever a new inventory is created
for group_name in group_names:
if not inventory.get_group(group_name):
inventory.add_group(Group(group_name))
# because we are explicitly adding groups, we also need to make sure that a
# playbook basedir is set so that `group_vars` can be loaded from the
# correct directory.
inventory.set_playbook_basedir(playbook_basedir)
inventory_hosts = inventory.get_hosts()
# for each group specified, ensure that the inventory's host (localhost) is
# explicitly in the group.
for group_name in group_names:
group = inventory.get_group(group_name)
if group.get_hosts():
continue
for host in inventory.get_hosts():
group.add_host(host)
return inventory
class AnsibleInventory(object):
def __init__(self, inventory_generator):
"""
:type inventory_generator: ops.inventory.generator.InventoryGenerator
"""
self.inventory_generator = inventory_generator
self.generated_path, self.ssh_config_path = inventory_generator.generate(
)
# clean up variables cache for tests
ansible_vars.VARIABLE_CACHE = dict()
ansible_vars.HOSTVARS_CACHE = dict()
ansible_inventory.HOSTS_PATTERNS_CACHE = dict()
self.variable_manager = VariableManager()
loader = DataLoader()
self.inventory = Inventory(loader=loader,
variable_manager=self.variable_manager,
host_list=self.generated_path)
self.variable_manager.set_inventory(self.inventory)
self.inventory.set_playbook_basedir(self.generated_path)
def get_hosts(self, limit):
return self.inventory.get_hosts(limit)
def get_host(self, host):
return self.inventory.get_host(unicode(host))
def get_vars(self, host):
return self.inventory.get_vars(unicode(host))
def get_ssh_config(self):
return self.ssh_config_path
class AnsibleEnvironment():
_cache = {}
def __init__(self):
ansible_basedir = os.path.join(
os.environ.get("PROJECT_ENVIRONMENT_FILES_PATH"), "ansible")
loader = CustomLoader()
loader.set_basedir(ansible_basedir)
var_manager = VariableManager()
# load the inventory, set the basic playbook directory
self._inventory = Inventory(loader=loader,
variable_manager=var_manager)
self._inventory.set_playbook_basedir(ansible_basedir)
group = self._inventory.get_group("all")
# make sure we load all magic variables on top of the global variables
self._vars = combine_vars(
self._inventory.get_group_vars(group, return_results=True),
var_manager._get_magic_variables(loader, False, None, None, False,
False))
self._vars['groups'] = self._inventory.get_group_dict()
self._vars['env'] = os.environ
hostvars = {}
for host in self._inventory.get_hosts():
hostvars[host.name] = host.get_vars()
self._vars['hostvars'] = hostvars
# create the template renderer
self._templar = Templar(loader=loader, variables=self._vars)
# setup some easy variables that we use a lot
self._vars['control_ip'] = self.get_var(
"hostvars[groups['control'][0]]['ansible_host']")
self._vars['edge_ip'] = self.get_var(
"hostvars[groups['edge'][0]]['ansible_host']")
self._vars['monitor_ip'] = self.get_var(
"hostvars[groups['monitor'][0]]['ansible_host']")
def get_var(self, name, cache=True):
if name not in self._cache or not cache:
try:
self._cache[name] = self._templar.template("{{%s}}" % name)
except AnsibleUndefinedVariable:
self._cache[name] = None
return self._cache.get(name)
def set_var(self, name, value):
self._vars[name] = value
def template(self, *templates):
return '\n'.join([self._templar.template(tpl) for tpl in templates])
def playbook(**kwargs):
group_vars = kwargs['group_vars']
groups = kwargs['groups']
host_list = kwargs['host_list']
playbook_basedir = os.sep.join([ANSIBLE_PLAYBOOK_PATH, kwargs['playbook_basedir']])
playbooks = []
for pb in kwargs['playbooks']:
playbooks.append(os.sep.join([playbook_basedir, pb]))
job_id = kwargs['job_id']
loader = DataLoader()
vars = VariableManager()
# 指定inventory为一个目录，设置所有主机，包含group和host
invertory = Inventory(loader, vars,
host_list=host_list)
invertory.set_playbook_basedir(playbook_basedir)
for group_name, hosts in groups.items():
t_group = Group(group_name)
for host in hosts:
t_host = Host(host)
t_group.add_host(t_host)
invertory.add_group(t_group)
vars.set_inventory(invertory)
display = LogDisplay(logname=job_id)
callback = CALLBACKMODULE[CALLBACK](display=display)
Options = namedtuple('Options', ['connection', 'module_path', 'forks', 'timeout', 'remote_user',
'ask_pass', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become', 'become_method', 'become_user',
'ask_value_pass', 'verbosity', 'check', 'listhosts',
'listtags', 'listtasks', 'syntax'])
options = Options(connection='smart', module_path='/usr/share/ansible', forks=100, timeout=10,
remote_user='******', ask_pass=False, private_key_file=None, ssh_common_args=None,
ssh_extra_args=None, sftp_extra_args=None, scp_extra_args=None, become=None,
become_method=None, become_user='******', ask_value_pass=False, verbosity=None,
check=False, listhosts=None, listtags=None, listtasks=None, syntax=None)
passwords = dict()
pb_executor = PlaybookExecutor(playbooks, invertory, vars, loader, options, passwords)
pb_executor._tqm._stdout_callback = callback
pb_executor.run()
return display.get_log_json()
def get_play_prereqs_2(self, options):
loader = DataLoader()
if self.vault_pass:
loader.set_vault_password(self.vault_pass)
variable_manager = VariableManager()
variable_manager.extra_vars = self.extra_vars
variable_manager.options_vars = {
'ansible_version': self.version_info(ansible_version)
}
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=options.inventory)
variable_manager.set_inventory(inventory)
# let inventory know which playbooks are using so it can know the
# basedirs
inventory.set_playbook_basedir(os.path.dirname(self.playbook_file))
return loader, inventory, variable_manager
def playbook(**kwargs):
group_vars = kwargs['group_vars']
groups = kwargs['groups']
host_list = kwargs['host_list']
playbook_basedir = os.sep.join(
[ANSIBLE_PLAYBOOK_PATH, kwargs['playbook_basedir']])
playbooks = []
for pb in kwargs['playbooks']:
playbooks.append(os.sep.join([playbook_basedir, pb]))
job_id = kwargs['job_id']
loader = DataLoader()
vars = VariableManager()
# 指定inventory为一个目录，设置所有主机，包含group和host
invertory = Inventory(loader, vars, host_list=host_list)
invertory.set_playbook_basedir(playbook_basedir)
for group_name, hosts in groups.items():
t_group = Group(group_name)
for host in hosts:
t_host = Host(host)
t_group.add_host(t_host)
invertory.add_group(t_group)
vars.set_inventory(invertory)
display = LogDisplay(logname=job_id)
callback = CALLBACKMODULE[CALLBACK](display=display)
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'timeout', 'remote_user',
'ask_pass', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become', 'become_method',
'become_user', 'ask_value_pass', 'verbosity', 'check', 'listhosts',
'listtags', 'listtasks', 'syntax'
])
options = Options(connection='smart',
module_path='/usr/share/ansible',
forks=100,
timeout=10,
remote_user='******',
ask_pass=False,
private_key_file=None,
ssh_common_args=None,
ssh_extra_args=None,
sftp_extra_args=None,
scp_extra_args=None,
become=None,
become_method=None,
become_user='******',
ask_value_pass=False,
verbosity=None,
check=False,
listhosts=None,
listtags=None,
listtasks=None,
syntax=None)
passwords = dict()
pb_executor = PlaybookExecutor(playbooks, invertory, vars, loader, options,
passwords)
pb_executor._tqm._stdout_callback = callback
pb_executor.run()
return display.get_log_json()
def test_variable_manager_precedence(self, mock_basedir):
'''
Tests complex variations and combinations of get_vars() with different
objects to modify the context under which variables are merged.
'''
v = VariableManager()
v._fact_cache = defaultdict(dict)
fake_loader = DictDataLoader({
# inventory1
'/etc/ansible/inventory1': """
[group2:children]
group1
[group1]
host1 host_var=host_var_from_inventory_host1
[group1:vars]
group_var = group_var_from_inventory_group1
[group2:vars]
group_var = group_var_from_inventory_group2
""",
# role defaults_only1
'/etc/ansible/roles/defaults_only1/defaults/main.yml': """
default_var: "default_var_from_defaults_only1"
host_var: "host_var_from_defaults_only1"
group_var: "group_var_from_defaults_only1"
group_var_all: "group_var_all_from_defaults_only1"
extra_var: "extra_var_from_defaults_only1"
""",
'/etc/ansible/roles/defaults_only1/tasks/main.yml': """
- debug: msg="here i am"
""",
# role defaults_only2
'/etc/ansible/roles/defaults_only2/defaults/main.yml': """
default_var: "default_var_from_defaults_only2"
host_var: "host_var_from_defaults_only2"
group_var: "group_var_from_defaults_only2"
group_var_all: "group_var_all_from_defaults_only2"
extra_var: "extra_var_from_defaults_only2"
""",
})
mock_basedir.return_value = './'
inv1 = Inventory(loader=fake_loader, variable_manager=v, host_list='/etc/ansible/inventory1')
inv1.set_playbook_basedir('./')
play1 = Play.load(dict(
hosts=['all'],
roles=['defaults_only1', 'defaults_only2'],
), loader=fake_loader, variable_manager=v)
# first we assert that the defaults as viewed as a whole are the merged results
# of the defaults from each role, with the last role defined "winning" when
# there is a variable naming conflict
res = v.get_vars(loader=fake_loader, play=play1)
self.assertEqual(res['default_var'], 'default_var_from_defaults_only2')
# next, we assert that when vars are viewed from the context of a task within a
# role, that task will see its own role defaults before any other role's
blocks = play1.compile()
task = blocks[1].block[0]
res = v.get_vars(loader=fake_loader, play=play1, task=task)
self.assertEqual(res['default_var'], 'default_var_from_defaults_only1')
# next we assert the precendence of inventory variables
v.set_inventory(inv1)
h1 = inv1.get_host('host1')
res = v.get_vars(loader=fake_loader, play=play1, host=h1)
self.assertEqual(res['group_var'], 'group_var_from_inventory_group1')
self.assertEqual(res['host_var'], 'host_var_from_inventory_host1')
# next we test with group_vars/ files loaded
fake_loader.push("/etc/ansible/group_vars/all", """
group_var_all: group_var_all_from_group_vars_all
""")
fake_loader.push("/etc/ansible/group_vars/group1", """
group_var: group_var_from_group_vars_group1
""")
fake_loader.push("/etc/ansible/group_vars/group3", """
# this is a dummy, which should not be used anywhere
group_var: group_var_from_group_vars_group3
""")
fake_loader.push("/etc/ansible/host_vars/host1", """
host_var: host_var_from_host_vars_host1
""")
fake_loader.push("group_vars/group1", """
playbook_group_var: playbook_group_var
""")
fake_loader.push("host_vars/host1", """
playbook_host_var: playbook_host_var
""")
v.add_group_vars_file("/etc/ansible/group_vars/all", loader=fake_loader)
v.add_group_vars_file("/etc/ansible/group_vars/group1", loader=fake_loader)
v.add_group_vars_file("/etc/ansible/group_vars/group2", loader=fake_loader)
v.add_group_vars_file("group_vars/group1", loader=fake_loader)
v.add_host_vars_file("/etc/ansible/host_vars/host1", loader=fake_loader)
v.add_host_vars_file("host_vars/host1", loader=fake_loader)
res = v.get_vars(loader=fake_loader, play=play1, host=h1)
self.assertEqual(res['group_var'], 'group_var_from_group_vars_group1')
self.assertEqual(res['group_var_all'], 'group_var_all_from_group_vars_all')
self.assertEqual(res['playbook_group_var'], 'playbook_group_var')
self.assertEqual(res['host_var'], 'host_var_from_host_vars_host1')
self.assertEqual(res['playbook_host_var'], 'playbook_host_var')
# add in the fact cache
v._fact_cache['host1'] = dict(fact_cache_var="fact_cache_var_from_fact_cache")
res = v.get_vars(loader=fake_loader, play=play1, host=h1)
self.assertEqual(res['fact_cache_var'], 'fact_cache_var_from_fact_cache')
def test_variable_manager_precedence(self, mock_basedir):
'''
Tests complex variations and combinations of get_vars() with different
objects to modify the context under which variables are merged.
'''
v = VariableManager()
v._fact_cache = defaultdict(dict)
fake_loader = DictDataLoader({
# inventory1
'/etc/ansible/inventory1':
"""
[group2:children]
group1
[group1]
host1 host_var=host_var_from_inventory_host1
[group1:vars]
group_var = group_var_from_inventory_group1
[group2:vars]
group_var = group_var_from_inventory_group2
""",
# role defaults_only1
'/etc/ansible/roles/defaults_only1/defaults/main.yml':
"""
default_var: "default_var_from_defaults_only1"
host_var: "host_var_from_defaults_only1"
group_var: "group_var_from_defaults_only1"
group_var_all: "group_var_all_from_defaults_only1"
extra_var: "extra_var_from_defaults_only1"
""",
'/etc/ansible/roles/defaults_only1/tasks/main.yml':
"""
- debug: msg="here i am"
""",
# role defaults_only2
'/etc/ansible/roles/defaults_only2/defaults/main.yml':
"""
default_var: "default_var_from_defaults_only2"
host_var: "host_var_from_defaults_only2"
group_var: "group_var_from_defaults_only2"
group_var_all: "group_var_all_from_defaults_only2"
extra_var: "extra_var_from_defaults_only2"
""",
})
mock_basedir.return_value = './'
inv1 = Inventory(loader=fake_loader,
variable_manager=v,
host_list='/etc/ansible/inventory1')
inv1.set_playbook_basedir('./')
play1 = Play.load(dict(
hosts=['all'],
roles=['defaults_only1', 'defaults_only2'],
),
loader=fake_loader,
variable_manager=v)
# first we assert that the defaults as viewed as a whole are the merged results
# of the defaults from each role, with the last role defined "winning" when
# there is a variable naming conflict
res = v.get_vars(loader=fake_loader, play=play1)
self.assertEqual(res['default_var'], 'default_var_from_defaults_only2')
# next, we assert that when vars are viewed from the context of a task within a
# role, that task will see its own role defaults before any other role's
blocks = play1.compile()
task = blocks[1].block[0]
res = v.get_vars(loader=fake_loader, play=play1, task=task)
self.assertEqual(res['default_var'], 'default_var_from_defaults_only1')
# next we assert the precendence of inventory variables
v.set_inventory(inv1)
h1 = inv1.get_host('host1')
res = v.get_vars(loader=fake_loader, play=play1, host=h1)
self.assertEqual(res['group_var'], 'group_var_from_inventory_group1')
self.assertEqual(res['host_var'], 'host_var_from_inventory_host1')
# next we test with group_vars/ files loaded
fake_loader.push(
"/etc/ansible/group_vars/all", """
group_var_all: group_var_all_from_group_vars_all
""")
fake_loader.push(
"/etc/ansible/group_vars/group1", """
group_var: group_var_from_group_vars_group1
""")
fake_loader.push(
"/etc/ansible/group_vars/group3", """
# this is a dummy, which should not be used anywhere
group_var: group_var_from_group_vars_group3
""")
fake_loader.push(
"/etc/ansible/host_vars/host1", """
host_var: host_var_from_host_vars_host1
""")
v.add_group_vars_file("/etc/ansible/group_vars/all",
loader=fake_loader)
v.add_group_vars_file("/etc/ansible/group_vars/group1",
loader=fake_loader)
v.add_group_vars_file("/etc/ansible/group_vars/group2",
loader=fake_loader)
v.add_host_vars_file("/etc/ansible/host_vars/host1",
loader=fake_loader)
res = v.get_vars(loader=fake_loader, play=play1, host=h1)
self.assertEqual(res['group_var'], 'group_var_from_group_vars_group1')
self.assertEqual(res['group_var_all'],
'group_var_all_from_group_vars_all')
self.assertEqual(res['host_var'], 'host_var_from_host_vars_host1')
# add in the fact cache
v._fact_cache['host1'] = dict(
fact_cache_var="fact_cache_var_from_fact_cache")
res = v.get_vars(loader=fake_loader, play=play1, host=h1)
self.assertEqual(res['fact_cache_var'],
'fact_cache_var_from_fact_cache')
def run(cls, playbook_config, username, password, hostname, port):
"""
:param list host_list:
:param playbook_config:
:return: :raise errors.AnsibleError:
"""
from jetee.runtime.app import dispatcher
cls.set_defaults()
inventory = Inventory([u'%s:%s' % (hostname, port)])
# let inventory know which playbooks are using so it can know the basedirs
inventory.set_playbook_basedir(os.path.dirname(playbook_config.filename))
stats = AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=dispatcher.args.verbosity)
runner_cb = PlaybookRunnerCallbacks(stats, verbose=dispatcher.args.verbosity)
pb = PlayBook(
playbook=playbook_config.filename,
module_path=None,
inventory=inventory,
forks=5,
remote_user=username,
remote_pass=password,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
timeout=10,
transport=u'smart',
sudo=False,
sudo_user=u'root',
sudo_pass=None,
extra_vars={},
private_key_file=None,
only_tags=[u'all'],
skip_tags=None,
check=False,
diff=False,
su=False,
su_pass=None,
su_user=u'root',
vault_password=None,
force_handlers=None
)
failed_hosts = []
unreachable_hosts = []
try:
pb.run()
hosts = sorted(pb.stats.processed.keys())
playbook_cb.on_stats(pb.stats)
for h in hosts:
t = pb.stats.summarize(h)
display("%s : %s %s %s %s" % (
cls.hostcolor(h, t),
cls.colorize('ok', t['ok'], 'green'),
cls.colorize('changed', t['changed'], 'yellow'),
cls.colorize('unreachable', t['unreachable'], 'red'),
cls.colorize('failed', t['failures'], 'red')),
screen_only=True
)
display("%s : %s %s %s %s" % (
cls.hostcolor(h, t, False),
cls.colorize('ok', t['ok'], None),
cls.colorize('changed', t['changed'], None),
cls.colorize('unreachable', t['unreachable'], None),
cls.colorize('failed', t['failures'], None)),
log_only=True
)
if len(failed_hosts) > 0:
return 2
if len(unreachable_hosts) > 0:
return 3
except AnsibleError as e:
display("ERROR: %s" % e, color='red')
return 1