def get_ansible_tasks(self):
# Load playbooks
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources=[self.args.host])
variable_manager = VariableManager(loader=loader, inventory=inventory)
pb = Playbook.load(self.args.playbook,
variable_manager=variable_manager,
loader=loader)
plays = pb.get_plays()
all_vars = {}
all_vars.update(variable_manager.get_vars(host=Host(self.args.host)))
# enumarate all available plays
tasks = set()
for play in plays:
all_vars.update(variable_manager.get_vars(play=play))
templar = Templar(loader=loader, variables=all_vars)
for play in plays:
new_play = play.copy()
new_play.post_validate(templar)
for role in new_play.get_roles():
u = Unwrap()
block_container = role.compile(new_play)
u.unwrap_blocks(block_container)
tasks.update(u.tasks)
return templar, tasks
def inventory_hosts_by_group(self):
"""
filename is a path to an ansible inventory file
returns a mapping of group names ("webworker", "proxy", etc.)
to lists of hostnames as listed in the inventory file.
("Hostnames" can also be IP addresses.)
If the hostname in the file includes :<port>, that will be included here as well.
"""
inventory = self.inventory_manager
var_manager = VariableManager(self._ansible_inventory_data_loader,
inventory)
# use the ip address specified by ansible_host to ssh in if it's given
ssh_addr_map = {
host.name:
var_manager.get_vars(host=host).get('ansible_host', host.name)
for host in inventory.get_hosts()
}
# use the port specified by ansible_port to ssh in if it's given
port_map = {
host.name: var_manager.get_vars(host=host).get('ansible_port')
for host in inventory.get_hosts()
}
return {
group: [
'{}:{}'.format(ssh_addr_map[host], port_map[host])
if port_map[host] is not None else ssh_addr_map[host]
for host in hosts
]
for group, hosts in
self.inventory_manager.get_groups_dict().items()
}
def test_variable_manager_role_vars_dependencies(self):
'''
Tests vars from role dependencies with duplicate dependencies.
'''
mock_inventory = MagicMock()
fake_loader = DictDataLoader({
# role common-role
'/etc/ansible/roles/common-role/tasks/main.yml':
"""
- debug: msg="{{role_var}}"
""",
# We do not need allow_duplicates: yes for this role
# because eliminating duplicates is done by the execution
# strategy, which we do not test here.
# role role1
'/etc/ansible/roles/role1/vars/main.yml':
"""
role_var: "role_var_from_role1"
""",
'/etc/ansible/roles/role1/meta/main.yml':
"""
dependencies:
- { role: common-role }
""",
# role role2
'/etc/ansible/roles/role2/vars/main.yml':
"""
role_var: "role_var_from_role2"
""",
'/etc/ansible/roles/role2/meta/main.yml':
"""
dependencies:
- { role: common-role }
""",
})
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
v._fact_cache = defaultdict(dict)
play1 = Play.load(dict(
hosts=['all'],
roles=['role1', 'role2'],
),
loader=fake_loader,
variable_manager=v)
# The task defined by common-role exists twice because role1
# and role2 depend on common-role. Check that the tasks see
# different values of role_var.
blocks = play1.compile()
task = blocks[1].block[0]
res = v.get_vars(play=play1, task=task)
self.assertEqual(res['role_var'], 'role_var_from_role1')
task = blocks[2].block[0]
res = v.get_vars(play=play1, task=task)
self.assertEqual(res['role_var'], 'role_var_from_role2')
class AnsibleInventory(object):
virt_component = 'virtual_component'
def __init__(self, inventory: str = None, extra_vars: dict = None):
self._logger = logging.getLogger(self.__class__.__module__)
self.inventory = inventory
self.loader = DataLoader()
self._logger.info('Loading inventory: %s' % inventory)
self._logger.debug('Extra variables: %s' % extra_vars)
self.inv_mgr = InventoryManager(loader=self.loader,
sources=self.inventory)
self.var_mgr = VariableManager(loader=self.loader,
inventory=self.inv_mgr)
self.var_mgr._extra_vars = extra_vars or dict()
def get_hosts_containing(self, var: str = None) -> list:
hosts = []
for host in self.inv_mgr.get_hosts():
# If no specific var provided, then add it to the list
if not var:
hosts.append(host)
continue
# If var is provided and not part of host vars, ignore it
host_vars = self.var_mgr.get_vars(host=host)
if var not in host_vars:
continue
# Var has been found so adding it
hosts.append(host)
return hosts
def get_host_vars(self, host: Host):
data = self.var_mgr.get_vars(host=host)
templar = Templar(variables=data, loader=self.loader)
return templar.template(data, fail_on_undefined=False)
def get_virtual_components(self) -> list:
virtual_components = []
for group in self.inv_mgr.get_groups_dict():
group_vars = self.inv_mgr.groups[group].get_vars()
if self.virt_component in group_vars:
vcmp_type = group_vars.get(self.virt_component)
group_vars.pop('virtual_component')
vcmp_impl = group_vars.get('implementation')
group_vars.pop('implementation')
hosts = self.inv_mgr.groups[group].get_hosts()
vcmp = AnsibleVirtualComponent(name=group,
type=vcmp_type,
implementation=vcmp_impl,
members=hosts,
**group_vars)
virtual_components.append(vcmp)
return virtual_components
def test_variable_manager_role_vars_dependencies(self):
'''
Tests vars from role dependencies with duplicate dependencies.
'''
mock_inventory = MagicMock()
fake_loader = DictDataLoader({
# role common-role
'/etc/ansible/roles/common-role/tasks/main.yml': """
- debug: msg="{{role_var}}"
""",
# We do not need allow_duplicates: yes for this role
# because eliminating duplicates is done by the execution
# strategy, which we do not test here.
# role role1
'/etc/ansible/roles/role1/vars/main.yml': """
role_var: "role_var_from_role1"
""",
'/etc/ansible/roles/role1/meta/main.yml': """
dependencies:
- { role: common-role }
""",
# role role2
'/etc/ansible/roles/role2/vars/main.yml': """
role_var: "role_var_from_role2"
""",
'/etc/ansible/roles/role2/meta/main.yml': """
dependencies:
- { role: common-role }
""",
})
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
v._fact_cache = defaultdict(dict)
play1 = Play.load(dict(
hosts=['all'],
roles=['role1', 'role2'],
), loader=fake_loader, variable_manager=v)
# The task defined by common-role exists twice because role1
# and role2 depend on common-role. Check that the tasks see
# different values of role_var.
blocks = play1.compile()
task = blocks[1].block[0]
res = v.get_vars(play=play1, task=task)
self.assertEqual(res['role_var'], 'role_var_from_role1')
task = blocks[2].block[0]
res = v.get_vars(play=play1, task=task)
self.assertEqual(res['role_var'], 'role_var_from_role2')
def playbookrun(self, playbook_path):
playbook_inventory = InventoryManager(loader=self.loader,
sources=['playbook_hosts'])
playbook_variable = VariableManager(loader=self.loader,
inventory=playbook_inventory)
playbook_inventory.add_host(host="192.168.122.102",
port=22,
group='all')
playbook_inventory.add_group('test_group')
playbook_inventory.add_host(host="192.168.122.103",
port=22,
group="test_group")
print(playbook_inventory.get_groups_dict())
host = playbook_inventory.get_host(hostname="192.168.122.102")
host.set_variable("ansible_ssh_pass", "/*bankentan123")
host.set_variable("ansible_ssh_user", "root")
host.set_variable("ansible_ssh_port", "22")
host.set_variable("ansible_ssh_host", "192.168.122.102")
host.set_variable("myname", "bankentan")
vars = playbook_variable.get_vars(host=host)
print(vars)
context._init_global_context(self.ops)
playbook = PlaybookExecutor(playbooks=playbook_path,
inventory=playbook_inventory,
variable_manager=self.variable_manager,
loader=self.loader,
passwords=self.passwords)
result = playbook.run()
return result
def get_configured_replicasets():
inventory = InventoryManager(loader=DataLoader(), sources='hosts.yml')
variable_manager = VariableManager(loader=DataLoader(),
inventory=inventory)
replicasets = {}
for instance in inventory.hosts:
host_vars = variable_manager.get_vars(host=inventory.hosts[instance])
if 'replicaset_alias' not in host_vars:
continue
if instance_is_expelled(host_vars):
continue
replicaset_alias = host_vars['replicaset_alias']
if replicaset_alias not in replicasets:
replicasets[replicaset_alias] = {
'instances': [],
'failover_priority': host_vars.get('failover_priority'),
'roles': host_vars['roles'],
'all_rw': host_vars.get('all_rw'),
'weight': host_vars.get('weight'),
'vshard_group': host_vars.get('vshard_group')
}
replicasets[replicaset_alias]['instances'].append(instance)
return replicasets
def test_variable_manager_task_vars(self):
# FIXME: BCS make this work
return
# pylint: disable=unreachable
fake_loader = DictDataLoader({})
mock_task = MagicMock()
mock_task._role = None
mock_task.loop = None
mock_task.get_vars.return_value = dict(foo="bar")
mock_task.get_include_params.return_value = dict()
mock_all = MagicMock()
mock_all.get_vars.return_value = {}
mock_all.get_file_vars.return_value = {}
mock_host = MagicMock()
mock_host.get.name.return_value = 'test01'
mock_host.get_vars.return_value = {}
mock_host.get_host_vars.return_value = {}
mock_inventory = MagicMock()
mock_inventory.hosts.get.return_value = mock_host
mock_inventory.hosts.get.name.return_value = 'test01'
mock_inventory.get_host.return_value = mock_host
mock_inventory.groups.__getitem__.return_value = mock_all
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
self.assertEqual(
v.get_vars(task=mock_task, use_cache=False).get("foo"), "bar")
def test_variable_manager_task_vars(self):
# FIXME: BCS make this work
return
# pylint: disable=unreachable
fake_loader = DictDataLoader({})
mock_task = MagicMock()
mock_task._role = None
mock_task.loop = None
mock_task.get_vars.return_value = dict(foo="bar")
mock_task.get_include_params.return_value = dict()
mock_all = MagicMock()
mock_all.get_vars.return_value = {}
mock_all.get_file_vars.return_value = {}
mock_host = MagicMock()
mock_host.get.name.return_value = 'test01'
mock_host.get_vars.return_value = {}
mock_host.get_host_vars.return_value = {}
mock_inventory = MagicMock()
mock_inventory.hosts.get.return_value = mock_host
mock_inventory.hosts.get.name.return_value = 'test01'
mock_inventory.get_host.return_value = mock_host
mock_inventory.groups.__getitem__.return_value = mock_all
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
self.assertEqual(v.get_vars(task=mock_task, use_cache=False).get("foo"), "bar")
def test():
try:
loader = DataLoader()
ds = loader.load(get_values())
inventory = InventoryManager(loader=loader)
if ds is not None:
for key, value in ds.items():
inventory.groups['all'].set_variable(key, value)
variable_manager = VariableManager(
loader=loader,
inventory=inventory,
version_info=CLI.version_info(gitinfo=False))
templar = Templar(loader=loader)
templar.available_variables = variable_manager.get_vars(host=Host(
name='all'))
try:
rendered = templar.template(get_template(),
convert_data=False,
cache=False)
except Exception as e:
rendered = "Template rendering failed: {0}".format(e)
except Exception as e:
rendered = "Template syntax error: {0}".format(e)
result = {"result": str(rendered)}
return json.dumps(result)
def ansible_syntax_check():
# templ = """{{ lookup(
# 'template', '%s/ansible/roles/sigia/templates/ansible_extra_vars.j2')
# }}""" % ( config.rootdir )
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources='local,')
variable_manager = VariableManager(loader=loader, inventory=inventory)
pb = Playbook.load(config.rootdir +
"/infrastructure/ansible/aws-security.yml",
variable_manager=variable_manager,
loader=loader)
plays = pb.get_plays()
localhost = inventory.get_host("local")
for play in plays:
variable_manager.set_host_facts(localhost,
variable_manager.get_vars(play))
# local_vars = variable_manager.get_vars(host=localhost)
# templar = Templar(loader=loader, variables=local_vars)
# extra_vars = templar.template(templ)
# if extra_vars:
# variable_manager.extra_vars = extra_vars
r = ansible_runner.run(
private_data_dir=TEST_DIR,
inventory='local,',
playbook=str(config.rootdir) +
'/infrastructure/ansible/aws-security.yml',
# extravars=extra_vars,
artifact_dir="/tmp/ansible",
rotate_artifacts=5)
return r
def get(self):
if os.path.isdir(self.cluster_config_path):
return {"cluster": None, "inventory": None}
data_loader = DataLoader()
# data_loader.set_vault_password('627VR8*;YU99B')
variable_manager = VariableManager(loader=data_loader)
extra_vars = self.console_args.extra_vars[:]
configurations = [
'@' + config for config in self.cluster_config_path.split(',')
]
extra_vars.append('cluster=' +
get_cluster_name(self.cluster_config_path))
extra_vars.extend(configurations)
options = collections.namedtuple('options', 'extra_vars')
variable_manager.extra_vars = load_extra_vars(
loader=data_loader, options=options(extra_vars=extra_vars))
read_variables = variable_manager.get_vars()
templar = Templar(data_loader, variables=read_variables)
templar._filter_loader = self.template.filter_plugin_loader
return templar.template(read_variables, fail_on_undefined=True)
def _convert_to_dynamic_inventory(aelf, ansible_static_inventory):
variable_manager = VariableManager(loader=DataLoader(),
inventory=ansible_static_inventory)
ansible_dynamic_inventory = dict()
for group in ansible_static_inventory.groups.values():
ansible_dynamic_inventory[group.name] = dict()
group_hosts = group.get_hosts()
if len(group_hosts):
ansible_dynamic_inventory[group.name]["hosts"] = map(
str, group_hosts)
group_vars = group.get_vars()
if len(group_vars):
ansible_dynamic_inventory[group.name]["vars"] = group_vars
group_children = group.child_groups
if len(group_children):
ansible_dynamic_inventory[group.name]["children"] = map(
str, group_children)
ansible_dynamic_inventory["_meta"] = dict()
ansible_dynamic_inventory["_meta"]["hostvars"] = dict()
for host in ansible_static_inventory.get_hosts():
ansible_dynamic_inventory["_meta"]["hostvars"][
host.name] = variable_manager.get_vars(host=host)
del (ansible_dynamic_inventory['_meta']['hostvars'][host.name]
['groups'])
del (ansible_dynamic_inventory['_meta']['hostvars'][host.name]
['inventory_dir'])
del (ansible_dynamic_inventory['_meta']['hostvars'][host.name]
['inventory_file'])
del (ansible_dynamic_inventory['_meta']['hostvars'][host.name]
['omit'])
return ansible_dynamic_inventory
class Inventory24(Inventory):
def __init__(self, inventory, ask_vault_pass, vault_password_files, vault_ids):
from ansible.cli import CLI
super(Inventory24, self).__init__()
loader = DataLoader()
if vault_ids or vault_password_files or ask_vault_pass:
CLI.setup_vault_secrets(loader, vault_ids, vault_password_files, ask_vault_pass)
self.inventory = ansible.inventory.manager.InventoryManager(loader=loader, sources=inventory)
self.variable_manager = VariableManager(loader=loader)
self.variable_manager.set_inventory(self.inventory)
# internal fuctions that actually do the work
# adapted almost entirely from lib/ansible/vars/manager.py
def _plugins_inventory(self, entities):
import os
from ansible.plugins.loader import vars_loader
from ansible.utils.vars import combine_vars
''' merges all entities by inventory source '''
data = {}
for inventory_dir in self.variable_manager._inventory._sources:
if ',' in inventory_dir: # skip host lists
continue
elif not os.path.isdir(inventory_dir): # always pass 'inventory directory'
inventory_dir = os.path.dirname(inventory_dir)
for plugin in vars_loader.all():
data = combine_vars(data, self._get_plugin_vars(plugin, inventory_dir, entities))
return data
def _get_plugin_vars(self, plugin, path, entities):
from ansible.inventory.host import Host
data = {}
try:
data = plugin.get_vars(self.variable_manager._loader, path, entities)
except AttributeError:
for entity in entities:
if isinstance(entity, Host):
data.update(plugin.get_host_vars(entity.name))
else:
data.update(plugin.get_group_vars(entity.name))
return data
def get_group_vars(self, group):
return self._plugins_inventory([group])
def get_host_vars(self, host):
try:
all_vars = self.variable_manager.get_vars(host=host, include_hostvars=True)
except ansible.errors.AnsibleParserError:
raise NoVaultSecretFound
# play, host, task, include_hostvars, include_delegate_to
magic_vars = ['ansible_playbook_python', 'groups', 'group_names', 'inventory_dir',
'inventory_file', 'inventory_hostname', 'inventory_hostname_short',
'omit', 'playbook_dir']
return {k: v for (k, v) in all_vars.items() if k not in magic_vars}
def get_group(self, group_name):
return self.inventory.groups[group_name]
def get_ansible_host_ip():
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources='hosts')
variable_manager = VariableManager(loader=loader, inventory=inventory)
hostnames = []
for host in inventory.get_hosts():
hostnames.append(variable_manager.get_vars(host=host))
ip = ' '.join([str(i['ansible_host']) for i in hostnames])
return str(ip)
def get_ansible_host_ip():
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources='hosts')
variable_manager = VariableManager(loader=loader, inventory=inventory)
hostnames = []
for host in inventory.get_hosts():
hostnames.append(variable_manager.get_vars(host=host))
ip = ' '.join([str(i['ansible_host']) for i in hostnames])
return str(ip)
def __init__(self):
initial_dir = os.getcwd()
ansible_basedir = os.path.join(
os.environ.get("PROJECT_ENVIRONMENT_FILES_PATH"), "ansible")
# Move to project directory
os.chdir(os.environ.get("PROJECT_ENVIRONMENT_FILES_PATH"))
# Load list of inventories from config:w
config = ConfigManager('/etc/ansible/ansible.cfg')
sources = config.data.get_setting('DEFAULT_HOST_LIST').value
loader = CustomLoader()
loader.set_basedir(ansible_basedir)
# load the inventory, set the basic playbook directory
self._inventory = CustomInventoryManager(loader=loader,
sources=sources)
var_manager = VariableManager(loader=loader, inventory=self._inventory)
play = Play.load(dict(hosts=['all']),
loader=loader,
variable_manager=var_manager)
# Move back to directory of origin
os.chdir(initial_dir)
control_host = None
if 'control' in self._inventory.groups:
control_group = self._inventory.groups['control']
if len(control_group.get_hosts()) > 0:
control_host = control_group.get_hosts()[0]
# Hostvars
hostvars = {}
for host in self._inventory.get_hosts():
hostvars[host.name] = host.vars
# make sure we load all magic variables on top of the global variables
self._vars = combine_vars(
var_manager.get_vars(play=play, task=Task(), host=control_host), {
'hostvars': hostvars,
'env': os.environ
})
# create the template renderer
self._templar = Templar(loader=loader, variables=self._vars)
# setup some easy variables that we use a lot
self._vars['control_ip'] = self.get_var(
"hostvars[groups['control'][0]]['ansible_host']")
self._vars['edge_ip'] = self.get_var(
"hostvars[groups['edge'][0]]['ansible_host']")
self._vars['monitor_ip'] = self.get_var(
"hostvars[groups['monitor'][0]]['ansible_host']")
def test_variable_manager_play_vars(self):
fake_loader = DictDataLoader({})
mock_play = MagicMock()
mock_play.get_vars.return_value = dict(foo="bar")
mock_play.get_roles.return_value = []
mock_play.get_vars_files.return_value = []
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
self.assertEqual(v.get_vars(play=mock_play, use_cache=False).get("foo"), "bar")
def test_basic_manager(self):
fake_loader = DictDataLoader({})
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
variables = v.get_vars(use_cache=False)
# Check var manager expected values, never check: ['omit', 'vars']
# FIXME: add the following ['ansible_version', 'ansible_playbook_python', 'groups']
for varname, value in (('playbook_dir', os.path.abspath('.')), ):
self.assertEqual(variables[varname], value)
def test_variable_manager_play_vars(self):
fake_loader = DictDataLoader({})
mock_play = MagicMock()
mock_play.get_vars.return_value = dict(foo="bar")
mock_play.get_roles.return_value = []
mock_play.get_vars_files.return_value = []
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
self.assertEqual(v.get_vars(play=mock_play, use_cache=False).get("foo"), "bar")
def test_basic_manager(self):
fake_loader = DictDataLoader({})
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
variables = v.get_vars(use_cache=False)
# Check var manager expected values, never check: ['omit', 'vars']
# FIXME: add the following ['ansible_version', 'ansible_playbook_python', 'groups']
for varname, value in (('playbook_dir', os.path.abspath('.')), ):
self.assertEqual(variables[varname], value)
def test_basic_manager(self):
fake_loader = DictDataLoader({})
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
vars = v.get_vars(use_cache=False)
# FIXME: not sure why we remove all and only test playbook_dir
for remove in ['omit', 'vars', 'ansible_version', 'ansible_check_mode', 'ansible_playbook_python']:
if remove in vars:
del vars[remove]
self.assertEqual(vars, dict(playbook_dir=os.path.abspath('.')))
def test_variable_manager_options_vars(self):
fake_loader = DictDataLoader({})
options_vars = dict(a=1, b=2, c=3)
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
# override internal options_vars loading
v._extra_vars = options_vars
myvars = v.get_vars(use_cache=False)
for (key, val) in iteritems(options_vars):
self.assertEqual(myvars.get(key), val)
def list_inventory(ansible_inventory='/etc/ansible/hosts', ansible_groups=None):
from ansible.parsing.dataloader import DataLoader
from ansible.inventory.manager import InventoryManager
from ansible.vars.manager import VariableManager
data_loader = DataLoader()
inventory = InventoryManager(loader=data_loader, sources=ansible_inventory)
variable_manager = VariableManager(loader=data_loader, inventory=inventory)
for ag in ansible_groups:
for h in inventory.groups[ag].get_hosts():
rhost = (h.get_vars()['ansible_host'] if 'ansible_host' in h.get_vars() else h.get_vars()['inventory_hostname'])
ruser = (variable_manager.get_vars(host=h)['ansible_user'] if 'ansible_user' in variable_manager.get_vars(host=h) else 'root')
yield (rhost, ruser)
def test_variable_manager_extra_vars(self):
fake_loader = DictDataLoader({})
extra_vars = dict(a=1, b=2, c=3)
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
v.extra_vars = extra_vars
vars = v.get_vars(use_cache=False)
for (key, val) in iteritems(extra_vars):
self.assertEqual(vars.get(key), val)
self.assertIsNot(v.extra_vars, extra_vars)
def test_variable_manager_extra_vars(self):
fake_loader = DictDataLoader({})
extra_vars = dict(a=1, b=2, c=3)
mock_inventory = MagicMock()
v = VariableManager(loader=fake_loader, inventory=mock_inventory)
v.extra_vars = extra_vars
vars = v.get_vars(use_cache=False)
for (key, val) in iteritems(extra_vars):
self.assertEqual(vars.get(key), val)
self.assertIsNot(v.extra_vars, extra_vars)
def parse_inventory(inventory):
loader = DataLoader()
inv = InventoryManager(loader=loader, sources=[inventory])
vars = VariableManager(loader=loader, inventory=inv)
all_groups = inv.get_groups_dict()
tidb_nodes = all_groups['tidb_servers']
tikv_nodes = all_groups['tikv_servers']
tidb_servers = {}
tikv_servers = {}
for tidb in tidb_nodes:
var = vars.get_vars(host=inv.get_host(hostname=str(tidb)))
ip = var['ansible_host'] if 'ansible_host' in var else var[
'inventory_hostname']
tidb_port = var.get('tidb_port', 4000)
tidb_status_port = var.get('tidb_status_port', 10080)
deploy_dir = var['deploy_dir']
if ip in tidb_servers:
tidb_servers[ip].append([tidb_port, tidb_status_port, deploy_dir])
else:
tidb_servers[ip] = [[tidb_port, tidb_status_port, deploy_dir]]
for tikv in tikv_nodes:
var = vars.get_vars(host=inv.get_host(hostname=str(tikv)))
ip = var['ansible_host'] if 'ansible_host' in var else var[
'inventory_hostname']
tikv_port = var.get('tikv_port', 20160)
tikv_status_port = var.get('tikv_status_port', 20180)
deploy_dir = var['deploy_dir']
if ip in tikv_servers:
tikv_servers[ip].append([tikv_port, tikv_status_port, deploy_dir])
else:
tikv_servers[ip] = [[tikv_port, tikv_status_port, deploy_dir]]
return [tidb_servers, tikv_servers]
def execute(playbooks: list, context: dict):
'''
'''
assert playbooks
os.environ["ANSIBLE_CONDITIONAL_BARE_VARS"] = "False"
# ctx._init_global_context({})
# since the API is constructed for CLI it expects certain options to always be set in the context object
ctx.CLIARGS = ImmutableDict(connection='local',
module_path=['./playbooks/roles'],
forks=1,
become=False,
become_method="sudo",
syntax=False,
start_at_task=None,
diff=False,
verbosity=0)
# initialize needed objects
loader = DataLoader(
) # Takes care of finding and reading yaml, json and ini files
# passwords = dict(become_pass="******")
passwords = dict()
# Instantiate our ResultCallback for handling results as they come in. Ansible expects this to be one of its main display outlets
results_callback = ResultCallback()
# create inventory, use path to host config file as source or hosts in a comma separated string
inventory = InventoryManager(loader=loader, sources='localhost,')
host = Host(name="localhost")
# variable manager takes care of merging all the different sources to give you a unified view of variables available in each context
variable_manager = VariableManager(loader=loader, inventory=inventory)
variable_manager.set_host_variable(host, "current_directory",
context.get("current_directory", ""))
pbex = PlaybookExecutor(playbooks=playbooks,
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords)
result = pbex.run()
post_command = variable_manager.get_vars()["hostvars"]["localhost"].get(
"mondrik_post_command")
return result, post_command
def inventory_load(inventory_sources, vault_secret):
""" Load the inventory
"""
loader = DataLoader()
vault_secrets = [('default', TextVaultSecret(vault_secret))]
loader.set_vault_secrets(vault_secrets)
inventory = InventoryManager(loader=loader, sources=inventory_sources)
result = {}
for hostname in inventory.hosts:
host = inventory.get_host(hostname)
variable_manager = VariableManager(loader=loader, inventory=inventory)
magic_vars = ['ansible_playbook_python', 'groups', 'group_names', 'inventory_dir',
'inventory_file', 'inventory_hostname', 'inventory_hostname_short',
'omit', 'playbook_dir']
all_vars = variable_manager.get_vars(host=host, include_hostvars=True)
cleaned = ({k: v for (k, v) in all_vars.items() if k not in magic_vars})
result[hostname] = cleaned
return result
def get(self):
data_loader = DataLoader()
# data_loader.set_vault_password()
variable_manager = VariableManager(loader=data_loader)
extra_vars = self.console_args.extra_vars[:]
extra_vars.append('cluster=' +
get_cluster_name(self.cluster_config_path))
options = collections.namedtuple('options', 'extra_vars')
variable_manager.extra_vars = load_extra_vars(
loader=data_loader, options=options(extra_vars=extra_vars))
variables = variable_manager.get_vars()
rendered = self.template.render(self.cluster_config_path, variables)
return yaml.safe_load(rendered)
def play_vars(ctx):
# create inventory, use path to host config file as source or hosts in a comma
# separated string
inventory = InventoryManager(loader=loader, sources="inventory")
# variable manager takes care of merging all the different sources to give you
# a unified view of variables available in each context
variable_manager = VariableManager(loader=loader, inventory=inventory)
# create data structure that represents our play, including tasks, this is basically
# what our YAML loader does internally.
play_source = dict(name="Ansible Play",
hosts="all",
gather_facts="no",
tasks=[])
loader.set_vault_secrets([("default", VaultSecret(_bytes=to_bytes("TBD")))
])
# Create play object, playbook objects use .load instead of init or new methods,
# this will also automatically create the task objects from the info provided
# in play_source
play = Play().load(play_source,
variable_manager=variable_manager,
loader=loader)
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=dict(),
# Use our custom callback instead of the ``default`` callback
# plugin, which prints to stdout
stdout_callback=results_callback,
)
# most interesting data for a play is actually
# sent to the callback's methods
tqm.run(play)
finally:
# we always need to cleanup child procs and the structures we use to
# communicate with them
if tqm is not None:
tqm.cleanup()
# Remove ansible tmpdir
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
ctx.hostvars = variable_manager.get_vars()["hostvars"]
def CallBackPlaybook(playbooks_path,extra_vars={},source=settings.ANSIBLE_SOURCE):
"""
执行playbook
:param playbooks_path: playbook的路径 list的类型
:param source: ansible配置文件hosts
:return:
"""
dataloader = DataLoader()
inventory = InventoryManager(loader=dataloader, sources=source)
variableManager = VariableManager(loader=dataloader, inventory=inventory)
variableManager.extra_vars=extra_vars #传入参数
Options = namedtuple("Options", [
"connection", "remote_user", "ask_sudo_pass", "verbosity", "ack_pass",
"module_path", "forks", "become", "become_method", "become_user", "check",
"listhosts", "listtasks", "listtags", "syntax", "sudo_user", "sudo", "diff"
])
options = Options(connection='ssh', remote_user=None, ack_pass=None, sudo_user=None, forks=settings.ANSIBLE_FORKS, sudo=None,
ask_sudo_pass=False,
verbosity=5, module_path=None, become=None, become_method=None, become_user=None, check=False,
diff=False,
listhosts=None, listtasks=None, listtags=None, syntax=None)
passwords = dict()
try:
logger = logging.getLogger("django")
playbook = PlaybookExecutor(playbooks=playbooks_path, inventory=inventory, variable_manager=variableManager, loader=dataloader, options=options, passwords=passwords)
callback = PlaybookCallbackBase()
playbook._tqm._stdout_callback = callback # 配置callback
playbook.run()
logger_info={"playbooks":playbooks_path,"variable_manager":variableManager.get_vars(),"passwords":passwords}
logger.info(logger_info)
status_list=['ok','failed','unreachable','skipped','stats']
result_raw = {"ok": {}, "failed": {}, "unreachable": {}, "skipped": {}, "stats": {}}
for status in status_list :
for host,result in getattr(callback,'task_'+status).items() :
result_raw[status][host] = result._result
result_raw[status][host]["task"]=result._task.get_name()
logger.info(str(result_raw))
result_raw=get_result(result_raw)
return result_raw
except Exception as e:
print(e)
traceback.print_exc()
def read_inventory_file(filename):
"""
filename is a path to an ansible inventory file
returns a mapping of group names ("webworker", "proxy", etc.)
to lists of hostnames as listed in the inventory file.
("Hostnames" can also be IP addresses.)
If the hostname in the file includes :<port>, that will be included here as well.
"""
data_loader = DataLoader()
inventory = get_inventory(filename, data_loader=data_loader)
var_manager = VariableManager(data_loader, inventory)
port_map = {host.name: var_manager.get_vars(host=host).get('ansible_port')
for host in inventory.get_hosts()}
return {group: [
'{}:{}'.format(host, port_map[host])
if port_map[host] is not None else host
for host in hosts
] for group, hosts in get_inventory(filename).get_groups_dict().items()}
def read_hosts(sources):
hosts_list = []
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources=sources)
variable_manager = VariableManager(loader=loader, inventory=inventory)
host_list = inventory.hosts
for _host in host_list:
host = inventory.get_host(hostname=_host)
extra_vars = variable_manager.get_vars(host=host)
ans_port = extra_vars.get("ansible_ssh_port") if extra_vars.get(
"ansible_ssh_port") else extra_vars.get("ansible_port")
ans_host = extra_vars.get("ansible_ssh_host") if extra_vars.get(
"ansible_ssh_host") else extra_vars.get("ansible_host")
hosts_list.append({
"hostname": extra_vars.get("inventory_hostname"),
"host": ans_host,
"port": ans_port if ans_port else 22,
"group_name": extra_vars.get("group_names")
})
return hosts_list
def ansible_inventory(inventory_dir, host, ignore_ansible_stderr=True):
from ansible.vars.manager import VariableManager
from ansible.inventory.manager import InventoryManager
from ansible.parsing.dataloader import DataLoader
# bug solved in ansible 2.6+: printing an empty stderr!
if ignore_ansible_stderr: sys.stderr = open(os.devnull, 'w')
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources=unicode(inventory_dir))
vm = VariableManager(loader=loader, inventory=inventory)
hosts = inventory.get_hosts(unicode(host))
if len(hosts) != 1:
raise KeyError('Host %s does not match a (single) host in %s' % (host, inventory_dir))
hostvars = vm.get_vars(host=hosts[0], include_hostvars=False)
# restore default
if ignore_ansible_stderr: sys.stderr = sys.__stderr__
return hostvars
def main():
parser = argparse.ArgumentParser(
description='Execute troubleshooting operation(s)')
parser.add_argument('-u',
'--user',
dest='user',
metavar='<username>',
help='provide username for ssh login to devices')
parser.add_argument('-p',
'--pass',
dest='passwd',
metavar='<password>',
help='provide ssh password or passphrase')
parser.add_argument('-n',
'--nopass',
action='store_true',
help='disable password prompting')
parser.add_argument('-c',
'--config',
dest='ssh_config',
metavar='<ssh_config>',
default='',
help='provide ssh config path')
parser.add_argument('-i',
'--inventory',
dest='inventory_path',
metavar='<inventory_path>',
help='provide ansible inventory path')
parser.add_argument('-l',
'--limit',
dest='limit',
metavar='<limit>',
help='specify host or group to run operations on')
parser.add_argument('-q',
'--quiet',
action='store_true',
help='disable optional interactive prompts')
args = parser.parse_args()
print(
f"{Fore.YELLOW}Welcome to the Python license management script for Junos boxes using PyEZ{Style.RESET_ALL}"
)
if (not args.user and not args.inventory_path and not args.quiet
and validate_bool(
"Would you like to print the command line help? (y/n) "
"(type n to continue in interactive mode) ")):
parser.print_help()
sys.exit(0)
user = validate_str("Enter your username: "******"Enter your password: "******"Enter path to ssh config: ",
cli_input=args.ssh_config)
else:
ssh_config = None
if not args.inventory_path:
inventory_dir = Path("inventory")
inventory_choices = [x for x in inventory_dir.iterdir() if x.is_dir()]
inventory_choices.sort()
print("\nAvailable Datacenters:")
for idx, choice in enumerate(inventory_choices):
print(f"{idx+1}: {choice.name}")
user_choice = validate_int(
"\nSelect Datacenter (Type Number only and press Enter):",
input_min=1,
input_max=inventory_choices.__len__())
choice = inventory_choices[user_choice - 1]
datacenter = choice.as_posix()
print(f"Datacenter {choice.name} selected")
else:
datacenter = args.inventory_path
# Ensure inventory path exists. Safeguard mainly when user provides path via cmd line
# extract the datacenter name from the path
dc_name = datacenter.rstrip('/').split('/')[-1]
if not Path(datacenter).exists():
print(f"Inventory Path '{datacenter}' does not exist. quitting...")
sys.exit(1)
if (not args.limit and not args.quiet and validate_bool(
"Do you want to limit the execution to a specific set of hosts or groups? (y/n) "
)):
limit = validate_str(
"Wildcard matching is supported like * and ? or [1-6] or [a:d] "
"i.e. qfx5?00-[a:d] or qfx5100*\nEnter your limit: ")
elif args.limit:
limit = args.limit
else:
limit = None
loader = DataLoader()
inventory = InventoryManager(loader=loader, sources=datacenter)
variables = VariableManager(loader=loader, inventory=inventory)
if limit:
if "*" in limit:
limit = limit.replace("*", ".*")
if "?" in limit:
limit = limit.replace("?", ".")
if ":" in limit:
limit = limit.replace(":", "-")
license_status_path = Path(f'{dc_name}-license_status.yml')
if not license_status_path.exists():
license_status_path.touch()
license_status_path.chmod(0o660)
license_status = {'licensed': {}, 'unlicensed': {}}
yaml.dump(license_status, license_status_path)
else:
license_status = yaml.load(license_status_path)
for host in inventory.get_hosts():
hostname = host.get_name()
match = False
if limit:
if re.match(limit, hostname):
match = True
else:
for group in (str(g) for g in host.get_groups()):
if re.match(limit, group):
match = True
if not match:
continue
netconf_port = variables.get_vars(host=host)['netconf_port']
try:
ansible_host = variables.get_vars(host=host)['ansible_host']
except:
ansible_host = hostname
# Begin Device Output to User
print(
f"{Fore.BLUE}{Style.BRIGHT}Checking existing host_vars structure for device {hostname}{Style.RESET_ALL}"
)
try:
host_path = Path(f'{datacenter}/host_vars/{hostname}')
license_path = Path(
f'{datacenter}/host_vars/{hostname}/licenses.yml')
if not host_path.exists():
print(f"Creating host_vars directory for host {hostname}")
host_path.mkdir()
host_path.chmod(0o770)
old_yml_path = Path(f'{datacenter}/host_vars/{hostname}.yml')
new_yml_path = Path(
f'{datacenter}/host_vars/{hostname}/system.yml')
if old_yml_path.exists():
print(
f"Moving old host_vars file for {hostname} to {new_yml_path}"
)
old_yml_path.rename(new_yml_path)
licenses = None
if license_path.exists():
print(f"loading license.yml for host {hostname}")
licenses = yaml.load(license_path)
else:
print(f"initializing license.yml for host {hostname}")
license_path.touch(
) # mode in touch does not set correctly. could be a umask issue
license_path.chmod(0o660) # mode works with chmod as expected.
if licenses == None:
licenses = {}
if 'license_keys' not in licenses:
licenses['license_keys'] = []
yaml.dump(licenses, license_path)
# Saving a copy for comparison later to see if we need to write updates to the yml
original_licenses_in_yml = licenses['license_keys'].copy()
print(
f"{Fore.BLUE}{Style.BRIGHT}Retrieving existing licenses for device {hostname}{Style.RESET_ALL}"
)
with Device(host=ansible_host,
port=netconf_port,
user=user,
passwd=passwd,
ssh_config=ssh_config,
auto_probe=5) as dev:
serial = dev.facts['serialnumber']
model = dev.facts['model']
hostname = dev.facts['hostname']
print(f"device {hostname} has serialnumber {serial}")
license_keys = dev.rpc.get_license_key_information()
for key in license_keys:
key_data = key.xpath('key-data')
key_data = key_data[0].text.strip()
key_data = ' '.join(key_data.split('\n'))
key_data = ' '.join(key_data.split())
if key_data not in licenses['license_keys']:
key_name = key_data.split(' ')[0]
print(
f"{Fore.MAGENTA}Found unmanaged license key {key_name} on device {hostname}. "
f"Saving...{Style.RESET_ALL}")
licenses['license_keys'].append(key_data)
new_license_paths = Path("licenses/").glob(f"{serial}*.txt")
for new_license_path in new_license_paths:
# These two lines processed license files in the old multiline format which had a header that we skipped
# new_license = new_license_path.read_text().splitlines()[8:]
# new_license = ' '.join([line.strip()
# for line in new_license if line is not ''])
# This is the new one line format where the files are generated by the
# build_individual_licenses.py helper script
new_license = new_license_path.read_text().strip()
if new_license not in licenses['license_keys']:
key_name = new_license.split(' ')[0]
print(
f"{Fore.MAGENTA}Found new license key {key_name} in file '{new_license_path}' "
f"for device {hostname}. Converting...{Style.RESET_ALL}"
)
licenses['license_keys'].append(new_license)
if licenses['license_keys'] != original_licenses_in_yml:
print(f"Updating licenses.yml for host {hostname}...")
yaml.dump(licenses, license_path)
else:
print(
f"{Fore.GREEN}Licenses already in sync for host {hostname}{Style.RESET_ALL}"
)
# Track what devices are licensed by datacenter
if licenses['license_keys'] == []:
try:
del license_status['licensed'][model][hostname]
except KeyError:
pass
if model not in license_status['unlicensed'].keys():
license_status['unlicensed'][model] = {}
if hostname not in license_status['unlicensed'][model].keys():
license_status['unlicensed'][model][hostname] = serial
else:
try:
del license_status['unlicensed'][model][hostname]
except KeyError:
pass
if model not in license_status['licensed'].keys():
license_status['licensed'][model] = {}
if hostname not in license_status['licensed'][model].keys():
license_status['licensed'][model][hostname] = serial
except ConnectAuthError as err:
print("Unable to login. Check username/password")
print("Exiting so you don't lock yourself out :)")
sys.exit(1)
except (ProbeError, ConnectError) as err:
print(
"Cannot connect to device: \nMake sure device is reachable and "
"'set system services netconf ssh' is set")
except Exception as err:
print(err.__class__.__name__ + ": " + err)
sys.exit(1)
# Remove empty subkeys (Model numbers with no entries)
license_status_copy = copy.deepcopy(license_status)
for k, v in license_status_copy['licensed'].items():
if v == {}:
del license_status['licensed'][k]
for k, v in license_status_copy['unlicensed'].items():
if v == {}:
del license_status['unlicensed'][k]
yaml.dump(license_status, license_status_path)
print(f"{Fore.YELLOW}License Sync Complete!{Style.RESET_ALL}")
class AnsibleRunner:
def __init__(self,
inventory: List[str] = None):
self.options = Options()
self.data_loader = DataLoader()
self.inventory_manager = InventoryManager(
loader=self.data_loader,
sources=inventory
)
self.variable_manager = VariableManager(
loader=self.data_loader,
inventory=self.inventory_manager
)
def add_host(self, host: Host) -> None:
self.inventory_manager.add_host(
host.address,
group=host.group
)
h = self.inventory_manager.get_host(
host.address
)
h.set_variable(
'ansible_ssh_user',
host.username
)
h.set_variable(
'ansible_ssh_private_key_file',
host.private_key_file
)
for (key, val) in host.variables.items():
h.set_variable(key, val)
def get_host_vars(self, host: Host) -> Optional[Dict[str, any]]:
h = self.inventory_manager.get_host(
host.address
)
return h.get_vars() if h is not None else None
def get_local_facts(self, host: Host) -> Optional[Dict[str, any]]:
h = self.inventory_manager.get_host(
host.address
)
if h is None:
return None
v = self.variable_manager.get_vars(host=h)
return v.get('ansible_facts', {}).get('ansible_local', {})
def play(self,
playbook: str,
targets: List[str] = None,
extra_vars: Dict[str, str] = None) -> bool:
if extra_vars is None:
extra_vars = {}
if targets is not None:
extra_vars['targets'] = ','.join(targets)
self.variable_manager.extra_vars = extra_vars
pbex = PlaybookExecutor(
playbooks=[playbook],
inventory=self.inventory_manager,
variable_manager=self.variable_manager,
loader=self.data_loader,
options=self.options,
passwords=None
)
return pbex.run() == 0
class TestIncludeRole(unittest.TestCase):
def setUp(self):
self.loader = DictDataLoader({
'/etc/ansible/roles/l1/tasks/main.yml': """
- shell: echo 'hello world from l1'
- include_role: name=l2
""",
'/etc/ansible/roles/l1/tasks/alt.yml': """
- shell: echo 'hello world from l1 alt'
- include_role: name=l2 tasks_from=alt defaults_from=alt
""",
'/etc/ansible/roles/l1/defaults/main.yml': """
test_variable: l1-main
l1_variable: l1-main
""",
'/etc/ansible/roles/l1/defaults/alt.yml': """
test_variable: l1-alt
l1_variable: l1-alt
""",
'/etc/ansible/roles/l2/tasks/main.yml': """
- shell: echo 'hello world from l2'
- include_role: name=l3
""",
'/etc/ansible/roles/l2/tasks/alt.yml': """
- shell: echo 'hello world from l2 alt'
- include_role: name=l3 tasks_from=alt defaults_from=alt
""",
'/etc/ansible/roles/l2/defaults/main.yml': """
test_variable: l2-main
l2_variable: l2-main
""",
'/etc/ansible/roles/l2/defaults/alt.yml': """
test_variable: l2-alt
l2_variable: l2-alt
""",
'/etc/ansible/roles/l3/tasks/main.yml': """
- shell: echo 'hello world from l3'
""",
'/etc/ansible/roles/l3/tasks/alt.yml': """
- shell: echo 'hello world from l3 alt'
""",
'/etc/ansible/roles/l3/defaults/main.yml': """
test_variable: l3-main
l3_variable: l3-main
""",
'/etc/ansible/roles/l3/defaults/alt.yml': """
test_variable: l3-alt
l3_variable: l3-alt
"""
})
self.var_manager = VariableManager(loader=self.loader)
def tearDown(self):
pass
def get_tasks_vars(self, play, tasks):
for task in flatten_tasks(tasks):
role = task._role
if not role:
continue
yield (role.get_name(),
self.var_manager.get_vars(play=play, task=task))
@patch('ansible.playbook.role.definition.unfrackpath',
mock_unfrackpath_noop)
def test_simple(self):
"""Test one-level include with default tasks and variables"""
play = Play.load(dict(
name="test play",
hosts=['foo'],
gather_facts=False,
tasks=[
{'include_role': 'name=l3'}
]
), loader=self.loader, variable_manager=self.var_manager)
tasks = play.compile()
for role, task_vars in self.get_tasks_vars(play, tasks):
self.assertEqual(task_vars.get('l3_variable'), 'l3-main')
self.assertEqual(task_vars.get('test_variable'), 'l3-main')
@patch('ansible.playbook.role.definition.unfrackpath',
mock_unfrackpath_noop)
def test_simple_alt_files(self):
"""Test one-level include with alternative tasks and variables"""
play = Play.load(dict(
name="test play",
hosts=['foo'],
gather_facts=False,
tasks=[{'include_role': 'name=l3 tasks_from=alt defaults_from=alt'}]),
loader=self.loader, variable_manager=self.var_manager)
tasks = play.compile()
for role, task_vars in self.get_tasks_vars(play, tasks):
self.assertEqual(task_vars.get('l3_variable'), 'l3-alt')
self.assertEqual(task_vars.get('test_variable'), 'l3-alt')
@patch('ansible.playbook.role.definition.unfrackpath',
mock_unfrackpath_noop)
def test_nested(self):
"""
Test nested includes with default tasks and variables.
Variables from outer roles should be inherited, but overridden in inner
roles.
"""
play = Play.load(dict(
name="test play",
hosts=['foo'],
gather_facts=False,
tasks=[
{'include_role': 'name=l1'}
]
), loader=self.loader, variable_manager=self.var_manager)
tasks = play.compile()
for role, task_vars in self.get_tasks_vars(play, tasks):
# Outer-most role must not have variables from inner roles yet
if role == 'l1':
self.assertEqual(task_vars.get('l1_variable'), 'l1-main')
self.assertEqual(task_vars.get('l2_variable'), None)
self.assertEqual(task_vars.get('l3_variable'), None)
self.assertEqual(task_vars.get('test_variable'), 'l1-main')
# Middle role must have variables from outer role, but not inner
elif role == 'l2':
self.assertEqual(task_vars.get('l1_variable'), 'l1-main')
self.assertEqual(task_vars.get('l2_variable'), 'l2-main')
self.assertEqual(task_vars.get('l3_variable'), None)
self.assertEqual(task_vars.get('test_variable'), 'l2-main')
# Inner role must have variables from both outer roles
elif role == 'l3':
self.assertEqual(task_vars.get('l1_variable'), 'l1-main')
self.assertEqual(task_vars.get('l2_variable'), 'l2-main')
self.assertEqual(task_vars.get('l3_variable'), 'l3-main')
self.assertEqual(task_vars.get('test_variable'), 'l3-main')
@patch('ansible.playbook.role.definition.unfrackpath',
mock_unfrackpath_noop)
def test_nested_alt_files(self):
"""
Test nested includes with alternative tasks and variables.
Variables from outer roles should be inherited, but overridden in inner
roles.
"""
play = Play.load(dict(
name="test play",
hosts=['foo'],
gather_facts=False,
tasks=[
{'include_role': 'name=l1 tasks_from=alt defaults_from=alt'}
]
), loader=self.loader, variable_manager=self.var_manager)
tasks = play.compile()
for role, task_vars in self.get_tasks_vars(play, tasks):
# Outer-most role must not have variables from inner roles yet
if role == 'l1':
self.assertEqual(task_vars.get('l1_variable'), 'l1-alt')
self.assertEqual(task_vars.get('l2_variable'), None)
self.assertEqual(task_vars.get('l3_variable'), None)
self.assertEqual(task_vars.get('test_variable'), 'l1-alt')
# Middle role must have variables from outer role, but not inner
elif role == 'l2':
self.assertEqual(task_vars.get('l1_variable'), 'l1-alt')
self.assertEqual(task_vars.get('l2_variable'), 'l2-alt')
self.assertEqual(task_vars.get('l3_variable'), None)
self.assertEqual(task_vars.get('test_variable'), 'l2-alt')
# Inner role must have variables from both outer roles
elif role == 'l3':
self.assertEqual(task_vars.get('l1_variable'), 'l1-alt')
self.assertEqual(task_vars.get('l2_variable'), 'l2-alt')
self.assertEqual(task_vars.get('l3_variable'), 'l3-alt')
self.assertEqual(task_vars.get('test_variable'), 'l3-alt')
class MyInventory():
'''
resource = [{'hostid': '1231', 'hostname': 'h1', 'hostip': '1.1.1.1'},
{'hostid': '2345', 'hostname': 'h2', 'hostip': '2.2.2.2'},
]
resource = {'groupname1': {
'hosts': [
{'hostid': '1231', 'hostname': 'h1', 'hostip': '1.1.1.1'},
{'hostid': '2231', 'hostname': 'h2', 'hostip': '1.1.1.2'},
],
'groupvars': {"k1":"v1"}
},
'groupname2': {'hosts': [], 'groupvars': {}},
}
'''
# edit ori code ansible/inventory/manage.pay line215 try if C.InventoryManager_PARSE_NOSOURCE:pass
constants.InventoryManager_PARSE_NOSOURCE = True
def __init__(self, resource):
self.resource = resource
self.loader = DataLoader()
# self.inventory=InventoryManager(loader=self.loader,sources=['/etc/ansible/hosts'])
self.inventory = InventoryManager(loader=self.loader)
self.variable_manager = VariableManager(loader=self.loader,
inventory=self.inventory)
self._parse(self.resource)
def _parse(self, resource):
if isinstance(resource, list):
self._addGroupHosts(self.resource)
elif isinstance(resource, dict):
# logging.info('parsing resuorce: %s'%(self.resource))
for groupname, hosts_and_groupvars in self.resource.items():
print("[1] groupname: %s |hostsandvars: %s" %
(groupname, hosts_and_groupvars)) # debug [1]
self._addGroupHosts(hosts_and_groupvars.get('hosts'),
groupname,
hosts_and_groupvars.get('groupvars'))
else:
logging.error('resource error ,need dict or list')
def _addGroupHosts(self, hosts, groupname='default', groupvars=None):
self.inventory.add_group(group=groupname)
group = Group(groupname)
if groupvars:
for k, v in groupvars.items():
group.set_variable(k, v)
# hosts=[{'hostid':'123','hostname':'h1','hostip':'192.168.188.20'}
for host in hosts:
hostid = host.get('hostid')
hostname = host.get('hostname')
hostip = host.get('hostip')
username = host.get('username')
password = host.get('password')
port = host.get('port', 22)
sshkey = host.get('sshkey')
if hostname:
self.inventory.add_host(
host=hostname, group=groupname
) # by default, indentify by hostname and need
hostobj = self.inventory.get_host(
hostname=hostname) # add host= , get hostname=
self.variable_manager.set_host_variable(
host=hostobj, varname='ansible_ssh_host', value=hostip)
self.variable_manager.set_host_variable(
host=hostobj, varname='ansible_ssh_port', value=port)
self.variable_manager.set_host_variable(
host=hostobj, varname='ansible_ssh_user', value=username)
self.variable_manager.set_host_variable(
host=hostobj, varname='ansible_ssh_pass', value=password)
self.variable_manager.set_host_variable(
host=hostobj,
varname='ansible_ssh_private_key_file',
value=sshkey)
# TODO: other vars such as become-method-user-pass
#hostobj.set_variable('ansible_ssh_port',port)
for k, v in host.items():
if k not in [
'hostip', 'port', 'username', 'password', 'sshkey'
]:
hostobj.set_variable(k, v)
else:
logging.warning('resource error:cant get hostname from | %s' %
resource)
def testcase(self):
print(self.inventory.get_groups_dict())
host = self.inventory.get_host(hostname='h1')
print(self.variable_manager.get_vars(host=host))
def test_variable_manager_precedence(self):
# FIXME: this needs to be redone as dataloader is not the automatic source of data anymore
return
# pylint: disable=unreachable
'''
Tests complex variations and combinations of get_vars() with different
objects to modify the context under which variables are merged.
'''
# FIXME: BCS makethiswork
# return True
mock_inventory = MagicMock()
inventory1_filedata = """
[group2:children]
group1
[group1]
host1 host_var=host_var_from_inventory_host1
[group1:vars]
group_var = group_var_from_inventory_group1
[group2:vars]
group_var = group_var_from_inventory_group2
"""
fake_loader = DictDataLoader({
# inventory1
'/etc/ansible/inventory1': inventory1_filedata,
# role defaults_only1
'/etc/ansible/roles/defaults_only1/defaults/main.yml': """
default_var: "default_var_from_defaults_only1"
host_var: "host_var_from_defaults_only1"
group_var: "group_var_from_defaults_only1"
group_var_all: "group_var_all_from_defaults_only1"
extra_var: "extra_var_from_defaults_only1"
""",
'/etc/ansible/roles/defaults_only1/tasks/main.yml': """
- debug: msg="here i am"
""",
# role defaults_only2
'/etc/ansible/roles/defaults_only2/defaults/main.yml': """
default_var: "default_var_from_defaults_only2"
host_var: "host_var_from_defaults_only2"
group_var: "group_var_from_defaults_only2"
group_var_all: "group_var_all_from_defaults_only2"
extra_var: "extra_var_from_defaults_only2"
""",
})
inv1 = InventoryManager(loader=fake_loader, sources=['/etc/ansible/inventory1'])
v = VariableManager(inventory=mock_inventory, loader=fake_loader)
v._fact_cache = defaultdict(dict)
play1 = Play.load(dict(
hosts=['all'],
roles=['defaults_only1', 'defaults_only2'],
), loader=fake_loader, variable_manager=v)
# first we assert that the defaults as viewed as a whole are the merged results
# of the defaults from each role, with the last role defined "winning" when
# there is a variable naming conflict
res = v.get_vars(play=play1)
self.assertEqual(res['default_var'], 'default_var_from_defaults_only2')
# next, we assert that when vars are viewed from the context of a task within a
# role, that task will see its own role defaults before any other role's
blocks = play1.compile()
task = blocks[1].block[0]
res = v.get_vars(play=play1, task=task)
self.assertEqual(res['default_var'], 'default_var_from_defaults_only1')
# next we assert the precedence of inventory variables
v.set_inventory(inv1)
h1 = inv1.get_host('host1')
res = v.get_vars(play=play1, host=h1)
self.assertEqual(res['group_var'], 'group_var_from_inventory_group1')
self.assertEqual(res['host_var'], 'host_var_from_inventory_host1')
# next we test with group_vars/ files loaded
fake_loader.push("/etc/ansible/group_vars/all", """
group_var_all: group_var_all_from_group_vars_all
""")
fake_loader.push("/etc/ansible/group_vars/group1", """
group_var: group_var_from_group_vars_group1
""")
fake_loader.push("/etc/ansible/group_vars/group3", """
# this is a dummy, which should not be used anywhere
group_var: group_var_from_group_vars_group3
""")
fake_loader.push("/etc/ansible/host_vars/host1", """
host_var: host_var_from_host_vars_host1
""")
fake_loader.push("group_vars/group1", """
playbook_group_var: playbook_group_var
""")
fake_loader.push("host_vars/host1", """
playbook_host_var: playbook_host_var
""")
res = v.get_vars(play=play1, host=h1)
# self.assertEqual(res['group_var'], 'group_var_from_group_vars_group1')
# self.assertEqual(res['group_var_all'], 'group_var_all_from_group_vars_all')
# self.assertEqual(res['playbook_group_var'], 'playbook_group_var')
# self.assertEqual(res['host_var'], 'host_var_from_host_vars_host1')
# self.assertEqual(res['playbook_host_var'], 'playbook_host_var')
# add in the fact cache
v._fact_cache['host1'] = dict(fact_cache_var="fact_cache_var_from_fact_cache")
res = v.get_vars(play=play1, host=h1)
self.assertEqual(res['fact_cache_var'], 'fact_cache_var_from_fact_cache')
