def __init__(self, ctrl, vault_password=None):
from ploy_ansible import get_playbooks_directory
kwargs = dict(host_list=[])
if vault_password is not None:
kwargs['vault_password'] = vault_password
BaseInventory.__init__(self, **kwargs)
self.ctrl = ctrl
self.set_playbook_basedir(get_playbooks_directory(ctrl.config))
groups = {}
groups['all'] = self.get_group('all')
seen = set()
for instance in self.ctrl.instances.values():
if instance.uid in seen:
continue
seen.add(instance.uid)
h = Host(instance.uid)
add_to = ['all', '%ss' % instance.sectiongroupname]
if hasattr(instance, 'master'):
master = instance.master
if instance == getattr(master, 'instance', None):
add_to.append('masters')
else:
add_to.append('%s-instances' % master.id)
for group in add_to:
g = groups.get(group)
if g is None:
g = self.get_group(group)
if g is None:
g = Group(group)
self.add_group(g)
groups[group] = g
g.add_host(h)
self._vars_plugins = [x for x in utils.plugins.vars_loader.all(self)]
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""dynamic inventory group"""
my_group = Group(name=groupname)
if groupvars:
for key, value in groupvars.items():
my_group.set_variable(key, value)
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in host.items():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def __init__(self, aws):
BaseInventory.__init__(
self,
host_list=[])
self.aws = aws
ansible_config = aws.config.get('global', {}).get('ansible', {})
if 'playbooks-directory' in ansible_config:
self.set_playbook_basedir(ansible_config['playbooks-directory'])
groups = {}
groups['all'] = self.get_group('all')
for instance in self.aws.instances.values():
h = Host(instance.id)
add_to = ['all', '%ss' % instance.sectiongroupname]
if hasattr(instance, 'master'):
master = instance.master
if instance == getattr(master, 'instance', None):
add_to.append('masters')
else:
add_to.append('%s-instances' % master.id)
for group in add_to:
g = groups.get(group)
if g is None:
g = self.get_group(group)
if g is None:
g = Group(group)
self.add_group(g)
groups[group] = g
g.add_host(h)
self._vars_plugins = []
self._vars_plugins.append(VarsModule(self))
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def run_ansible_in_python(task):
stats = callbacks.AggregateStats()
## everything between this comment
## and the "end" should hopefully not be necessary
## after Ansible 1.4.6 or 1.5 is released, since there
## will be an implicit sys.executable interpreter for the localhost
## host. This will almost certainly need tweaking anyway for dynamic inventory
## in ec2. I'm not sure yet how this would work.
localhost = Host('localhost')
localhost.set_variable('ansible_python_interpreter', sys.executable)
all_group = Group('all')
all_group.add_host(localhost)
inventory = Inventory(None)
inventory.add_group(all_group)
os.putenv('EC2_INI_PATH', 'lib/glue/ec2-external.ini')
ec2_inventory = InventoryScript(filename='lib/glue/ec2.py')
inventory.parser = ec2_inventory
[inventory.add_group(group) for group in ec2_inventory.groups.values()]
## end
pb = playbook.PlayBook(playbook=task.inputs[0].abspath(),
inventory=inventory,
stats=stats,
callbacks=callbacks.PlaybookCallbacks(verbose=3),
runner_callbacks=callbacks.PlaybookRunnerCallbacks(stats, verbose=3)
)
pb.run()
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.items():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
# print("my_group--->", my_group.hosts)
self.inventory.add_group(my_group)
def playbook(**kwargs):
group_vars = kwargs['group_vars']
groups = kwargs['groups']
host_list = kwargs['host_list']
playbook_basedir = os.sep.join([ANSIBLE_PLAYBOOK_PATH, kwargs['playbook_basedir']])
playbooks = []
for pb in kwargs['playbooks']:
playbooks.append(os.sep.join([playbook_basedir, pb]))
job_id = kwargs['job_id']
loader = DataLoader()
vars = VariableManager()
# 指定inventory为一个目录，设置所有主机，包含group和host
invertory = Inventory(loader, vars,
host_list=host_list)
invertory.set_playbook_basedir(playbook_basedir)
for group_name, hosts in groups.items():
t_group = Group(group_name)
for host in hosts:
t_host = Host(host)
t_group.add_host(t_host)
invertory.add_group(t_group)
vars.set_inventory(invertory)
display = LogDisplay(logname=job_id)
callback = CALLBACKMODULE[CALLBACK](display=display)
Options = namedtuple('Options', ['connection', 'module_path', 'forks', 'timeout', 'remote_user',
'ask_pass', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become', 'become_method', 'become_user',
'ask_value_pass', 'verbosity', 'check', 'listhosts',
'listtags', 'listtasks', 'syntax'])
options = Options(connection='smart', module_path='/usr/share/ansible', forks=100, timeout=10,
remote_user='******', ask_pass=False, private_key_file=None, ssh_common_args=None,
ssh_extra_args=None, sftp_extra_args=None, scp_extra_args=None, become=None,
become_method=None, become_user='******', ask_value_pass=False, verbosity=None,
check=False, listhosts=None, listtags=None, listtasks=None, syntax=None)
passwords = dict()
pb_executor = PlaybookExecutor(playbooks, invertory, vars, loader, options, passwords)
pb_executor._tqm._stdout_callback = callback
pb_executor.run()
return display.get_log_json()
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
:param hosts:
:param groupname:
:param groupvars:
:return:
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
if username == 'root':
keyfile = "/root/.ssh/id_rsa"
else:
keyfile = "/home/{user}/.ssh/id_rsa".format(user=username)
ssh_key = host.get("ssh_key", keyfile)
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostip = host.get("ip")
# hostname = host.get("ip",hostip)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
connection = host.get("connection", 'smart')
sudo_pass = host.get("sudo_passwd")
if username == 'root': ssh_key = "/root/.ssh/id_rsa"
else: ssh_key = "/home/{user}/.ssh/id_rsa".format(user=username)
if not os.path.exists(ssh_key): ssh_key = host.get("ssh_key")
my_host = Host(name=hostip, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_sudo_pass', sudo_pass)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
my_host.set_variable('ansible_connection', connection)
# set other variables
for key, value in host.iteritems():
if key not in ["ip", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
try:
self.add_group(my_group)
except Exception as ex:
logger.error(msg="ansible添加资产组失败: {ex}".format(ex=ex))
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
动态创建主机组
:param hosts: 主机列表
:param groupname: 组机组名
:param groupvars: 组参数
:return:
"""
my_group = Group(name=groupname) # 创建主机组对象
if groupvars:
# 设置主机组参数
for key, value in groupvars.items():
my_group.set_variable(key, value)
for host in hosts:
# 设置链接参数
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport) # 创建主机对象
# 设置主机参数
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_sudo_pass', password)
my_host.set_variable('ansible_sudo', 'yes')
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in host.items():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
my_group.add_host(my_host) # 将主机对象添加到主机组对象中
self.inventory.add_group(my_group) # 将主机组添加到主机配置清单中
def parse_inventory(self, host_list):
# 根据传入 主机列表生成 Inventory
"""用于生成动态构建Ansible Inventory.
self.host_list: [
{"name": "asset_name",
"ip": <ip>,
"port": <port>,
"user": <user>,
"pass": <pass>,
"key": <sshKey>,
"groups": ['group1', 'group2'],
"other_host_var": <other>},
{...},
]
:return: 返回一个Ansible的inventory对象
"""
# TODO: 验证输入
# 创建Ansible Group,如果没有则创建default组
ungrouped = Group('ungrouped')
all = Group('all')
all.add_child_group(ungrouped)
self.groups = dict(all=all, ungrouped=ungrouped)
for asset in host_list:
host = JMSHost(asset=asset) # 迭代获取到host
asset_groups = asset.get('groups')
if asset_groups:
for group_name in asset_groups:
if group_name not in self.groups:
group = Group(group_name)
self.groups[group_name] = group
else:
group = self.groups[group_name]
group.add_host(host)
else:
ungrouped.add_host(host)
all.add_host(host) # 添加到Inventory
def parse_inventory(self, host_list):
"""用于生成动态构建Ansible Inventory.
self.host_list: [
{"name": "asset_name",
"ip": <ip>,
"port": <port>,
"user": <user>,
"pass": <pass>,
"key": <sshKey>,
"groups": ['group1', 'group2'],
"other_host_var": <other>},
{...},
]
:return: 返回一个Ansible的inventory对象
"""
# TODO: 验证输入
# 创建Ansible Group,如果没有则创建default组
ungrouped = Group('ungrouped')
all = Group('all')
all.add_child_group(ungrouped)
self.groups = dict(all=all, ungrouped=ungrouped)
for asset in host_list:
host = JMSHost(asset=asset)
asset_groups = asset.get('groups')
if asset_groups:
for group_name in asset_groups:
if group_name not in self.groups:
group = Group(group_name)
self.groups[group_name] = group
else:
group = self.groups[group_name]
group.add_host(host)
else:
ungrouped.add_host(host)
all.add_host(host)
def runInv(request, **kwargs):
# Constants
C.HOST_KEY_CHECKING = False
template = JobTemplates.objects.get(pk=kwargs['pk'])
hostobjects = Hosts.objects.filter(inventory=template.inventory.pk)
inventoryName = template.inventory.name
inventory = Inventory()
# TODO: need to implement groups functionality
#Groups
#group = Group(inventoryName)
group = Group('mygroup')
for hostobj in hostobjects:
# Host defining
hostname = hostobj.name
port = hostobj.port
ip = hostobj.ipAddress
username = hostobj.username
password = hostobj.password
host = Host(hostname)
if ip:
host.set_variable('ansible_ssh_host', ip)
elif port:
host.set_variable('ansible_ssh_port', port)
host.set_variable('ansible_ssh_user', username)
host.set_variable('ansible_ssh_pass', password)
# Hostvars
hostvars = yaml.load(hostobj.variables)
if hostvars:
host.vars.update(hostvars)
group.add_host(host)
inventory.add_group(group)
# ExtraVars
extraVars = yaml.load(template.extra_variables)
# Callbacks
utils.VERBOSITY = 3
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
stats = callbacks.AggregateStats()
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
#runner_cb = myCustCallback(stats, template.pk, verbose=utils.VERBOSITY)
# Playbook gathering
playbookPath = os.path.join(template.project.directory, template.playbook)
# Playbook run
pb = PlayBook(
playbook=playbookPath,
host_list=None,
inventory=inventory, # Our hosts, the rendered inventory file
#remote_user='******',
#remote_pass='******',
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
extra_vars=extraVars,
#private_key_file='/path/to/key.pem'
)
pb.run()
return HttpResponseRedirect(reverse('jobtemplates:index'))
def playbook(**kwargs):
group_vars = kwargs['group_vars']
groups = kwargs['groups']
host_list = kwargs['host_list']
playbook_basedir = os.sep.join(
[ANSIBLE_PLAYBOOK_PATH, kwargs['playbook_basedir']])
playbooks = []
for pb in kwargs['playbooks']:
playbooks.append(os.sep.join([playbook_basedir, pb]))
job_id = kwargs['job_id']
loader = DataLoader()
vars = VariableManager()
# 指定inventory为一个目录，设置所有主机，包含group和host
invertory = Inventory(loader, vars, host_list=host_list)
invertory.set_playbook_basedir(playbook_basedir)
for group_name, hosts in groups.items():
t_group = Group(group_name)
for host in hosts:
t_host = Host(host)
t_group.add_host(t_host)
invertory.add_group(t_group)
vars.set_inventory(invertory)
display = LogDisplay(logname=job_id)
callback = CALLBACKMODULE[CALLBACK](display=display)
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'timeout', 'remote_user',
'ask_pass', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become', 'become_method',
'become_user', 'ask_value_pass', 'verbosity', 'check', 'listhosts',
'listtags', 'listtasks', 'syntax'
])
options = Options(connection='smart',
module_path='/usr/share/ansible',
forks=100,
timeout=10,
remote_user='******',
ask_pass=False,
private_key_file=None,
ssh_common_args=None,
ssh_extra_args=None,
sftp_extra_args=None,
scp_extra_args=None,
become=None,
become_method=None,
become_user='******',
ask_value_pass=False,
verbosity=None,
check=False,
listhosts=None,
listtags=None,
listtasks=None,
syntax=None)
passwords = dict()
pb_executor = PlaybookExecutor(playbooks, invertory, vars, loader, options,
passwords)
pb_executor._tqm._stdout_callback = callback
pb_executor.run()
return display.get_log_json()
def parse_inventory(self,host_list):
all = Group('all')
for host in host_list:
h = YoHost(host)
all.add_host(h)
self.groups = dict(all=all)
def get_info(self, ip, key_file):
"""
Get's information from the host via ansible.
:param ip: IP address to check.
:type ip: str
:param key_file: Full path the the file holding the private SSH key.
:type key_file: str
:returns: tuple -- (exitcode(int), facts(dict)).
"""
options = self.Options(connection='ssh',
module_path=None,
forks=1,
remote_user='******',
private_key_file=key_file,
ssh_common_args=None,
ssh_extra_args=None,
sftp_extra_args=None,
scp_extra_args=None,
become=None,
become_method=None,
become_user=None,
verbosity=None,
check=False)
# create inventory and pass to var manager
inventory = Inventory(loader=self.loader,
variable_manager=self.variable_manager,
host_list=ip)
# TODO: Fix this ... weird but works
group = Group(ip)
group.add_host(Host(ip, 22))
inventory.groups.update({ip: group})
# ---
self.variable_manager.set_inventory(inventory)
# create play with tasks
play_source = {
'name': 'gather',
'hosts': ip,
'gather_facts': 'yes',
'tasks': []
}
play = Play().load(play_source,
variable_manager=self.variable_manager,
loader=self.loader)
# actually run it
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=options,
passwords=self.passwords,
stdout_callback=LogForward(),
)
result = tqm.run(play)
fact_cache = self.variable_manager._fact_cache[ip]
facts = {}
facts['os'] = fact_cache['ansible_distribution'].lower()
facts['cpus'] = fact_cache['ansible_processor_cores']
facts['memory'] = fact_cache['ansible_memory_mb']['real']['total']
space = 0
for x in fact_cache['ansible_mounts']:
space += x['size_total']
facts['space'] = space
return (result, facts)
finally:
if tqm is not None:
tqm.cleanup()
def run_ansible_on_host(host, logger, config):
from ansible.plugins.callback import CallbackBase
# A rough logger that logs dict messages to standard logger
class ResultCallback(CallbackBase):
def __init__(self):
super(ResultCallback, self).__init__()
def v2_runner_on_ok(self, result, **kwargs):
self.log('ok :' + str(result._result), info=True)
def v2_runner_on_failed(self, result, **kwargs):
self.log(result._result, info=True)
def v2_runner_on_skipped(self, result, **kwargs):
self.log(result._result, info=True)
def v2_runner_on_unreachable(self, result, **kwargs):
self.log(result._result, info=True)
def v2_playbook_on_no_hosts_matched(self, *args, **kwargs):
self.log('no hosts matched!')
def v2_playbook_on_no_hosts_remaining(self, *args, **kwargs):
self.log('NO MORE HOSTS LEFT')
def v2_playbook_on_task_start(self, task, **kwargs):
self.log('starting task: ' + str(task))
def v2_playbook_on_start(self, playbook, **kwargs):
self.log('starting playbook' + str(playbook), info=True)
def v2_playbook_on_play_start(self, play, **kwargs):
self.log('starting play' + str(play), info=True)
def v2_playbook_on_stats(self, stats, info=True, **kwargs):
self.log('STATS FOR PLAY')
hosts = sorted(stats.processed.keys())
hosts.extend(stats.failures.keys())
hosts.extend(stats.dark.keys())
hosts.extend(stats.changed.keys())
hosts.extend(stats.skipped.keys())
hosts.extend(stats.ok.keys())
for h in hosts:
t = stats.summarize(h)
self.log(str(t))
def log(self, param, info=False):
if not info:
logger.debug(str(param))
else:
logger.info(str(param))
from ansible.parsing.dataloader import DataLoader
from ansible.inventory import Inventory, Group, Host
from ansible.executor import playbook_executor
from ansible.vars import VariableManager
from collections import namedtuple
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'become', 'become_method',
'become_user', 'check', 'ansible_user', 'listhosts', 'listtasks',
'listtags', 'syntax', 'ssh_private_key_file', 'host_key_checking'
])
options = Options(connection='ssh',
become=True,
become_method='sudo',
become_user='******',
check=False,
module_path=None,
forks=100,
ansible_user='******',
listhosts=False,
listtasks=False,
listtags=False,
syntax=False,
ssh_private_key_file=PEBBLES_SSH_KEY_LOCATION,
host_key_checking=False)
variable_manager = VariableManager()
loader = DataLoader()
a_host = Host(name=host['private_ip'])
a_host.set_variable('ansible_host', host['private_ip'])
a_group = Group(name='notebook_host')
a_group.add_host(a_host)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[host['private_ip']])
inventory.add_group(a_group)
variable_manager.set_inventory(inventory)
logger.debug('HOST:')
logger.debug(a_host.serialize())
logger.debug('HOSTs from inventory:')
# for some reason setting these before adding the host to inventory didn't work so well
# ToDo: read up on variable_manager and figure out a more elegant way to set the variables
for h_ in inventory.get_hosts():
h_.set_variable('ansible_user', 'cloud-user')
h_.set_variable('ansible_ssh_common_args',
'-o StrictHostKeyChecking=no')
h_.set_variable('ansible_ssh_private_key_file',
'/home/pebbles/.ssh/id_rsa')
extra_vars = dict()
extra_vars['ansible_ssh_extra_args'] = '-o StrictHostKeyChecking=no'
if 'DD_HOST_DATA_VOLUME_DEVICE' in config:
extra_vars['notebook_host_block_dev_path'] = config[
'DD_HOST_DATA_VOLUME_DEVICE']
variable_manager.extra_vars = extra_vars
pb_executor = playbook_executor.PlaybookExecutor(
playbooks=[
'/webapps/pebbles/source/ansible/notebook_playbook.yml'
],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None)
rescb = ResultCallback()
pb_executor._tqm._stdout_callback = rescb
logger.info('_prepare_host(): running ansible')
logger.info('_prepare_host(): inventory hosts')
for h_ in inventory.get_hosts():
logger.info(h_.serialize())
logger.info(h_.get_vars())
pb_executor.run()
stats = pb_executor._tqm._stats
run_success = True
hosts_list = sorted(stats.processed.keys())
if len(hosts_list) == 0:
logger.debug('no hosts handled')
for h in hosts_list:
t = stats.summarize(h)
logger.debug(t)
logger.debug(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
if run_success:
logger.debug('_prepare_host(): run successfull')
else:
logger.debug('_prepare_host(): run failed')
if getattr(pb_executor, '_unreachable_hosts', False):
logger.debug('UNREACHABLE HOSTS ' +
str(pb_executor._unreachable_hosts))
if getattr(pb_executor, '_failed_hosts', False):
logger.debug('FAILED_HOSTS ' + str(pb_executor._failed_hosts))
raise RuntimeError('run_ansible_on_host(%s) failed' % host['id'])
logger.debug('_prepare_host(): done running ansible')
def testRolleRun():
utils.VERBOSITY = 3
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
stats = callbacks.AggregateStats()
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
# hosts = ['dborrac1.ea.intropro.com']
myInventory = Inventory()
host = 'dborrac1.ea.intropro.com'
group = 'appSERVERS'
hosts = Host(host)
groups = Group(group)
myInventory.groups = [groups, ]
groups.add_host(hosts)
local = True
gather_facts = True
if gather_facts:
gather_facts = 'yes'
else:
gather_facts = 'no'
if local:
hosts.set_variable('ansible_connection', 'local')
role = 'bfmartin.ssh_known_hosts'
hostsPlay = 'appSERVERS'
extraVars = {
'ssh_known_hosts': [
{'name': host,
'state': 'present',
'aliases': [host.split('.')[0], ]},
],
'ssh_known_hosts_path': '~vagrant/.ssh/known_hosts',
}
playbookContent = """---
- hosts: %s
gather_facts: %s
roles:
- %s
""" % (hostsPlay, gather_facts, role)
tempFile = NamedTemporaryFile(delete=False)
tempFile.write(playbookContent)
tempFile.close()
pb = PlayBook(
playbook=tempFile.name,
inventory=myInventory, # Our hosts, the rendered inventory file
remote_user='******',
remote_pass='******',
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
extra_vars=extraVars,
#private_key_file='/path/to/key.pem'
)
pb.run()
os.remove(tempFile.name)
def get_info(self, ip, key_file):
"""
Get's information from the host via ansible.
:param ip: IP address to check.
:type ip: str
:param key_file: Full path the the file holding the private SSH key.
:type key_file: str
:returns: tuple -- (exitcode(int), facts(dict)).
"""
options = self.Options(
connection='ssh', module_path=None, forks=1,
remote_user='******', private_key_file=key_file,
ssh_common_args=None, ssh_extra_args=None,
sftp_extra_args=None, scp_extra_args=None,
become=None, become_method=None, become_user=None,
verbosity=None, check=False)
# create inventory and pass to var manager
inventory = Inventory(
loader=self.loader,
variable_manager=self.variable_manager,
host_list=ip)
# TODO: Fix this ... weird but works
group = Group(ip)
group.add_host(Host(ip, 22))
inventory.groups.update({ip: group})
# ---
self.variable_manager.set_inventory(inventory)
# create play with tasks
play_source = {
'name': 'gather',
'hosts': ip,
'gather_facts': 'yes',
'tasks': []
}
play = Play().load(
play_source,
variable_manager=self.variable_manager,
loader=self.loader)
# actually run it
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=options,
passwords=self.passwords,
stdout_callback=LogForward(),
)
result = tqm.run(play)
fact_cache = self.variable_manager._fact_cache[ip]
facts = {}
facts['os'] = fact_cache['ansible_distribution'].lower()
facts['cpus'] = fact_cache['ansible_processor_cores']
facts['memory'] = fact_cache['ansible_memory_mb']['real']['total']
space = 0
for x in fact_cache['ansible_mounts']:
space += x['size_total']
facts['space'] = space
return (result, facts)
finally:
if tqm is not None:
tqm.cleanup()
def _run(self,
ips,
key_file,
play_file,
expected_results=[0],
play_vars={}):
"""
Common code used for each run.
:param ips: IP address(es) to check.
:type ips: str or list
:param key_file: Full path the the file holding the private SSH key.
:type key_file: string
:param play_file: Path to the ansible play file.
:type play_file: str
:param expected_results: List of expected return codes. Default: [0]
:type expected_results: list
:returns: Ansible exit code
:type: int
"""
if type(ips) != list:
ips = [ips]
ssh_args = ('-o StrictHostKeyChecking=no -o '
'ControlMaster=auto -o ControlPersist=60s')
options = self.Options(connection='ssh',
module_path=None,
forks=1,
remote_user='******',
private_key_file=key_file,
ssh_common_args=ssh_args,
ssh_extra_args=ssh_args,
sftp_extra_args=None,
scp_extra_args=None,
become=None,
become_method=None,
become_user=None,
verbosity=None,
check=False)
# create inventory and pass to var manager
inventory = Inventory(loader=self.loader,
variable_manager=self.variable_manager,
host_list=ips)
self.logger.debug('Options: {0}'.format(options))
group = Group('commissaire_targets')
for ip in ips:
host = Host(ip, 22)
group.add_host(host)
inventory.groups.update({'commissaire_targets': group})
self.logger.debug('Inventory: {0}'.format(inventory))
self.variable_manager.set_inventory(inventory)
play_source = self.loader.load_from_file(play_file)[0]
play = Play().load(play_source,
variable_manager=self.variable_manager,
loader=self.loader)
# Add any variables provided into the play
play.vars.update(play_vars)
self.logger.debug(
'Running play for hosts {0}: play={1}, vars={2}'.format(
ips, play_source, play.vars))
# actually run it
for cnt in range(0, 3):
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=options,
passwords=self.passwords,
stdout_callback=LogForward(),
)
result = tqm.run(play)
# Deal with unreachable hosts (result == 3) by retrying
# up to 3 times, sleeping 5 seconds after each attempt.
if result == 3 and cnt < 2:
self.logger.warn(
'One or more hosts in {0} is unreachable, '
'retrying in 5 seconds...'.format(ips))
sleep(5)
else:
break
finally:
if tqm is not None:
self.logger.debug(
'Cleaning up after the TaskQueueManager.')
tqm.cleanup()
if result in expected_results:
self.logger.debug('{0}: Good result {1}'.format(ip, result))
fact_cache = self.variable_manager._fact_cache.get(ip, {})
return (result, fact_cache)
self.logger.debug('{0}: Bad result {1}'.format(ip, result))
raise Exception('Can not run for {0}'.format(ip))
def __init__(self, groups, playbook, private_key_file, display, become_pass={}, extraVars={}):
self.extraVars = extraVars
self.options = Options()
self.options.private_key_file = private_key_file
self.options.verbosity = display.verbosity
self.options.connection = 'ssh' # Need a connection type 'smart' or 'ssh'
self.options.become = True
self.options.become_method = 'sudo'
self.options.become_user = '******'
# Executor appears to have it's own
# verbosity object/setting as well
playbook_executor.verbosity = self.options.verbosity
# Become Pass Needed if not logging in as user root
passwords = {'become_pass':become_pass}
# Gets data from YAML/JSON files
self.loader = DataLoader()
self.loader.set_vault_password(os.environ.get('VAULT_PASS', 'xiaoy_pandan'))
# All the variables from all the various places
self.variable_manager = VariableManager()
self.variable_manager.extra_vars = self.extraVars
# Set inventory, using most of above objects
self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=[])
self.variable_manager.set_inventory(self.inventory)
# Set groups info
for info in groups:
group = Group(info['name'])
self.inventory.add_group(group)
hostInfos = info.get('hosts')
for hostInfo in hostInfos:
hostname = hostInfo.get('hostname')
hostip = hostInfo.get('ip', hostname)
hostport = hostInfo.get('port')
username = hostInfo.get('username')
password = hostInfo.get('password')
ssh_key = hostInfo.get('ssh_key')
host = Host(name=hostname, port=hostport)
host.set_variable('ansible_ssh_host', hostip)
host.set_variable('ansible_ssh_port', hostport)
host.set_variable('ansible_ssh_user', username)
host.set_variable('ansible_ssh_pass', password)
host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in hostInfo.iteritems():
if key not in ['hostname', 'port', 'username', 'password', 'ssh_key']:
host.set_variable(key, value)
group.add_host(host)
varInfos = info.get('vars')
if varInfos:
for key, value in varInfos.iteritems():
group.set_variable(key, value)
# Playbook to run. Assumes it is
# local to this python file
pb_dir = os.path.dirname(__file__)
playbook = '%s/%s' % (pb_dir, playbook)
# Setup playbook executor, but don't run until run() called
self.pbex = playbook_executor.PlaybookExecutor(
playbooks=[playbook],
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords=passwords)
self.display = display
self.callback = CallbackModule(self.display)
self.pbex._tqm._stdout_callback = self.callback
def _run(self, ips, key_file, play_file,
expected_results=[0], play_vars={}):
"""
Common code used for each run.
:param ips: IP address(es) to check.
:type ips: str or list
:param key_file: Full path to the file holding the private SSH key.
:type key_file: string
:param play_file: Path to the ansible play file.
:type play_file: str
:param expected_results: List of expected return codes. Default: [0]
:type expected_results: list
:returns: Ansible exit code
:type: int
"""
if type(ips) != list:
ips = [ips]
ssh_args = ('-o StrictHostKeyChecking=no -o '
'ControlMaster=auto -o ControlPersist=60s')
become = {
'become': None,
'become_user': None,
}
if self.remote_user != 'root':
self.logger.debug('Using user {0} for ssh communication.'.format(
self.remote_user))
become['become'] = True
become['become_user'] = '******'
options = self.Options(
connection='ssh', module_path=None, forks=1,
remote_user=self.remote_user, private_key_file=key_file,
ssh_common_args=ssh_args, ssh_extra_args=ssh_args,
sftp_extra_args=None, scp_extra_args=None,
become=become['become'], become_method='sudo',
become_user=become['become_user'],
verbosity=None, check=False)
# create inventory and pass to var manager
inventory = Inventory(
loader=self.loader,
variable_manager=self.variable_manager,
host_list=ips)
self.logger.debug('Options: {0}'.format(options))
group = Group('commissaire_targets')
for ip in ips:
host = Host(ip, 22)
group.add_host(host)
inventory.groups.update({'commissaire_targets': group})
self.logger.debug('Inventory: {0}'.format(inventory))
self.variable_manager.set_inventory(inventory)
play_source = self.loader.load_from_file(play_file)[0]
play = Play().load(
play_source,
variable_manager=self.variable_manager,
loader=self.loader)
# Add any variables provided into the play
play.vars.update(play_vars)
self.logger.debug(
'Running play for hosts {0}: play={1}, vars={2}'.format(
ips, play_source, play.vars))
# actually run it
for cnt in range(0, 3):
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=options,
passwords=self.passwords,
stdout_callback=LogForward(),
)
result = tqm.run(play)
# Deal with unreachable hosts (result == 3) by retrying
# up to 3 times, sleeping 5 seconds after each attempt.
if result == 3 and cnt < 2:
self.logger.warn(
'One or more hosts in {0} is unreachable, '
'retrying in 5 seconds...'.format(ips))
sleep(5)
else:
break
finally:
if tqm is not None:
self.logger.debug(
'Cleaning up after the TaskQueueManager.')
tqm.cleanup()
if result in expected_results:
self.logger.debug('{0}: Good result {1}'.format(ip, result))
fact_cache = self.variable_manager._fact_cache.get(ip, {})
return (result, fact_cache)
self.logger.debug('{0}: Bad result {1}'.format(ip, result))
raise Exception('Can not run for {0}'.format(ip))
def parse_inventory(self, host_list):
all = Group('all')
for host in host_list:
h = YoHost(host)
all.add_host(h)
self.groups = dict(all=all)
def _run(self, ip, key_file, play_source, expected_results=[0]):
"""
Common code used for each run.
:param ip: IP address to check.
:type ip: str
:param key_file: Full path the the file holding the private SSH key.
:type key_file: string
:param play_source: Ansible play.
:type play_source: dict
:param expected_results: List of expected return codes. Default: [0]
:type expected_results: list
:returns: Ansible exit code
:type: int
"""
ssh_args = ('-o StrictHostKeyChecking=no -o '
'ControlMaster=auto -o ControlPersist=60s')
options = self.Options(
connection='ssh', module_path=None, forks=1,
remote_user='******', private_key_file=key_file,
ssh_common_args=ssh_args, ssh_extra_args=ssh_args,
sftp_extra_args=None, scp_extra_args=None,
become=None, become_method=None, become_user=None,
verbosity=None, check=False)
# create inventory and pass to var manager
inventory = Inventory(
loader=self.loader,
variable_manager=self.variable_manager,
host_list=ip)
# TODO: Fix this ... weird but works
group = Group(ip)
group.add_host(Host(ip, 22))
inventory.groups.update({ip: group})
# ---
self.variable_manager.set_inventory(inventory)
play = Play().load(
play_source,
variable_manager=self.variable_manager,
loader=self.loader)
# actually run it
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=options,
passwords=self.passwords,
stdout_callback=LogForward(),
)
result = tqm.run(play)
finally:
if tqm is not None:
tqm.cleanup()
if result in expected_results:
self.logger.debug('{0}: Good result {1}'.format(ip, result))
fact_cache = self.variable_manager._fact_cache.get(ip, {})
return (result, fact_cache)
# TODO: Do something :-)
self.logger.debug('{0}: Bad result {1}'.format(ip, result))
raise Exception('Can not run for {0}'.format(ip))
def run_ansible_on_host(host, logger, driver_config):
from ansible.plugins.callback import CallbackBase
# A rough logger that logs dict messages to standard logger
class ResultCallback(CallbackBase):
def __init__(self):
super(ResultCallback, self).__init__()
def v2_runner_on_ok(self, result, **kwargs):
self.log('ok :' + str(result._result))
def v2_runner_on_failed(self, result, **kwargs):
warnings = result._result['warnings']
error = result._result['stderr']
if warnings:
self.log('warning : ' + str(result._result))
elif error:
self.log('error : ' + str(result._result), info=True)
def v2_runner_on_skipped(self, result, **kwargs):
self.log('skipped : ' + str(result._result))
def v2_runner_on_unreachable(self, result, **kwargs):
self.log('unreachable : ' + str(result._result), info=True)
def v2_playbook_on_no_hosts_matched(self, *args, **kwargs):
self.log('no hosts matched!')
def v2_playbook_on_no_hosts_remaining(self, *args, **kwargs):
self.log('NO MORE HOSTS LEFT')
def v2_playbook_on_task_start(self, task, **kwargs):
self.log('starting task: ' + str(task))
def v2_playbook_on_start(self, playbook, **kwargs):
self.log('starting playbook' + str(playbook), info=True)
def v2_playbook_on_play_start(self, play, **kwargs):
self.log('starting play' + str(play), info=True)
def v2_playbook_on_stats(self, stats, info=True, **kwargs):
self.log('STATS FOR PLAY')
hosts = sorted(stats.processed.keys())
hosts.extend(stats.failures.keys())
hosts.extend(stats.dark.keys())
hosts.extend(stats.changed.keys())
hosts.extend(stats.skipped.keys())
hosts.extend(stats.ok.keys())
for h in hosts:
t = stats.summarize(h)
self.log(str(t))
def log(self, param, info=False):
if not info:
logger.debug(str(param))
else:
logger.info(str(param))
from ansible.parsing.dataloader import DataLoader
from ansible.inventory import Inventory, Group, Host
from ansible.executor import playbook_executor
from ansible.vars import VariableManager
from collections import namedtuple
Options = namedtuple(
'Options',
[
'connection',
'module_path',
'forks',
'become',
'become_method',
'become_user',
'check',
'ansible_user',
'listhosts',
'listtasks',
'listtags',
'syntax',
'ssh_private_key_file',
'host_key_checking'
]
)
options = Options(
connection='ssh',
become=True,
become_method='sudo',
become_user='******',
check=False,
module_path=None,
forks=100,
ansible_user='******',
listhosts=False,
listtasks=False,
listtags=False,
syntax=False,
ssh_private_key_file=PEBBLES_SSH_KEY_LOCATION,
host_key_checking=False
)
variable_manager = VariableManager()
loader = DataLoader()
a_host = Host(name=host['private_ip'])
a_host.set_variable('ansible_host', host['private_ip'])
a_group = Group(name='notebook_host')
a_group.add_host(a_host)
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=[host['private_ip']])
inventory.add_group(a_group)
variable_manager.set_inventory(inventory)
logger.debug('HOST:')
logger.debug(a_host.serialize() )
logger.debug('HOSTs from inventory:')
# for some reason setting these before adding the host to inventory didn't work so well
# ToDo: read up on variable_manager and figure out a more elegant way to set the variables
for h_ in inventory.get_hosts():
h_.set_variable('ansible_user', 'cloud-user')
h_.set_variable('ansible_ssh_common_args', '-o StrictHostKeyChecking=no')
h_.set_variable('ansible_ssh_private_key_file', '/home/pebbles/.ssh/id_rsa')
extra_vars = dict()
extra_vars['ansible_ssh_extra_args'] = '-o StrictHostKeyChecking=no'
logger.debug('Setting driver config....')
if 'DD_HOST_DATA_VOLUME_DEVICE' in driver_config:
extra_vars['notebook_host_block_dev_path'] = driver_config['DD_HOST_DATA_VOLUME_DEVICE']
variable_manager.extra_vars = extra_vars
pb_executor = playbook_executor.PlaybookExecutor(
playbooks=['/webapps/pebbles/source/ansible/notebook_playbook.yml'],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None
)
rescb = ResultCallback()
pb_executor._tqm._stdout_callback = rescb
logger.info('_prepare_host(): running ansible')
logger.info('_prepare_host(): inventory hosts')
for h_ in inventory.get_hosts():
logger.info(h_.serialize())
logger.info(h_.get_vars())
pb_executor.run()
stats = pb_executor._tqm._stats
run_success = True
hosts_list = sorted(stats.processed.keys())
if len(hosts_list) == 0:
logger.debug('no hosts handled')
for h in hosts_list:
t = stats.summarize(h)
logger.debug(t)
logger.debug(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
if run_success:
logger.debug('_prepare_host(): run successfull')
else:
logger.debug('_prepare_host(): run failed')
if getattr(pb_executor, '_unreachable_hosts', False):
logger.debug('UNREACHABLE HOSTS ' + str(pb_executor._unreachable_hosts))
if getattr(pb_executor, '_failed_hosts', False):
logger.debug('FAILED_HOSTS ' + str(pb_executor._failed_hosts))
raise RuntimeError('run_ansible_on_host(%s) failed' % host['id'])
logger.debug('_prepare_host(): done running ansible')