class NetworkTest(object):
def __init__(self, playbook):
self.inventory = Inventory(host_list=[])
self.playbook = playbook
self.stac_nodes_group = Group(name='stac_nodes')
self.inventory.add_group(self.stac_nodes_group)
self.inv_vars = dict()
def set_inventory_vars(self, inv_vars):
self.inv_vars.update(inv_vars)
def add_stac_node(self, node):
stac_node = Host(name=node)
self.stac_nodes_group.add_host(stac_node)
def run(self):
stats = AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=utils.VERBOSITY)
runner_cb = PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = playbook.PlayBook(playbook=self.playbook,
stats=stats,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
inventory=self.inventory,
extra_vars=self.inv_vars,
check=False)
pr = pb.run()
print json.dumps(pr, sort_keys=True, indent=4, separators=(',', ': '))
def build_inventory(loader, variable_manager, group_names, playbook_basedir):
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['localhost'])
# because we just pass in a "host list" which isn't a real inventory file,
# we explicitly have to add all of the desired groups to the inventory. By
# default an "all" group is created whenever a new inventory is created
for group_name in group_names:
if not inventory.get_group(group_name):
inventory.add_group(Group(group_name))
# because we are explicitly adding groups, we also need to make sure that a
# playbook basedir is set so that `group_vars` can be loaded from the
# correct directory.
inventory.set_playbook_basedir(playbook_basedir)
inventory_hosts = inventory.get_hosts()
# for each group specified, ensure that the inventory's host (localhost) is
# explicitly in the group.
for group_name in group_names:
group = inventory.get_group(group_name)
if group.get_hosts():
continue
for host in inventory.get_hosts():
group.add_host(host)
return inventory
def run_ansible_in_python(task):
stats = callbacks.AggregateStats()
## everything between this comment
## and the "end" should hopefully not be necessary
## after Ansible 1.4.6 or 1.5 is released, since there
## will be an implicit sys.executable interpreter for the localhost
## host. This will almost certainly need tweaking anyway for dynamic inventory
## in ec2. I'm not sure yet how this would work.
localhost = Host('localhost')
localhost.set_variable('ansible_python_interpreter', sys.executable)
all_group = Group('all')
all_group.add_host(localhost)
inventory = Inventory(None)
inventory.add_group(all_group)
os.putenv('EC2_INI_PATH', 'lib/glue/ec2-external.ini')
ec2_inventory = InventoryScript(filename='lib/glue/ec2.py')
inventory.parser = ec2_inventory
[inventory.add_group(group) for group in ec2_inventory.groups.values()]
## end
pb = playbook.PlayBook(playbook=task.inputs[0].abspath(),
inventory=inventory,
stats=stats,
callbacks=callbacks.PlaybookCallbacks(verbose=3),
runner_callbacks=callbacks.PlaybookRunnerCallbacks(stats, verbose=3)
)
pb.run()
class NetworkTest(object):
def __init__(self, playbook):
self.inventory = Inventory(host_list=[])
self.playbook = playbook
self.stac_nodes_group = Group(name='stac_nodes')
self.inventory.add_group(self.stac_nodes_group)
self.inv_vars = dict()
def set_inventory_vars(self, inv_vars):
self.inv_vars.update(inv_vars)
def add_stac_node(self, node):
stac_node = Host(name=node)
self.stac_nodes_group.add_host(stac_node)
def run(self):
stats = AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=utils.VERBOSITY)
runner_cb = PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = playbook.PlayBook(playbook=self.playbook,
stats=stats,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
inventory=self.inventory,
extra_vars=self.inv_vars,
check=False)
pr = pb.run()
print json.dumps(pr, sort_keys=True, indent=4, separators=(',', ': '))
def build_inventory(loader, variable_manager, group_names, playbook_basedir):
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['localhost'])
# because we just pass in a "host list" which isn't a real inventory file,
# we explicitly have to add all of the desired groups to the inventory. By
# default an "all" group is created whenever a new inventory is created
for group_name in group_names:
if not inventory.get_group(group_name):
inventory.add_group(Group(group_name))
# because we are explicitly adding groups, we also need to make sure that a
# playbook basedir is set so that `group_vars` can be loaded from the
# correct directory.
inventory.set_playbook_basedir(playbook_basedir)
# for each group specified, ensure that the inventory's host (localhost) is
# explicitly in the group.
for group_name in group_names:
group = inventory.get_group(group_name)
if group.get_hosts():
continue
for host in inventory.get_hosts():
group.add_host(host)
return inventory
def add_group(self, group):
if isinstance(group, basestring):
group = Group(group)
try:
self.log.debug('add inventory group: %s' % group.name)
AnsibleInventory.add_group(self, group)
except AnsibleError, emsg:
raise InventoryError(emsg)
class MyInventory(Inventory):
"""
this is my ansible inventory object.
"""
def __init__(self, resource, loader, variable_manager):
self.resource = resource
self.inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[])
self.dynamic_inventory()
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def dynamic_inventory(self):
"""
add hosts to inventory.
"""
if isinstance(self.resource, list):
self.add_dynamic_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.iteritems():
self.add_dynamic_group(hosts_and_vars.get("hosts"), groupname,
hosts_and_vars.get("vars"))
class MyInventory(object):
def __init__(self, resource, loader, variable_manager):
self.resource = resource
# 创建主机配置清单对象
self.inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=[])
self.dynamic_inventory()
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
动态创建主机组
:param hosts: 主机列表
:param groupname: 组机组名
:param groupvars: 组参数
:return:
"""
my_group = Group(name=groupname) # 创建主机组对象
if groupvars:
# 设置主机组参数
for key, value in groupvars.items():
my_group.set_variable(key, value)
for host in hosts:
# 设置链接参数
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport) # 创建主机对象
# 设置主机参数
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_sudo_pass', password)
my_host.set_variable('ansible_sudo', 'yes')
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in host.items():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
my_group.add_host(my_host) # 将主机对象添加到主机组对象中
self.inventory.add_group(my_group) # 将主机组添加到主机配置清单中
def dynamic_inventory(self):
"""
动态配置主机清单对象
:return:
"""
if isinstance(self.resource, list):
# 只是主机列表，也就是所有的主机都属于同一个默认的主机组
self.add_dynamic_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
# 包含组的主机列表，也就是有若干个主机组
for groupname, hosts_and_vars in self.resource.items():
self.add_dynamic_group(hosts_and_vars.get("hosts"), groupname, hosts_and_vars.get("vars"))
class MyInventory(Inventory):
"""
this is my ansible inventory object.
"""
def __init__(self):
self.inventory = Inventory(my_inventory)
#self.gen_inventory()
def my_add_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def gen_inventory(self):
"""
add hosts to inventory.
"""
if isinstance(self.resource, list):
self.my_add_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.iteritems():
self.my_add_group(
hosts_and_vars.get("hosts"), groupname,
hosts_and_vars.get("vars"))
class MyInventory(Inventory):
def __init__(self, resource):
"""
resource的数据格式是一个列表字典，比如
{
"group1": {
"hosts": [{"hostname": "10.10.10.10", "port": "22", "username": "******", "password": "******"}, ...],
"vars": {"var1": value1, "var2": value2, ...}
}
}
如果你只传入1个列表，这默认该列表内的所有主机属于my_group组,比如
[{"hostname": "10.10.10.10", "port": "22", "username": "******", "password": "******"}, ...]
"""
self.resource = resource
self.inventory = Inventory(host_list=[])
self.gen_inventory()
def my_add_group(self, hosts, groupname, groupvars=None):
my_group = Group(name=groupname)
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
for host in hosts:
hostname = host.get('hostname')
hostip = host.get('ip', hostname)
hostport = host.get('port', 22)
username = host.get('username', 'root')
password = host.get('password')
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in host.iteritems():
if key not in ['hostname', 'port', 'username', 'password']:
my_host.set_variable(key, value)
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def gen_inventory(self):
if isinstance(self.resource, list):
self.my_add_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.iteritems():
self.my_add_group(hosts_and_vars.get('hosts'), groupname, hosts_and_vars.get('vars'))
class MyInventory(Inventory):
"""inventory"""
def __init__(self, resource, loader, variable_manager):
self.resource = resource
self.inventory = Inventory(
loader=loader, variable_manager=variable_manager, host_list=[])
self.dynamic_inventory()
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""dynamic inventory group"""
my_group = Group(name=groupname)
if groupvars:
for key, value in groupvars.items():
my_group.set_variable(key, value)
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in host.items():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def dynamic_inventory(self):
if isinstance(self.resource, list):
self.add_dynamic_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.items():
self.add_dynamic_group(
hosts_and_vars.get("hosts"), groupname,
hosts_and_vars.get("vars"))
class MyInventory(Inventory):
def __init__(self, resource, loader, variable_manager):
self.resource = resource
self.inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[])
self.dynamic_inventory()
# logger.info(self.resource)
# logger.info(self.inventory.list_hosts("all"))
def add_dynamic_group(self, hosts, groupname, groupvars=None):
my_group = Group(name=groupname)
if groupvars:
for key, value in groupvars.items():
my_group.set_variable(key, value)
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
# hostport = host.get("port")
# username = host.get("username")
# password = host.get("password")
# ssh_key = host.get("ssh_key")
# my_host = Host(name=hostname, port=hostport)
my_host = Host(name=hostname)
# my_host.set_variable('ansible_ssh_host', hostip)
# my_host.set_variable('ansible_ssh_port', hostport)
# my_host.set_variable('ansible_ssh_user', username)
# my_host.set_variable('ansible_ssh_pass', password)
# my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in host.items():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def dynamic_inventory(self):
if isinstance(self.resource, list):
self.add_dynamic_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.items():
self.add_dynamic_group(hosts_and_vars.get("hosts"), groupname,
hosts_and_vars.get("vars"))
def playbook(**kwargs):
group_vars = kwargs['group_vars']
groups = kwargs['groups']
host_list = kwargs['host_list']
playbook_basedir = os.sep.join([ANSIBLE_PLAYBOOK_PATH, kwargs['playbook_basedir']])
playbooks = []
for pb in kwargs['playbooks']:
playbooks.append(os.sep.join([playbook_basedir, pb]))
job_id = kwargs['job_id']
loader = DataLoader()
vars = VariableManager()
# 指定inventory为一个目录，设置所有主机，包含group和host
invertory = Inventory(loader, vars,
host_list=host_list)
invertory.set_playbook_basedir(playbook_basedir)
for group_name, hosts in groups.items():
t_group = Group(group_name)
for host in hosts:
t_host = Host(host)
t_group.add_host(t_host)
invertory.add_group(t_group)
vars.set_inventory(invertory)
display = LogDisplay(logname=job_id)
callback = CALLBACKMODULE[CALLBACK](display=display)
Options = namedtuple('Options', ['connection', 'module_path', 'forks', 'timeout', 'remote_user',
'ask_pass', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become', 'become_method', 'become_user',
'ask_value_pass', 'verbosity', 'check', 'listhosts',
'listtags', 'listtasks', 'syntax'])
options = Options(connection='smart', module_path='/usr/share/ansible', forks=100, timeout=10,
remote_user='******', ask_pass=False, private_key_file=None, ssh_common_args=None,
ssh_extra_args=None, sftp_extra_args=None, scp_extra_args=None, become=None,
become_method=None, become_user='******', ask_value_pass=False, verbosity=None,
check=False, listhosts=None, listtags=None, listtasks=None, syntax=None)
passwords = dict()
pb_executor = PlaybookExecutor(playbooks, invertory, vars, loader, options, passwords)
pb_executor._tqm._stdout_callback = callback
pb_executor.run()
return display.get_log_json()
def _load(self):
variable_manager = VariableManager()
variable_manager.extra_vars = {}
variable_manager.options_vars = {'ansible_check_mode': False}
loader = DataLoader()
passwords = dict(vault_pass='******')
#display seems not to work yet :-(
display = Display()
display.verbosity = self.options.verbosity
# playbook_executor.display.verbosity = self.options.verbosity
extra_vars = {
'elastic_url': Configuration.elastic_url,
'uuid': self._job.uuid
}
self.variable_manager.extra_vars = extra_vars
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=self.inventoryContainer.allHosts())
for group in self.inventoryContainer.groups:
if group.name == 'all':
if group.vars:
inventory.get_group('all').vars.update(group.vars)
else:
group.parent_groups.append(inventory.get_group('all'))
inventory.add_group(group)
variable_manager.set_inventory(inventory)
playbook = "%s/%s" % (self.pb_dir__, self.playbook__)
pbx = PlaybookExecutor(playbooks=[playbook],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=self.options,
passwords=passwords)
pbx.run()
class NetworkTest(object):
def __init__(self, playbook):
self.inventory = Inventory()
self.playbook = playbook
self.sender_group = Group(name = 'sender')
self.inventory.add_group(self.sender_group)
self.receiver_group = Group(name = 'receiver')
self.inventory.add_group(self.receiver_group)
self.master_group = Group(name = 'master')
self.inventory.add_group(self.master_group)
self.inv_vars = dict()
def set_inventory_vars(self, inv_vars):
self.inv_vars.update(inv_vars)
def add_sender(self, sender):
sender_host = Host(name = sender)
self.sender_group.add_host(sender_host)
def add_receiver(self, receiver):
receiver_host = Host(name = receiver)
self.receiver_group.add_host(receiver_host)
def add_master(self, master):
master_host = Host(name = master)
self.master_group.add_host(master_host)
def run(self):
stats = AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=utils.VERBOSITY)
runner_cb = PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = playbook.PlayBook(playbook = self.playbook,
stats = stats,
callbacks = playbook_cb,
runner_callbacks = runner_cb,
inventory = self.inventory,
extra_vars = self.inv_vars,
check=False)
pr = pb.run()
print json.dumps(pr, sort_keys=True, indent=4, separators=(',', ': '))
class StorageTest(object):
def __init__(self, playbook):
self.inventory = Inventory(host_list=[])
self.playbook = playbook
self.sender_group = Group(name='sender')
self.inventory.add_group(self.sender_group)
self.receiver_group = Group(name='receiver')
self.inventory.add_group(self.receiver_group)
self.master_group = Group(name='master')
self.inventory.add_group(self.master_group)
self.inv_vars = dict()
def set_inventory_vars(self, inv_vars):
self.inv_vars.update(inv_vars)
def add_sender(self, sender):
sender_host = Host(name=sender)
self.sender_group.add_host(sender_host)
def add_receiver(self, receiver):
receiver_host = Host(name=receiver)
self.receiver_group.add_host(receiver_host)
def add_master(self, master):
master_host = Host(name=master)
self.master_group.add_host(master_host)
def run(self):
stats = AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=utils.VERBOSITY)
runner_cb = PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = playbook.PlayBook(playbook=self.playbook,
stats=stats,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
inventory=self.inventory,
extra_vars=self.inv_vars,
check=False)
pr = pb.run()
print json.dumps(pr, sort_keys=True, indent=4, separators=(',', ': '))
class NetworkTest(object):
def __init__(self, playbook):
self.inventory = Inventory(host_list=[])
self.playbook = playbook
self.producer_group = Group(name='producer')
self.inventory.add_group(self.producer_group)
self.consumer_group = Group(name='consumer')
self.inventory.add_group(self.consumer_group)
self.orchestrator_group = Group(name='orchestrator')
self.inventory.add_group(self.orchestrator_group)
self.inv_vars = dict()
def set_inventory_vars(self, inv_vars):
self.inv_vars.update(inv_vars)
def add_producer(self, producer):
producer_host = Host(name=producer)
self.producer_group.add_host(producer_host)
def add_consumer(self, consumer):
consumer_host = Host(name=consumer)
self.consumer_group.add_host(consumer_host)
def add_orchestrator(self, orchestrator):
orchestrator_host = Host(name=orchestrator)
self.orchestrator_group.add_host(orchestrator_host)
def run(self):
stats = AggregateStats()
playbook_cb = PlaybookCallbacks(verbose=utils.VERBOSITY)
runner_cb = PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
pb = playbook.PlayBook(playbook=self.playbook,
stats=stats,
callbacks=playbook_cb,
runner_callbacks=runner_cb,
inventory=self.inventory,
extra_vars=self.inv_vars,
check=False)
pr = pb.run()
print json.dumps(pr, sort_keys=True, indent=4, separators=(',', ': '))
def run(self):
# insert node
for ip in self.hosts:
self._node_map[ip] = Service.new_node(self.task_id, ip)
variable_manager = VariableManager()
Logger.debug("start write ssh_key for task: {} global_id : {}".format(
self.task_id, self.global_id))
key_files = []
group = Group(self.task_id)
for h in self.hosts:
# get ssh_key content
key_content = _get_ssh_key(h)
Logger.debug("read ssh_key for host: {} global_id: {}".format(
h, self.global_id))
# write ssh private key
key_path = _write_ssh_key(h, key_content)
#key_path="./tmp/97"
Logger.debug("write ssh_key for host: {} global_id: {}".format(
h, self.global_id))
host_vars = dict(ansible_port=22,
ansible_user=self.user,
ansible_ssh_private_key_file="./" + key_path)
Logger.debug("key_path: {} global_id: {}".format(
key_path, self.global_id))
key_files.append(key_path)
host = Host(h)
host.vars = host_vars
group.add_host(host)
# add params to each host
if self.params is not None and isinstance(self.params, dict):
for h in group.hosts:
for key in self.params.keys():
variable_manager.set_host_variable(h, key,
self.params[key])
Logger.debug("success write ssh_key for task: {} global_id: {}".format(
self.task_id, self.global_id))
# other options
ssh_args = '-oControlMaster=auto -oControlPersist=60s -oStrictHostKeyChecking=no'
options = _Options(connection='ssh',
module_path='./ansible/library',
forks=self.forks,
timeout=10,
remote_user=None,
private_key_file=None,
ssh_common_args=ssh_args,
ssh_extra_args=None,
sftp_extra_args=None,
scp_extra_args=None,
become=None,
become_method=None,
become_user=None,
verbosity=None,
check=False)
if self.tasktype == "ansible_task":
Logger.debug(
"ansible tasks set******************* global_id: {}".format(
self.global_id))
play_source = dict(name=self.task_id,
hosts=self.task_id,
gather_facts='yes',
tasks=self.tasks)
else:
Logger.debug(
"ansible role set******************* global_id: {}".format(
self.global_id))
play_source = dict(name=self.task_id,
hosts=self.task_id,
gather_facts='yes',
roles=self.tasks)
Logger.debug("start load play for task: {} global_id: {}".format(
self.task_id, self.global_id))
# make playbook
playbook = Play().load(play_source,
variable_manager=variable_manager,
loader=_Loader)
inventory = Inventory(loader=_Loader,
variable_manager=variable_manager)
inventory.add_group(group)
call_back = SyncCallbackModule(debug=True,
step_callback=self._step_callback,
global_id=self.global_id,
source=self.source,
tag_hosts=self.hosts)
Logger.debug("success load play for task: {} global_id: {}".format(
self.task_id, self.global_id))
# task queue
tqm = TaskQueueManager(inventory=inventory,
variable_manager=variable_manager,
loader=_Loader,
options=options,
passwords=None,
stdout_callback=call_back)
try:
back = tqm.run(playbook)
Logger.info("back: {} global_id : {}".format(
str(back), self.global_id))
if back != 0:
raise Exception("playbook run failed")
return back
finally:
if tqm is not None:
tqm.cleanup()
_rm_tmp_key(key_files)
class MyInventory(Inventory):
"""
this is my ansible inventory object.
"""
def __init__(self, resource, loader, variable_manager):
"""
resource的数据格式是一个列表字典，比如
{
"group1": {
"hosts": [{"hostname": "10.0.0.0", "port": "22", "username": "******", "password": "******"}, ...],
"vars": {"var1": value1, "var2": value2, ...}
}
}
如果你只传入1个列表，这默认该列表内的所有主机属于my_group组,比如
[{"hostname": "10.0.0.0", "port": "22", "username": "******", "password": "******"}, ...]
"""
self.resource = resource
self.inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[])
self.gen_inventory()
def my_add_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def gen_inventory(self):
"""
add hosts to inventory.
"""
if isinstance(self.resource, list):
self.my_add_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.iteritems():
self.my_add_group(hosts_and_vars.get("hosts"), groupname,
hosts_and_vars.get("vars"))
class NetworkTest(object):
def __init__(self, playbook):
self.variable_manager = VariableManager()
self.loader = DataLoader()
self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager,host_list=[])
self.variable_manager.set_inventory(self.inventory)
self.playbook = playbook
self.sender_group = Group(name = 'sender')
self.inventory.add_group(self.sender_group)
self.receiver_group = Group(name = 'receiver')
self.inventory.add_group(self.receiver_group)
self.master_group = Group(name = 'master')
self.inventory.add_group(self.master_group)
def set_inventory_vars(self, inv_vars):
self.variable_manager.extra_vars = inv_vars
def add_sender(self, sender):
sender_host = Host(name = sender)
self.sender_group.add_host(sender_host)
def add_receiver(self, receiver):
receiver_host = Host(name = receiver)
self.receiver_group.add_host(receiver_host)
def add_master(self, master):
master_host = Host(name = master)
self.master_group.add_host(master_host)
def run(self):
Options = namedtuple('Options',
['listtags', 'listtasks', 'listhosts',
'syntax', 'connection', 'module_path',
'forks', 'remote_user', 'private_key_file',
'ssh_common_args', 'ssh_extra_args', 'sftp_extra_args',
'scp_extra_args', 'become', 'become_method',
'become_user', 'verbosity', 'check'])
options = Options(listtags=False, listtasks=False, listhosts=False,
syntax=False, connection='ssh', module_path=None,
forks=100, remote_user='******', private_key_file=None,
ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None,
scp_extra_args=None, become=True, become_method=None,
become_user='******', verbosity=None, check=False)
passwords = {}
pbex = PlaybookExecutor(playbooks=[self.playbook],
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=options,
passwords=passwords)
results = pbex.run()
print json.dumps(results, sort_keys=True, indent=4, separators=(',', ': '))
def run(self):
sshpass = None
becomepass = None
b_vault_pass = None
passwords = {}
# initial error check, to make sure all specified playbooks are accessible
# before we start running anything through the playbook executor
for playbook in self.args:
if not os.path.exists(playbook):
raise AnsibleError("the playbook: %s could not be found" %
playbook)
if not (os.path.isfile(playbook)
or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise AnsibleError(
"the playbook: %s does not appear to be a file" % playbook)
# don't deal with privilege escalation or passwords when we don't need to
if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax:
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
loader = DataLoader()
if self.options.vault_password_file:
# read vault_pass from a file
b_vault_pass = CLI.read_vault_password_file(
self.options.vault_password_file, loader=loader)
loader.set_vault_password(b_vault_pass)
elif self.options.ask_vault_pass:
b_vault_pass = self.ask_vault_passwords()
loader.set_vault_password(b_vault_pass)
# create the variable manager, which will be shared throughout
# the code, ensuring a consistent view of global variables
variable_manager = VariableManager()
variable_manager.extra_vars = load_extra_vars(loader=loader,
options=self.options)
# extra_vars = {'elastic_url':Configuration.elastic_url,
# 'uuid':self._job.uuid }
variable_manager.options_vars = load_options_vars(self.options)
# create the inventory, and filter it based on the subset specified (if any)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=self.options.inventory)
# added by [email protected]
inventory.host_list = []
inventory.host_list = self.options.inventory
for group in self.inventoryContainer.groups:
if group.name == 'all':
if group.vars:
inventory.get_group('all').vars.update(group.vars)
else:
group.parent_groups.append(inventory.get_group('all'))
inventory.get_group('all').child_groups.append(group)
# inventory.get_group('all').hosts.extend(group.hosts)
# inventory.get_group('all').vars.update(group.vars)
inventory.add_group(group)
variable_manager.set_inventory(inventory)
# (which is not returned in list_hosts()) is taken into account for
# warning if inventory is empty. But it can't be taken into account for
# checking if limit doesn't match any hosts. Instead we don't worry about
# limit if only implicit localhost was in inventory to start with.
#
# Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts())
no_hosts = False
# if len(inventory.list_hosts()) == 0:
# Empty inventory
# display.warning("provided hosts list is empty, only localhost is available")
# no_hosts = True
inventory.subset(self.options.subset)
# if len(inventory.list_hosts()) == 0 and no_hosts is False:
# Invalid limit
# raise AnsibleError("Specified --limit does not match any hosts")
# flush fact cache if requested
if self.options.flush_cache:
self._flush_cache(inventory, variable_manager)
# create the playbook executor, which manages running the plays via a task queue manager
self.pbex = PlaybookExecutor(playbooks=[playbook],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=self.options,
passwords=passwords)
results = self.pbex.run()
self.apijob.stats = self.pbex._tqm._stats
self.apijob.finished()
if isinstance(results, list):
for p in self.results:
display.display('\nplaybook: %s' % p['playbook'])
for idx, play in enumerate(p['plays']):
if play._included_path is not None:
loader.set_basedir(play._included_path)
else:
pb_dir = os.path.realpath(
os.path.dirname(p['playbook']))
loader.set_basedir(pb_dir)
msg = "\n play #%d (%s): %s" % (idx + 1, ','.join(
play.hosts), play.name)
mytags = set(play.tags)
msg += '\tTAGS: [%s]' % (','.join(mytags))
if self.options.listhosts:
playhosts = set(inventory.get_hosts(play.hosts))
msg += "\n pattern: %s\n hosts (%d):" % (
play.hosts, len(playhosts))
for host in playhosts:
msg += "\n %s" % host
display.display(msg)
all_tags = set()
if self.options.listtags or self.options.listtasks:
taskmsg = ''
if self.options.listtasks:
taskmsg = ' tasks:\n'
def _process_block(b):
taskmsg = ''
for task in b.block:
if isinstance(task, Block):
taskmsg += _process_block(task)
else:
if task.action == 'meta':
continue
all_tags.update(task.tags)
if self.options.listtasks:
cur_tags = list(
mytags.union(set(task.tags)))
cur_tags.sort()
if task.name:
taskmsg += " %s" % task.get_name(
)
else:
taskmsg += " %s" % task.action
taskmsg += "\tTAGS: [%s]\n" % ', '.join(
cur_tags)
return taskmsg
all_vars = variable_manager.get_vars(loader=loader,
play=play)
play_context = PlayContext(play=play,
options=self.options)
for block in play.compile():
block = block.filter_tagged_tasks(
play_context, all_vars)
if not block.has_tasks():
continue
taskmsg += _process_block(block)
if self.options.listtags:
cur_tags = list(mytags.union(all_tags))
cur_tags.sort()
taskmsg += " TASK TAGS: [%s]\n" % ', '.join(
cur_tags)
display.display(taskmsg)
return 0
else:
return results
class MyInventory(Inventory):
"""
this is my ansible inventory object.
"""
def __init__(self, resource):
"""
resource的数据格式是一个列表字典，比如
{
"group1": {
"hosts": [{"hostname": "10.10.10.10", "port": "22", "username": "******", "password": "******"}, ...],
"vars": {"var1": value1, "var2": value2, ...}
}
}
如果你只传入1个列表，这默认该列表内的所有主机属于my_group组,比如
[{"hostname": "10.10.10.10", "port": "22", "username": "******", "password": "******"}, ...]
"""
self.resource = resource
self.inventory = Inventory(host_list=[])
self.gen_inventory()
def my_add_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def gen_inventory(self):
"""
add hosts to inventory.
"""
if isinstance(self.resource, list):
self.my_add_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, hosts_and_vars in self.resource.iteritems():
self.my_add_group(hosts_and_vars.get("hosts"), groupname, hosts_and_vars.get("vars"))
def runInv(request, **kwargs):
# Constants
C.HOST_KEY_CHECKING = False
template = JobTemplates.objects.get(pk=kwargs['pk'])
hostobjects = Hosts.objects.filter(inventory=template.inventory.pk)
inventoryName = template.inventory.name
inventory = Inventory()
# TODO: need to implement groups functionality
#Groups
#group = Group(inventoryName)
group = Group('mygroup')
for hostobj in hostobjects:
# Host defining
hostname = hostobj.name
port = hostobj.port
ip = hostobj.ipAddress
username = hostobj.username
password = hostobj.password
host = Host(hostname)
if ip:
host.set_variable('ansible_ssh_host', ip)
elif port:
host.set_variable('ansible_ssh_port', port)
host.set_variable('ansible_ssh_user', username)
host.set_variable('ansible_ssh_pass', password)
# Hostvars
hostvars = yaml.load(hostobj.variables)
if hostvars:
host.vars.update(hostvars)
group.add_host(host)
inventory.add_group(group)
# ExtraVars
extraVars = yaml.load(template.extra_variables)
# Callbacks
utils.VERBOSITY = 3
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
stats = callbacks.AggregateStats()
runner_cb = callbacks.PlaybookRunnerCallbacks(stats, verbose=utils.VERBOSITY)
#runner_cb = myCustCallback(stats, template.pk, verbose=utils.VERBOSITY)
# Playbook gathering
playbookPath = os.path.join(template.project.directory, template.playbook)
# Playbook run
pb = PlayBook(
playbook=playbookPath,
host_list=None,
inventory=inventory, # Our hosts, the rendered inventory file
#remote_user='******',
#remote_pass='******',
callbacks=playbook_cb,
runner_callbacks=runner_cb,
stats=stats,
extra_vars=extraVars,
#private_key_file='/path/to/key.pem'
)
pb.run()
return HttpResponseRedirect(reverse('jobtemplates:index'))
class ResourceBase(object):
"""
gen_inventory methods.
Attributes:
resource: the inventory's resource. format:
{
"group1": {
"hosts": [{"hostname": "10.10.10.10", "port": "22", "username": "******", "password": "******"}, ...],
"vars": {"var1": value1, "var2": value2, ...}
}
}
if your pass a list, the list will add the default group(default_group)
[{"hostname": "10.10.10.10", "port": "22", "username": "******", "password": "******"}, ...]
inventory: ansible inventory object use gen_inventory to generate.
Methods:
gen_inventory: generate a ansible inventory object.
"""
def __init__(self, resource):
self.inventory = Inventory(host_list=[])
self.resource = resource
self.gen_inventory()
@staticmethod
def add_group_vars(group, group_vars=None):
"""
if group_vars exists then, add group variable to group
Args:
group: <ansible group object> ansible group object
group_vars: <dict> group variables
"""
assert isinstance(group, Group), "the group must be an ansible group object."
if group_vars:
for key, value in group_vars.iteritems():
group.set_variable(key, value)
@staticmethod
def gen_hosts(hosts=None):
"""
if host_vars exists then, generate hosts
Args:
hosts: <list> [<host variable dict>, <host variable dict>, ...]
Returns:
host_objs: <list> [<host object>, <host object>, ...]
"""
assert isinstance(hosts, list), "the hosts must be a list"
host_objs = []
if hosts:
for host in hosts:
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
ssh_key = host.get("ssh_key")
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
if password:
my_host.set_variable('ansible_ssh_pass', password)
if ssh_key:
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password", "ip", "ssh_key"]:
my_host.set_variable(key, value)
host_objs.append(my_host)
return host_objs
def my_add_group(self, hosts_vars, group_name, group_vars=None):
"""
add hosts to group. use to generate a inventory.
Args:
hosts_vars: the hosts variables
group_name: group name
group_vars: group variables
"""
my_group = Group(name=group_name)
self.add_group_vars(my_group, group_vars)
for host in self.gen_hosts(hosts_vars):
my_group.add_host(host)
self.inventory.add_group(my_group)
def gen_inventory(self):
"""
add hosts to an inventory.
"""
if isinstance(self.resource, list):
self.my_add_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for group_name, hosts_vars in self.resource.iteritems():
self.my_add_group(hosts_vars.get("hosts"), group_name, hosts_vars.get("vars"))
def playbook(**kwargs):
group_vars = kwargs['group_vars']
groups = kwargs['groups']
host_list = kwargs['host_list']
playbook_basedir = os.sep.join(
[ANSIBLE_PLAYBOOK_PATH, kwargs['playbook_basedir']])
playbooks = []
for pb in kwargs['playbooks']:
playbooks.append(os.sep.join([playbook_basedir, pb]))
job_id = kwargs['job_id']
loader = DataLoader()
vars = VariableManager()
# 指定inventory为一个目录，设置所有主机，包含group和host
invertory = Inventory(loader, vars, host_list=host_list)
invertory.set_playbook_basedir(playbook_basedir)
for group_name, hosts in groups.items():
t_group = Group(group_name)
for host in hosts:
t_host = Host(host)
t_group.add_host(t_host)
invertory.add_group(t_group)
vars.set_inventory(invertory)
display = LogDisplay(logname=job_id)
callback = CALLBACKMODULE[CALLBACK](display=display)
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'timeout', 'remote_user',
'ask_pass', 'private_key_file', 'ssh_common_args', 'ssh_extra_args',
'sftp_extra_args', 'scp_extra_args', 'become', 'become_method',
'become_user', 'ask_value_pass', 'verbosity', 'check', 'listhosts',
'listtags', 'listtasks', 'syntax'
])
options = Options(connection='smart',
module_path='/usr/share/ansible',
forks=100,
timeout=10,
remote_user='******',
ask_pass=False,
private_key_file=None,
ssh_common_args=None,
ssh_extra_args=None,
sftp_extra_args=None,
scp_extra_args=None,
become=None,
become_method=None,
become_user='******',
ask_value_pass=False,
verbosity=None,
check=False,
listhosts=None,
listtags=None,
listtasks=None,
syntax=None)
passwords = dict()
pb_executor = PlaybookExecutor(playbooks, invertory, vars, loader, options,
passwords)
pb_executor._tqm._stdout_callback = callback
pb_executor.run()
return display.get_log_json()
def handle():
host1 = Host("127.0.0.1")
host1.vars = dict(ansible_port=22,
ansible_user="******",
ansible_ssh_private_key_file="../tmp/127.0.0.1")
# none variable
variable_manager = VariableManager()
g = Group("group_wb")
g.add_host(host1)
# target ip list
inventory = Inventory(loader=loader, variable_manager=variable_manager)
inventory.add_group(g)
# other options
options = Options(listtags=False, listtasks=False, listhosts=False, syntax=False, connection='ssh',
module_path=None, forks=1,
remote_user='******', ssh_common_args=None,
ssh_extra_args=None,
sftp_extra_args=None, scp_extra_args=None, become=False, become_method=None,
become_user="******",
verbosity=None, check=False)
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None
)
# test tasks
task0 = dict(action=dict(module='shell', args='sleep 6'))
task1 = dict(action=dict(module='shell', args='ls'))
task2 = dict(action=dict(module='shell', args='echo "2333"'))
play_source = dict(
name="Test Play",
hosts="group_wb",
gather_facts='no',
tasks=[
task0, task1, task2
]
)
play = Play().load(play_source, variable_manager=variable_manager, loader=loader)
try:
result = tqm.run(play)
print("Result code: " + str(result))
print("Type: " + str(type(result)))
# close queue manager
finally:
if tqm is not None:
tqm.cleanup()
class ResourceBase(object):
"""
generate inventory
:param resource: inventory resource, format:
{
"hosts" : {
"host1": {"port": "22", "username": "******", "password": "******"},
"host2": {"port": "22", "username": "******", "password": "******"},
},
"groups": {
"group1": {"hosts": ["host1", "host2",...], "vars": {'var1':'xxxx', 'var2':'yyy',...} },
"group2": {"hosts": ["host1", "host2",...], "child": ["group1"], "vars": {'var1':'xxxx', 'var2':'yyy',...} },
}
}
"""
def __init__(self, resource=None):
host_list = not resource and ANS_CONS.DEFAULT_HOST_LIST or []
self.loader = DataLoader()
self.variable_manager = VariableManager()
self.resource = resource
self.inventory = Inventory(loader=self.loader,
variable_manager=self.variable_manager,
host_list=host_list)
#resource and self.gen_inventory()
self.gen_inventory()
@staticmethod
def gen_host(host_name=None, host_vars=None):
"""
Generate ansible Host object
:param host_name: <string> ansible inventory hostname
:param host_vars: <dict> host variables
:return: Host object
"""
if host_vars is None:
host_vars = {}
ssh_host = host_vars.get('ip', host_name)
ssh_port = host_vars.get('port', ANS_CONS.DEFAULT_REMOTE_PORT)
ssh_user = host_vars.get('username')
ssh_pass = host_vars.get('password')
ssh_fkey = host_vars.get('ssh_key')
# init Host
host = Host(name=host_name, port=ssh_port)
host.set_variable('ansible_ssh_host', ssh_host)
# shortcut variables
ssh_user and host.set_variable('ansible_ssh_user', ssh_user)
ssh_pass and host.set_variable('ansible_ssh_pass', ssh_pass)
ssh_fkey and host.set_variable('ansible_private_key_file', ssh_fkey)
# extra variables
for key, value in host_vars.items():
if key not in ['ip', 'port', 'username', 'password', 'ssh_key']:
host.set_variable(key, value)
# return Host object
return host
@staticmethod
def gen_group(group_name=None, group_vars=None):
"""
Generate ansible Group object
:param group_name: <string> Group Name
:param group_vars: <dict> Group Variables
:return: ansible Group object
"""
if group_vars is None:
group_vars = {}
group = Group(name=group_name)
for key, value in group_vars.items():
group.set_variable(key, value)
return group
def gen_inventory(self):
"""
:return: None
"""
# set hosts
if 'hosts' in self.resource.keys():
for host, info in self.resource['hosts'].items():
obj_host = self.gen_host(host, info)
self.inventory.get_group('all').add_host(obj_host)
# add group
if 'groups' in self.resource.keys():
for group, detail in self.resource['groups'].items():
obj_group = self.gen_group(group, detail.get('vars', {}))
for host in detail.get('hosts', []):
obj_group.add_host(self.inventory.get_host(host))
if 'child' in detail.get('child', []):
for child in detail.get('child', []):
obj_group.add_child_group(
self.inventory.get_group(child))
self.inventory.add_group(obj_group)
def get_lists(self):
print("Host: ")
print("=================")
for host in self.inventory.list_hosts():
print(host)
print("Group: ")
print("=================")
for group in self.inventory.list_groups():
print(group)
# -*-coding:utf-8 -*-
class Runner(object):
def __init__(self, groups, playbook, private_key_file, display, become_pass={}, extraVars={}):
self.extraVars = extraVars
self.options = Options()
self.options.private_key_file = private_key_file
self.options.verbosity = display.verbosity
self.options.connection = 'ssh' # Need a connection type 'smart' or 'ssh'
self.options.become = True
self.options.become_method = 'sudo'
self.options.become_user = '******'
# Executor appears to have it's own
# verbosity object/setting as well
playbook_executor.verbosity = self.options.verbosity
# Become Pass Needed if not logging in as user root
passwords = {'become_pass':become_pass}
# Gets data from YAML/JSON files
self.loader = DataLoader()
self.loader.set_vault_password(os.environ.get('VAULT_PASS', 'xiaoy_pandan'))
# All the variables from all the various places
self.variable_manager = VariableManager()
self.variable_manager.extra_vars = self.extraVars
# Set inventory, using most of above objects
self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=[])
self.variable_manager.set_inventory(self.inventory)
# Set groups info
for info in groups:
group = Group(info['name'])
self.inventory.add_group(group)
hostInfos = info.get('hosts')
for hostInfo in hostInfos:
hostname = hostInfo.get('hostname')
hostip = hostInfo.get('ip', hostname)
hostport = hostInfo.get('port')
username = hostInfo.get('username')
password = hostInfo.get('password')
ssh_key = hostInfo.get('ssh_key')
host = Host(name=hostname, port=hostport)
host.set_variable('ansible_ssh_host', hostip)
host.set_variable('ansible_ssh_port', hostport)
host.set_variable('ansible_ssh_user', username)
host.set_variable('ansible_ssh_pass', password)
host.set_variable('ansible_ssh_private_key_file', ssh_key)
for key, value in hostInfo.iteritems():
if key not in ['hostname', 'port', 'username', 'password', 'ssh_key']:
host.set_variable(key, value)
group.add_host(host)
varInfos = info.get('vars')
if varInfos:
for key, value in varInfos.iteritems():
group.set_variable(key, value)
# Playbook to run. Assumes it is
# local to this python file
pb_dir = os.path.dirname(__file__)
playbook = '%s/%s' % (pb_dir, playbook)
# Setup playbook executor, but don't run until run() called
self.pbex = playbook_executor.PlaybookExecutor(
playbooks=[playbook],
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords=passwords)
self.display = display
self.callback = CallbackModule(self.display)
self.pbex._tqm._stdout_callback = self.callback
def run(self):
# Results of PlaybookExecutor
self.pbex.run()
# calc result
result = True
hostResults = {}
stats = self.pbex._tqm._stats
hosts = sorted(stats.processed.keys())
for h in hosts:
t = stats.summarize(h)
if t['unreachable'] > 0 or t['failures'] > 0:
hostResult = result = False
else:
hostResult = True
hostResults[h] = hostResult
if result:
self.display.display('success')
else:
self.display.display('fail', stderr=True)
self.display.display('')
return result, hostResults
def run_ansible_on_host(host, logger, config):
from ansible.plugins.callback import CallbackBase
# A rough logger that logs dict messages to standard logger
class ResultCallback(CallbackBase):
def __init__(self):
super(ResultCallback, self).__init__()
def v2_runner_on_ok(self, result, **kwargs):
self.log('ok :' + str(result._result), info=True)
def v2_runner_on_failed(self, result, **kwargs):
self.log(result._result, info=True)
def v2_runner_on_skipped(self, result, **kwargs):
self.log(result._result, info=True)
def v2_runner_on_unreachable(self, result, **kwargs):
self.log(result._result, info=True)
def v2_playbook_on_no_hosts_matched(self, *args, **kwargs):
self.log('no hosts matched!')
def v2_playbook_on_no_hosts_remaining(self, *args, **kwargs):
self.log('NO MORE HOSTS LEFT')
def v2_playbook_on_task_start(self, task, **kwargs):
self.log('starting task: ' + str(task))
def v2_playbook_on_start(self, playbook, **kwargs):
self.log('starting playbook' + str(playbook), info=True)
def v2_playbook_on_play_start(self, play, **kwargs):
self.log('starting play' + str(play), info=True)
def v2_playbook_on_stats(self, stats, info=True, **kwargs):
self.log('STATS FOR PLAY')
hosts = sorted(stats.processed.keys())
hosts.extend(stats.failures.keys())
hosts.extend(stats.dark.keys())
hosts.extend(stats.changed.keys())
hosts.extend(stats.skipped.keys())
hosts.extend(stats.ok.keys())
for h in hosts:
t = stats.summarize(h)
self.log(str(t))
def log(self, param, info=False):
if not info:
logger.debug(str(param))
else:
logger.info(str(param))
from ansible.parsing.dataloader import DataLoader
from ansible.inventory import Inventory, Group, Host
from ansible.executor import playbook_executor
from ansible.vars import VariableManager
from collections import namedtuple
Options = namedtuple('Options', [
'connection', 'module_path', 'forks', 'become', 'become_method',
'become_user', 'check', 'ansible_user', 'listhosts', 'listtasks',
'listtags', 'syntax', 'ssh_private_key_file', 'host_key_checking'
])
options = Options(connection='ssh',
become=True,
become_method='sudo',
become_user='******',
check=False,
module_path=None,
forks=100,
ansible_user='******',
listhosts=False,
listtasks=False,
listtags=False,
syntax=False,
ssh_private_key_file=PEBBLES_SSH_KEY_LOCATION,
host_key_checking=False)
variable_manager = VariableManager()
loader = DataLoader()
a_host = Host(name=host['private_ip'])
a_host.set_variable('ansible_host', host['private_ip'])
a_group = Group(name='notebook_host')
a_group.add_host(a_host)
inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[host['private_ip']])
inventory.add_group(a_group)
variable_manager.set_inventory(inventory)
logger.debug('HOST:')
logger.debug(a_host.serialize())
logger.debug('HOSTs from inventory:')
# for some reason setting these before adding the host to inventory didn't work so well
# ToDo: read up on variable_manager and figure out a more elegant way to set the variables
for h_ in inventory.get_hosts():
h_.set_variable('ansible_user', 'cloud-user')
h_.set_variable('ansible_ssh_common_args',
'-o StrictHostKeyChecking=no')
h_.set_variable('ansible_ssh_private_key_file',
'/home/pebbles/.ssh/id_rsa')
extra_vars = dict()
extra_vars['ansible_ssh_extra_args'] = '-o StrictHostKeyChecking=no'
if 'DD_HOST_DATA_VOLUME_DEVICE' in config:
extra_vars['notebook_host_block_dev_path'] = config[
'DD_HOST_DATA_VOLUME_DEVICE']
variable_manager.extra_vars = extra_vars
pb_executor = playbook_executor.PlaybookExecutor(
playbooks=[
'/webapps/pebbles/source/ansible/notebook_playbook.yml'
],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None)
rescb = ResultCallback()
pb_executor._tqm._stdout_callback = rescb
logger.info('_prepare_host(): running ansible')
logger.info('_prepare_host(): inventory hosts')
for h_ in inventory.get_hosts():
logger.info(h_.serialize())
logger.info(h_.get_vars())
pb_executor.run()
stats = pb_executor._tqm._stats
run_success = True
hosts_list = sorted(stats.processed.keys())
if len(hosts_list) == 0:
logger.debug('no hosts handled')
for h in hosts_list:
t = stats.summarize(h)
logger.debug(t)
logger.debug(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
if run_success:
logger.debug('_prepare_host(): run successfull')
else:
logger.debug('_prepare_host(): run failed')
if getattr(pb_executor, '_unreachable_hosts', False):
logger.debug('UNREACHABLE HOSTS ' +
str(pb_executor._unreachable_hosts))
if getattr(pb_executor, '_failed_hosts', False):
logger.debug('FAILED_HOSTS ' + str(pb_executor._failed_hosts))
raise RuntimeError('run_ansible_on_host(%s) failed' % host['id'])
logger.debug('_prepare_host(): done running ansible')
class AnsibleInventoryManager():
"""Class to handle the ansible inventory file.
This class will allow you to add groups/hosts and remove them from the inventory file"""
#FROM http://www.ansibleworks.com/docs/patterns.html
ALLOWED_VARIABLES = ['ansible_ssh_host',
'ansible_ssh_port',
'ansible_ssh_user',
'ansible_ssh_pass',
'ansible_connection',
'ansible_ssh_private_key_file',
'ansible_syslog_facility',
'ansible_python_interpreter',
]
# NOTE:Works for anything such as ruby or perl and works just like
# ansible_python_interpreter.
# This replaces shebang of modules which will run on that host.
def __init__(self, inventory_file="/etc/ansible/hosts"):
self.__inventory_file = inventory_file
self.__dirty = False
if not os.path.exists(self.__inventory_file):
raise AnsibleInventoryManagerFileNotFound("File: %s Not found or not accessible" % self.__inventory_file)
self.__inventory = Inventory(inventory_file)
def get_hosts(self):
"""return the list of hosts
Returns a list host ips"""
host_list = [host.name for host in self.__inventory.get_hosts()]
return host_list
def get_groups(self):
"""return the groups
Returns a list of objects: ansible.inventory.group.Group"""
return self.__inventory.get_groups()
def get_groups_for_host(self,host):
"""return the groups list where the given host appears"""
return self.__inventory.groups_for_host(host)
def get_group(self, groupname):
"""Returns the given group"""
return self.__inventory.get_group(groupname)
def delete_host(self,host_ip,group=""):
"""Removes a host from a given group
if group is empty, removes the host from all the groups
"""
self.__dirty = True
groups = []
if group == "":
groups = [group.name for group in self.get_groups_for_host(host_ip)]
else:
groups.append(group)
for group in groups:
grp = self.__inventory.get_group(group)
new_host_list = [host for host in grp.get_hosts() if host.name != host_ip]
grp.hosts = new_host_list
def add_host(self, host_ip, add_to_root=True, group_list=[], var_list={}):
"""Add a host ip to the ansible host file
This is a simple function to add hosts to
add_to_root = Adds the host to the root group (unnamed)
groups_list: List of groupnames where the host should appears.
var_list: Variable list. see allowed_variables."""
#root group in unnamed, but in the inventory object
# is the 'ungrouped' group
self.__dirty = True
new_host = Host(host_ip)
for key,value in var_list.iteritems():
if self.is_allowed_variable(key):
new_host.set_variable(key,value)
if add_to_root:
if 'ungrouped' not in group_list:
group_list.append('ungrouped')
#Check groups. The ansible inventory should contain each of the groups.
for group in group_list:
if not self.__inventory.get_group(group):
new_group= Group(group)
self.__inventory.add_group(new_group)
grp = self.__inventory.get_group(group)
host_names = [host.name for host in grp.get_hosts()]
if new_host.name not in host_names:
grp.add_host(new_host)
def is_dirty(self):
return self.__dirty
def is_allowed_variable(self, variable):
"""Checks if the given variable is an allowed variable"""
if variable in self.ALLOWED_VARIABLES:
return True
elif re.match("ansible_(.+)_interpreter", variable):
return True
return False
def save_inventory(self, backup_file=""):
"""Saves the inventory file. If a backup_file is given,
a backup will be done before re-write the file"""
try:
if backup_file != "":
copyfile(self.__inventory_file, backup_file)
data = ""
for group in self.__inventory.get_groups():
ingroup = False
if group.name == "all":
continue
elif group.name != "ungrouped":
data += "[%s]\n" % group.name
ingroup = True
strvars = ""
for host in group.get_hosts():
for key, value in host.get_variables().iteritems():
if key in AnsibleInventoryManager.ALLOWED_VARIABLES:
strvars += "%s=%s " % (key, value)
if ingroup:
data += "\t%s\t%s\n" % (host.name, strvars)
else:
data += "%s\t%s\n" % (host.name, strvars)
ansiblehostfile = open(self.__inventory_file, "w")
ansiblehostfile.write(data)
ansiblehostfile.close()
except Exception, e:
error("Error doing the %s backup: %s" % (self.__inventory_file, str(e)))
class MyInventory(Inventory):
"""
this is my ansible inventory object.
"""
def __init__(self, resource, loader, variable_manager):
"""
ansible inventory object
:param resource: 一个列表字典，如
{
"group1": {
"hosts": [{"hostname": "10.0.0.0", "port": "22", "username": "******", "password": "******"}, ...],
"vars": {"var1": value1, "var2": value2, ...}
}
}
如果只传入1个列表，这默认该列表内的所有主机属于 my_group 组,比如
[{"hostname": "10.0.0.0", "port": "22", "username": "******", "password": "******"}, ...]
:param loader:
:param variable_manager:
"""
self.resource = resource
self.inventory = Inventory(loader=loader,
variable_manager=variable_manager,
host_list=[])
self.dynamic_inventory()
def add_dynamic_group(self, hosts, groupname, groupvars=None):
"""
add hosts to a group
:param hosts:
:param groupname:
:param groupvars:
:return:
"""
my_group = Group(name=groupname)
# if group variables exists, add them to group
if groupvars:
for key, value in groupvars.iteritems():
my_group.set_variable(key, value)
# add hosts to group
for host in hosts:
# set connection variables
hostname = host.get("hostname")
hostip = host.get('ip', hostname)
hostport = host.get("port")
username = host.get("username")
password = host.get("password")
if username == 'root':
keyfile = "/root/.ssh/id_rsa"
else:
keyfile = "/home/{user}/.ssh/id_rsa".format(user=username)
ssh_key = host.get("ssh_key", keyfile)
my_host = Host(name=hostname, port=hostport)
my_host.set_variable('ansible_ssh_host', hostip)
my_host.set_variable('ansible_ssh_port', hostport)
my_host.set_variable('ansible_ssh_user', username)
my_host.set_variable('ansible_ssh_pass', password)
my_host.set_variable('ansible_ssh_private_key_file', ssh_key)
# set other variables
for key, value in host.iteritems():
if key not in ["hostname", "port", "username", "password"]:
my_host.set_variable(key, value)
# add to group
my_group.add_host(my_host)
self.inventory.add_group(my_group)
def dynamic_inventory(self):
"""
add hosts to inventory.
:return:
"""
if isinstance(self.resource, list):
self.add_dynamic_group(self.resource, 'default_group')
elif isinstance(self.resource, dict):
for groupname, host_and_vars in self.resource.iteritems():
self.add_dynamic_group(host_and_vars.get("hosts"), groupname,
host_and_vars.get("vars"))
def run_ansible_on_host(host, logger, driver_config):
from ansible.plugins.callback import CallbackBase
# A rough logger that logs dict messages to standard logger
class ResultCallback(CallbackBase):
def __init__(self):
super(ResultCallback, self).__init__()
def v2_runner_on_ok(self, result, **kwargs):
self.log('ok :' + str(result._result))
def v2_runner_on_failed(self, result, **kwargs):
warnings = result._result['warnings']
error = result._result['stderr']
if warnings:
self.log('warning : ' + str(result._result))
elif error:
self.log('error : ' + str(result._result), info=True)
def v2_runner_on_skipped(self, result, **kwargs):
self.log('skipped : ' + str(result._result))
def v2_runner_on_unreachable(self, result, **kwargs):
self.log('unreachable : ' + str(result._result), info=True)
def v2_playbook_on_no_hosts_matched(self, *args, **kwargs):
self.log('no hosts matched!')
def v2_playbook_on_no_hosts_remaining(self, *args, **kwargs):
self.log('NO MORE HOSTS LEFT')
def v2_playbook_on_task_start(self, task, **kwargs):
self.log('starting task: ' + str(task))
def v2_playbook_on_start(self, playbook, **kwargs):
self.log('starting playbook' + str(playbook), info=True)
def v2_playbook_on_play_start(self, play, **kwargs):
self.log('starting play' + str(play), info=True)
def v2_playbook_on_stats(self, stats, info=True, **kwargs):
self.log('STATS FOR PLAY')
hosts = sorted(stats.processed.keys())
hosts.extend(stats.failures.keys())
hosts.extend(stats.dark.keys())
hosts.extend(stats.changed.keys())
hosts.extend(stats.skipped.keys())
hosts.extend(stats.ok.keys())
for h in hosts:
t = stats.summarize(h)
self.log(str(t))
def log(self, param, info=False):
if not info:
logger.debug(str(param))
else:
logger.info(str(param))
from ansible.parsing.dataloader import DataLoader
from ansible.inventory import Inventory, Group, Host
from ansible.executor import playbook_executor
from ansible.vars import VariableManager
from collections import namedtuple
Options = namedtuple(
'Options',
[
'connection',
'module_path',
'forks',
'become',
'become_method',
'become_user',
'check',
'ansible_user',
'listhosts',
'listtasks',
'listtags',
'syntax',
'ssh_private_key_file',
'host_key_checking'
]
)
options = Options(
connection='ssh',
become=True,
become_method='sudo',
become_user='******',
check=False,
module_path=None,
forks=100,
ansible_user='******',
listhosts=False,
listtasks=False,
listtags=False,
syntax=False,
ssh_private_key_file=PEBBLES_SSH_KEY_LOCATION,
host_key_checking=False
)
variable_manager = VariableManager()
loader = DataLoader()
a_host = Host(name=host['private_ip'])
a_host.set_variable('ansible_host', host['private_ip'])
a_group = Group(name='notebook_host')
a_group.add_host(a_host)
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=[host['private_ip']])
inventory.add_group(a_group)
variable_manager.set_inventory(inventory)
logger.debug('HOST:')
logger.debug(a_host.serialize() )
logger.debug('HOSTs from inventory:')
# for some reason setting these before adding the host to inventory didn't work so well
# ToDo: read up on variable_manager and figure out a more elegant way to set the variables
for h_ in inventory.get_hosts():
h_.set_variable('ansible_user', 'cloud-user')
h_.set_variable('ansible_ssh_common_args', '-o StrictHostKeyChecking=no')
h_.set_variable('ansible_ssh_private_key_file', '/home/pebbles/.ssh/id_rsa')
extra_vars = dict()
extra_vars['ansible_ssh_extra_args'] = '-o StrictHostKeyChecking=no'
logger.debug('Setting driver config....')
if 'DD_HOST_DATA_VOLUME_DEVICE' in driver_config:
extra_vars['notebook_host_block_dev_path'] = driver_config['DD_HOST_DATA_VOLUME_DEVICE']
variable_manager.extra_vars = extra_vars
pb_executor = playbook_executor.PlaybookExecutor(
playbooks=['/webapps/pebbles/source/ansible/notebook_playbook.yml'],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=None
)
rescb = ResultCallback()
pb_executor._tqm._stdout_callback = rescb
logger.info('_prepare_host(): running ansible')
logger.info('_prepare_host(): inventory hosts')
for h_ in inventory.get_hosts():
logger.info(h_.serialize())
logger.info(h_.get_vars())
pb_executor.run()
stats = pb_executor._tqm._stats
run_success = True
hosts_list = sorted(stats.processed.keys())
if len(hosts_list) == 0:
logger.debug('no hosts handled')
for h in hosts_list:
t = stats.summarize(h)
logger.debug(t)
logger.debug(h)
if t['unreachable'] > 0 or t['failures'] > 0:
run_success = False
if run_success:
logger.debug('_prepare_host(): run successfull')
else:
logger.debug('_prepare_host(): run failed')
if getattr(pb_executor, '_unreachable_hosts', False):
logger.debug('UNREACHABLE HOSTS ' + str(pb_executor._unreachable_hosts))
if getattr(pb_executor, '_failed_hosts', False):
logger.debug('FAILED_HOSTS ' + str(pb_executor._failed_hosts))
raise RuntimeError('run_ansible_on_host(%s) failed' % host['id'])
logger.debug('_prepare_host(): done running ansible')