def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
# first, we use the original parent method to correctly load the object
# via the munge/load_data system we normally use for other playbook objects
new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = new_obj.include
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)
# finally, playbook includes can specify a list of variables, which are simply
# used to update the vars of each play in the playbook
for entry in pb._entries:
entry.vars.update(new_obj.vars)
return pb
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = new_obj.include
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
entry.vars.update(new_obj.vars)
entry.tags = list(set(entry.tags).union(new_obj.tags))
return pb
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)
all_vars = self.vars.copy()
if variable_manager:
all_vars.update(variable_manager.get_vars(loader=loader))
templar = Templar(loader=loader, variables=all_vars)
try:
forward_conditional = False
if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
return None
except AnsibleError:
# conditional evaluation raised an error, so we set a flag to indicate
# we need to forward the conditionals on to the included play(s)
forward_conditional = True
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = templar.template(new_obj.include)
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(file_name)
# Check to see if we need to forward the conditionals on to the included
# plays. If so, we can take a shortcut here and simply prepend them to
# those attached to each block (if any)
if forward_conditional:
for task_block in entry.pre_tasks + entry.roles + entry.tasks + entry.post_tasks:
task_block.when = self.when[:] + task_block.when
return pb
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
from ansible.playbook.play import Play
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)
all_vars = self.vars.copy()
if variable_manager:
all_vars.update(variable_manager.get_vars())
templar = Templar(loader=loader, variables=all_vars)
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = templar.template(new_obj.import_playbook)
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
# conditional includes on a playbook need a marker to skip gathering
if new_obj.when and isinstance(entry, Play):
entry._included_conditional = new_obj.when[:]
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(file_name)
# Check to see if we need to forward the conditionals on to the included
# plays. If so, we can take a shortcut here and simply prepend them to
# those attached to each block (if any)
if new_obj.when:
for task_block in (entry.pre_tasks + entry.roles + entry.tasks + entry.post_tasks):
task_block._attributes['when'] = new_obj.when[:] + task_block.when[:]
return pb
def test_playbook_executor__get_serialized_batches(self):
fake_loader = DictDataLoader({
'no_serial.yml': '''
- hosts: all
gather_facts: no
tasks:
- debug: var=inventory_hostname
''',
'serial_int.yml': '''
- hosts: all
gather_facts: no
serial: 2
tasks:
- debug: var=inventory_hostname
''',
'serial_pct.yml': '''
- hosts: all
gather_facts: no
serial: 20%
tasks:
- debug: var=inventory_hostname
''',
})
mock_inventory = MagicMock()
mock_var_manager = MagicMock()
# fake out options to use the syntax CLI switch, which will ensure
# the PlaybookExecutor doesn't create a TaskQueueManager
mock_options = MagicMock()
mock_options.syntax.value = True
pbe = PlaybookExecutor(
playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml'],
inventory=mock_inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
options=mock_options,
passwords=[],
)
playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']])
playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = ['host0','host1','host2','host3','host4','host5','host6','host7','host8','host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0','host1'],['host2','host3'],['host4','host5'],['host6','host7'],['host8','host9']])
def create_playbook(playbook_path):
"""
Load the given playbook and return the ansible playbook.
:param playbook_path: Path to the playbook file.
:type playbook_path: :class:`str`
:returns: The loaded ansible playbook.
:rtype: :class:`ansible.playbook.Playbook`
"""
try:
playbook = Playbook.load(
playbook_path,
loader=LOADER,
variable_manager=VARIABLE_MANAGER,
)
scrape_playbook(playbook)
return playbook
except Exception as error: # pylint: disable=broad-except
logging.exception(
"Unexpected error scrapping playbook %r: %r",
playbook_path, error
)
raise argparse.ArgumentTypeError(error)
def test_basic_playbook(self):
fake_loader = DictDataLoader({
"test_file.yml":"""
- hosts: all
""",
})
p = Playbook.load("test_file.yml", loader=fake_loader)
plays = p.get_plays()
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude, self).load_data(ds, variable_manager, loader)
all_vars = dict()
if variable_manager:
all_vars = variable_manager.get_vars(loader=loader)
templar = Templar(loader=loader, variables=all_vars)
if not new_obj.evaluate_conditional(templar=templar, all_vars=all_vars):
return None
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = new_obj.include
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name, variable_manager=variable_manager)
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(file_name)
return pb
def test_play_iterator(self):
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
gather_facts: false
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml': """
- debug: msg="this is a role task"
""",
})
p = Playbook.load('test_play.yml', loader=fake_loader)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
connection_info = ConnectionInformation(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
connection_info=connection_info,
all_vars=dict(),
)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNone(task)
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
signal.signal(signal.SIGINT, self._cleanup)
result = 0
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
# FIXME: playbook entries are just plays, so we should rename them
for play in pb.get_entries():
self._inventory.remove_restriction()
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
new_play = play.copy()
new_play.post_validate(all_vars, fail_on_undefined=False)
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm._callback.playbook_on_play_start(new_play.name)
self._tqm._callback.playbook_on_no_hosts_matched()
result = 0
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
if result != 0:
break
if result != 0:
# FIXME: do something here, to signify the playbook execution failed
self._cleanup()
return result
except:
self._cleanup()
raise
self._cleanup()
return result
def test_playbook_iterator(self):
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml': """
- debug: msg="this is a role task"
""",
})
p = Playbook.load('test_play.yml', loader=fake_loader)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
itr = PlaybookIterator(inventory, None, p)
task = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
task = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
task = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
task = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNotNone(task)
task = itr.get_next_task_for_host(hosts[0])
print(task)
self.assertIsNone(task)
def build_plays(loader, variable_manager, playbook_path, plays=[], hosts=[]):
playbook = Playbook.load(playbook_path, variable_manager, loader)
plays = []
for play in playbook.get_plays():
if play.get_name() in plays:
plays.append(play)
continue
if play._ds['hosts'] in hosts:
plays.append(play)
continue
for piece in play._ds['hosts']:
if piece in hosts:
plays.append(play)
break
return plays
def run(self, playbook):
pb = Playbook.load(playbook, variable_manager=self.variable_manager, loader=self.loader)
# only support one playbook.yml
play = pb.get_plays()[0]
tqm = None
try:
tqm = TaskQueueManager(
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords={'conn_pass': None, 'become_pass': None},
stdout_callback=self.callback,
)
return tqm.run(play), self.callback.result
finally:
if tqm is not None:
tqm.cleanup()
def _execute_playbook(playbook_name):
"""
Execute a playbook stored in the *share_dir*.
"""
install_dir = os.path.dirname(os.path.dirname(sys.executable))
share_dir = os.path.join(install_dir, 'share', 'dws')
playbook_path = os.path.join(share_dir, 'playbooks', playbook_name)
if not os.path.exists(playbook_path):
# When running directly from within src_dir.
share_dir = os.path.join(install_dir, 'share')
playbook_path = os.path.join(share_dir, 'playbooks', playbook_name)
sysconf_dir = os.path.join(install_dir, 'etc')
Options = namedtuple('Options', ['connection', 'module_path', 'forks',
'become', 'become_method', 'become_user', 'check'])
options = Options(connection='local',
module_path=os.path.dirname(tero.__file__), forks=100, become=None,
become_method=None, become_user=None, check=False)
passwords = dict(vault_pass='******')
loader = DataLoader()
variable_manager = VariableManager()
inventory = Inventory(loader=loader, variable_manager=variable_manager,
host_list=os.path.join(sysconf_dir, 'ansible', 'hosts'))
variable_manager.set_inventory(inventory)
playbook = Playbook.load(playbook_path,
variable_manager=variable_manager, loader=loader)
tqm = None
try:
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=options,
passwords=passwords)
for play in playbook.get_plays():
result = tqm.run(play)
finally:
if tqm is not None:
tqm.cleanup()
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
entrylist = []
entry = {}
try:
# preload become/connection/shell to set config defs cached
list(connection_loader.all(class_only=True))
list(shell_loader.all(class_only=True))
list(become_loader.all(class_only=True))
for playbook in self._playbooks:
# deal with FQCN
resource = _get_collection_playbook_path(playbook)
if resource is not None:
playbook_path = resource[1]
playbook_collection = resource[2]
else:
playbook_path = playbook
# not fqcn, but might still be colleciotn playbook
playbook_collection = _get_collection_name_from_path(
playbook)
if playbook_collection:
display.warning(
"running playbook inside collection {0}".format(
playbook_collection))
AnsibleCollectionConfig.default_collection = playbook_collection
else:
AnsibleCollectionConfig.default_collection = None
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
# FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
i = 1
plays = pb.get_plays()
display.vv(u'%d plays in %s' %
(len(plays), to_text(playbook_path)))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
# Allow variables to be used in vars_prompt fields.
all_vars = self._variable_manager.get_vars(play=play)
templar = Templar(loader=self._loader, variables=all_vars)
setattr(play, 'vars_prompt',
templar.template(play.vars_prompt))
# FIXME: this should be a play 'sub object' like loop_control
if play.vars_prompt:
for var in play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = boolean(var.get("private", True))
confirm = boolean(var.get("confirm", False))
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
unsafe = var.get("unsafe", None)
if vname not in self._variable_manager.extra_vars:
if self._tqm:
self._tqm.send_callback(
'v2_playbook_on_vars_prompt', vname,
private, prompt, encrypt, confirm,
salt_size, salt, default, unsafe)
play.vars[vname] = display.do_var_prompt(
vname, private, prompt, encrypt,
confirm, salt_size, salt, default,
unsafe)
else: # we are either in --list-<option> or syntax check
play.vars[vname] = default
# Post validate so any play level variables are templated
all_vars = self._variable_manager.get_vars(play=play)
templar = Templar(loader=self._loader, variables=all_vars)
play.post_validate(templar)
if context.CLIARGS['syntax']:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(play)
else:
self._tqm._unreachable_hosts.update(
self._unreachable_hosts)
previously_failed = len(self._tqm._failed_hosts)
previously_unreachable = len(
self._tqm._unreachable_hosts)
break_play = False
# we are actually running plays
batches = self._get_serialized_batches(play)
if len(batches) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
for batch in batches:
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
try:
result = self._tqm.run(play=play)
except AnsibleEndPlay as e:
result = e.result
break_play = True
break
# break the play if the result equals the special return code
if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
result = self._tqm.RUN_FAILED_HOSTS
break_play = True
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
(previously_failed + previously_unreachable)
if len(batch) == failed_hosts_count:
break_play = True
break
# update the previous counts so they don't accumulate incorrectly
# over multiple serial batches
previously_failed += len(
self._tqm._failed_hosts) - previously_failed
previously_unreachable += len(
self._tqm._unreachable_hosts
) - previously_unreachable
# save the unreachable hosts from this batch
self._unreachable_hosts.update(
self._tqm._unreachable_hosts)
if break_play:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# send the stats callback for this playbook
if self._tqm is not None:
if C.RETRY_FILES_ENABLED:
retries = set(self._tqm._failed_hosts.keys())
retries.update(self._tqm._unreachable_hosts.keys())
retries = sorted(retries)
if len(retries) > 0:
if C.RETRY_FILES_SAVE_PATH:
basedir = C.RETRY_FILES_SAVE_PATH
elif playbook_path:
basedir = os.path.dirname(
os.path.abspath(playbook_path))
else:
basedir = '~/'
(retry_name, _) = os.path.splitext(
os.path.basename(playbook_path))
filename = os.path.join(basedir,
"%s.retry" % retry_name)
if self._generate_retry_inventory(
filename, retries):
display.display(
"\tto retry, use: --limit @%s\n" %
filename)
self._tqm.send_callback('v2_playbook_on_stats',
self._tqm._stats)
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._tqm.cleanup()
if self._loader:
self._loader.cleanup_all_tmp_files()
if context.CLIARGS['syntax']:
display.display("No issues encountered")
return result
if context.CLIARGS['start_at_task'] and not self._tqm._start_at_done:
display.error(
"No matching task \"%s\" found."
" Note: --start-at-task can only follow static includes." %
context.CLIARGS['start_at_task'])
return result
def test_play_iterator(self):
# import epdb; epdb.st()
fake_loader = DictDataLoader({
"test_play.yml":
"""
- hosts: all
gather_facts: false
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
- block:
- debug: msg="this is a block task"
- block:
- debug: msg="this is a sub-block in a block"
rescue:
- debug: msg="this is a rescue task"
- block:
- debug: msg="this is a sub-block in a rescue"
always:
- debug: msg="this is an always task"
- block:
- debug: msg="this is a sub-block in an always"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml':
"""
- name: role task
debug: msg="this is a role task"
- block:
- name: role block task
debug: msg="inside block in role"
always:
- name: role always task
debug: msg="always task in block in role"
- include: foo.yml
- name: role task after include
debug: msg="after include in role"
- block:
- name: starting role nested block 1
debug:
- block:
- name: role nested block 1 task 1
debug:
- name: role nested block 1 task 2
debug:
- name: role nested block 1 task 3
debug:
- name: end of role nested block 1
debug:
- name: starting role nested block 2
debug:
- block:
- name: role nested block 2 task 1
debug:
- name: role nested block 2 task 2
debug:
- name: role nested block 2 task 3
debug:
- name: end of role nested block 2
debug:
""",
'/etc/ansible/roles/test_role/tasks/foo.yml':
"""
- name: role included task
debug: msg="this is task in an include from a role"
"""
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml',
loader=fake_loader,
variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
mock_var_manager._fact_cache['host00'] = dict()
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# pre task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# role task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.name, "role task")
self.assertIsNotNone(task._role)
# role block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role block task")
self.assertIsNotNone(task._role)
# role block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role always task")
self.assertIsNotNone(task._role)
# role include task
# (host_state, task) = itr.get_next_task_for_host(hosts[0])
# self.assertIsNotNone(task)
# self.assertEqual(task.action, 'debug')
# self.assertEqual(task.name, "role included task")
# self.assertIsNotNone(task._role)
# role task after include
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role task after include")
self.assertIsNotNone(task._role)
# role nested block tasks
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "starting role nested block 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 1 task 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 1 task 2")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 1 task 3")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "end of role nested block 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "starting role nested block 2")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 2 task 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 2 task 2")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 2 task 3")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "end of role nested block 2")
self.assertIsNotNone(task._role)
# regular play task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNone(task._role)
# block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a block task"))
# sub-block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
# mark the host failed
itr.mark_host_failed(hosts[0])
# block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a rescue task"))
# sub-block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args,
dict(msg="this is a sub-block in a rescue"))
# block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is an always task"))
# sub-block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args,
dict(msg="this is a sub-block in an always"))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# post task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
# host 0 shouldn't be in the failed hosts, as the error
# was handled by a rescue block
failed_hosts = itr.get_failed_hosts()
self.assertNotIn(hosts[0], failed_hosts)
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
signal.signal(signal.SIGINT, self._cleanup)
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
i = 1
plays = pb.get_plays()
self._display.vv('%d plays in %s' %
(len(plays), playbook_path))
for play in plays:
self._inventory.remove_restriction()
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(
loader=self._loader, play=play)
new_play = play.copy()
new_play.post_validate(all_vars, fail_on_undefined=False)
if self._tqm is None:
# we are just doing a listing
pname = new_play.get_name().strip()
if pname == 'PLAY: <no name specified>':
pname = 'PLAY: #%d' % i
p = {'name': pname}
if self._options.listhosts:
p['pattern'] = play.hosts
p['hosts'] = set(
self._inventory.get_hosts(new_play.hosts))
#TODO: play tasks are really blocks, need to figure out how to get task objects from them
elif self._options.listtasks:
p['tasks'] = []
for task in play.get_tasks():
p['tasks'].append(task)
#p['tasks'].append({'name': task.get_name().strip(), 'tags': task.tags})
elif self._options.listtags:
p['tags'] = set(new_play.tags)
for task in play.get_tasks():
p['tags'].update(task)
#p['tags'].update(task.tags)
entry['plays'].append(p)
else:
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', new_play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
result = 0
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
if result != 0:
break
if result != 0:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._cleanup()
# FIXME: this stat summary stuff should be cleaned up and moved
# to a new method, if it even belongs here...
self._display.banner("PLAY RECAP")
hosts = sorted(self._tqm._stats.processed.keys())
for h in hosts:
t = self._tqm._stats.summarize(h)
self._display.display(
"%s : %s %s %s %s" %
(hostcolor(h, t), colorize('ok', t['ok'], 'green'),
colorize('changed', t['changed'], 'yellow'),
colorize('unreachable', t['unreachable'],
'red'), colorize('failed', t['failures'], 'red')),
screen_only=True)
self._display.display(
"%s : %s %s %s %s" %
(hostcolor(h, t, False), colorize('ok', t['ok'], None),
colorize('changed', t['changed'], None),
colorize('unreachable', t['unreachable'],
None), colorize('failed', t['failures'], None)),
log_only=True)
self._display.display("", screen_only=True)
# END STATS STUFF
return result
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
from ansible.playbook.play import Play
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude,
self).load_data(ds, variable_manager, loader)
all_vars = self.vars.copy()
if variable_manager:
all_vars.update(variable_manager.get_vars())
templar = Templar(loader=loader, variables=all_vars)
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = templar.template(new_obj.import_playbook)
# check for FQCN
resource = _get_collection_playbook_path(file_name)
if resource is not None:
playbook = resource[1]
playbook_collection = resource[2]
else:
# not FQCN try path
playbook = file_name
if not os.path.isabs(playbook):
playbook = os.path.join(basedir, playbook)
# might still be collection playbook
playbook_collection = _get_collection_name_from_path(playbook)
if playbook_collection:
# it is a collection playbook, setup default collections
AnsibleCollectionConfig.default_collection = playbook_collection
else:
# it is NOT a collection playbook, setup adjecent paths
AnsibleCollectionConfig.playbook_paths.append(
os.path.dirname(
os.path.abspath(
to_bytes(playbook, errors='surrogate_or_strict'))))
pb._load_playbook_data(file_name=playbook,
variable_manager=variable_manager,
vars=self.vars.copy())
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
# conditional includes on a playbook need a marker to skip gathering
if new_obj.when and isinstance(entry, Play):
entry._included_conditional = new_obj.when[:]
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(playbook)
# Check to see if we need to forward the conditionals on to the included
# plays. If so, we can take a shortcut here and simply prepend them to
# those attached to each block (if any)
if new_obj.when:
for task_block in (entry.pre_tasks + entry.roles +
entry.tasks + entry.post_tasks):
task_block._when = new_obj.when[:] + task_block.when[:]
return pb
def test_play_iterator_nested_blocks(self):
fake_loader = DictDataLoader({
"test_play.yml":
"""
- hosts: all
gather_facts: false
tasks:
- block:
- block:
- block:
- block:
- block:
- debug: msg="this is the first task"
- ping:
rescue:
- block:
- block:
- block:
- block:
- debug: msg="this is the rescue task"
always:
- block:
- block:
- block:
- block:
- debug: msg="this is the always task"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml',
loader=fake_loader,
variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
# get the first task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg='this is the first task'))
# fail the host
itr.mark_host_failed(hosts[0])
# get the resuce task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg='this is the rescue task'))
# get the always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg='this is the always task'))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
def test_empty_playbook(self):
fake_loader = DictDataLoader({})
p = Playbook(loader=fake_loader)
def test_get_serialized_batches(self):
fake_loader = DictDataLoader({
'no_serial.yml':
'''
- hosts: all
gather_facts: no
tasks:
- debug: var=inventory_hostname
''',
'serial_int.yml':
'''
- hosts: all
gather_facts: no
serial: 2
tasks:
- debug: var=inventory_hostname
''',
'serial_pct.yml':
'''
- hosts: all
gather_facts: no
serial: 20%
tasks:
- debug: var=inventory_hostname
''',
'serial_list.yml':
'''
- hosts: all
gather_facts: no
serial: [1, 2, 3]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_mixed.yml':
'''
- hosts: all
gather_facts: no
serial: [1, "20%", -1]
tasks:
- debug: var=inventory_hostname
''',
'serial_no_unreachable.yml':
'''
- hosts: all
gather_facts: no
serial: [[2, 1]]
tasks:
- debug: var=inventory_hostname
''',
'serial_pct_no_unreachable.yml':
'''
- hosts: all
gather_facts: no
serial: [[20%, 1]]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_no_unreachable.yml':
'''
- hosts: all
gather_facts: no
serial: [[1, 1], 2, [3, 0]]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_no_unreachable_mixed.yml':
'''
- hosts: all
gather_facts: no
serial: [1, ["20%", 1], [-1, 1]]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_no_unreachable_inv.yml':
'''
- hosts: all
gather_facts: no
serial: [[1, 1], [2, "not_valid"], [2, 1], [2, 10], [2, 0], 1]
tasks:
- debug: var=inventory_hostname
''',
})
mock_inventory = MagicMock()
mock_var_manager = MagicMock()
templar = Templar(loader=fake_loader)
pbe = PlaybookExecutor(
playbooks=[
'no_serial.yml', 'serial_int.yml', 'serial_pct.yml',
'serial_list.yml', 'serial_list_mixed.yml',
'serial_no_unreachable.yml', 'serial_pct_no_unreachable.yml',
'serial_list_no_unreachable.yml',
'serial_list_no_unreachable_mixed.yml',
'serial_list_no_unreachable_inv.yml'
],
inventory=mock_inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
passwords=[],
)
playbook = Playbook.load(pbe._playbooks[0],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(pbe._get_serialized_batches(play), ([[
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]], [0]))
playbook = Playbook.load(pbe._playbooks[1],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']], [0, 0, 0, 0, 0]))
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']], [0, 0, 0, 0, 0]))
playbook = Playbook.load(pbe._playbooks[3],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'],
['host6', 'host7', 'host8'], ['host9']], [0, 0, 0, 0, 0]))
playbook = Playbook.load(pbe._playbooks[4],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0'], ['host1', 'host2'],
['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
], [0, 0, 0]))
# Test when serial percent is under 1.0
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
self.assertEqual(pbe._get_serialized_batches(play),
([['host0'], ['host1'], ['host2']], [0, 0, 0]))
# Test when there is a remainder for serial as a percent
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9', 'host10'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9'], ['host10']
], [0, 0, 0, 0, 0, 0]))
playbook = Playbook.load(pbe._playbooks[5],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']], [1, 1, 1, 1, 1]))
playbook = Playbook.load(pbe._playbooks[6],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']], [1, 1, 1, 1, 1]))
playbook = Playbook.load(pbe._playbooks[7],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'],
['host6', 'host7', 'host8'], ['host9']], [1, 0, 0, 0, 0]))
playbook = Playbook.load(pbe._playbooks[8],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
([['host0'], ['host1', 'host2'],
['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
], [0, 1, 0]))
playbook = Playbook.load(pbe._playbooks[9],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(pbe._get_serialized_batches(play),
([['host0'], ['host1', 'host2'], ['host3', 'host4'],
['host5', 'host6'], ['host7', 'host8'], ['host9']
], [1, 0, 1, 0, 0, 0]))
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
signal.signal(signal.SIGINT, self._cleanup)
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
self._inventory.set_playbook_basedir(os.path.dirname(playbook_path))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
i = 1
plays = pb.get_plays()
self._display.vv('%d plays in %s' % (len(plays), playbook_path))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
if play.vars_prompt:
for var in play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = var.get("private", True)
confirm = var.get("confirm", False)
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
if vname not in play.vars:
if self._tqm:
self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default)
play.vars[vname] = self._do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(new_play)
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm._unreachable_hosts.update(self._unreachable_hosts)
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback('v2_playbook_on_play_start', new_play)
self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts)
if new_play.any_errors_fatal and failed_hosts_count > 0:
break
elif new_play.max_fail_percentage is not None and \
int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
break
elif len(batch) == failed_hosts_count:
break
# clear the failed hosts dictionaires in the TQM for the next batch
self._unreachable_hosts.update(self._tqm._unreachable_hosts)
self._tqm.clear_failed_hosts()
# if the last result wasn't zero or 3 (some hosts were unreachable),
# break out of the serial batch loop
if result not in (0, 3):
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._cleanup()
if self._options.syntax:
self.display.display("No issues encountered")
return result
# FIXME: this stat summary stuff should be cleaned up and moved
# to a new method, if it even belongs here...
self._display.banner("PLAY RECAP")
hosts = sorted(self._tqm._stats.processed.keys())
for h in hosts:
t = self._tqm._stats.summarize(h)
self._display.display(u"%s : %s %s %s %s" % (
hostcolor(h, t),
colorize(u'ok', t['ok'], 'green'),
colorize(u'changed', t['changed'], 'yellow'),
colorize(u'unreachable', t['unreachable'], 'red'),
colorize(u'failed', t['failures'], 'red')),
screen_only=True
)
self._display.display(u"%s : %s %s %s %s" % (
hostcolor(h, t, False),
colorize(u'ok', t['ok'], None),
colorize(u'changed', t['changed'], None),
colorize(u'unreachable', t['unreachable'], None),
colorize(u'failed', t['failures'], None)),
log_only=True
)
self._display.display("", screen_only=True)
# END STATS STUFF
return result
def test_get_serialized_batches(self):
fake_loader = DictDataLoader({
'no_serial.yml': '''
- hosts: all
gather_facts: no
tasks:
- debug: var=inventory_hostname
''',
'serial_int.yml': '''
- hosts: all
gather_facts: no
serial: 2
tasks:
- debug: var=inventory_hostname
''',
'serial_pct.yml': '''
- hosts: all
gather_facts: no
serial: 20%
tasks:
- debug: var=inventory_hostname
''',
'serial_list.yml': '''
- hosts: all
gather_facts: no
serial: [1, 2, 3]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_mixed.yml': '''
- hosts: all
gather_facts: no
serial: [1, "20%", -1]
tasks:
- debug: var=inventory_hostname
''',
})
mock_inventory = MagicMock()
mock_var_manager = MagicMock()
# fake out options to use the syntax CLI switch, which will ensure
# the PlaybookExecutor doesn't create a TaskQueueManager
mock_options = MagicMock()
mock_options.syntax.value = True
templar = Templar(loader=fake_loader)
pbe = PlaybookExecutor(
playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml', 'serial_list.yml', 'serial_list_mixed.yml'],
inventory=mock_inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
options=mock_options,
passwords=[],
)
playbook = Playbook.load(pbe._playbooks[0], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']])
playbook = Playbook.load(pbe._playbooks[1], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'], ['host6', 'host7'], ['host8', 'host9']]
)
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'], ['host6', 'host7'], ['host8', 'host9']]
)
playbook = Playbook.load(pbe._playbooks[3], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'], ['host6', 'host7', 'host8'], ['host9']]
)
playbook = Playbook.load(pbe._playbooks[4], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']
self.assertEqual(pbe._get_serialized_batches(play), [['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']])
# Test when serial percent is under 1.0
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
self.assertEqual(pbe._get_serialized_batches(play), [['host0'], ['host1'], ['host2']])
# Test when there is a remainder for serial as a percent
playbook = Playbook.load(pbe._playbooks[2], variable_manager=mock_var_manager, loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9', 'host10']
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'], ['host6', 'host7'], ['host8', 'host9'], ['host10']]
)
def test_empty_playbook(self):
p = Playbook()
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
signal.signal(signal.SIGINT, self._cleanup)
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
i = 1
plays = pb.get_plays()
self._display.vv('%d plays in %s' % (len(plays), playbook_path))
for play in plays:
self._inventory.remove_restriction()
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
pname = new_play.get_name().strip()
if pname == 'PLAY: <no name specified>':
pname = 'PLAY: #%d' % i
p = { 'name': pname }
if self._options.listhosts:
p['pattern']=play.hosts
p['hosts']=set(self._inventory.get_hosts(new_play.hosts))
#TODO: play tasks are really blocks, need to figure out how to get task objects from them
elif self._options.listtasks:
p['tasks'] = []
for task in play.get_tasks():
p['tasks'].append(task)
#p['tasks'].append({'name': task.get_name().strip(), 'tags': task.tags})
elif self._options.listtags:
p['tags'] = set(new_play.tags)
for task in play.get_tasks():
p['tags'].update(task)
#p['tags'].update(task.tags)
entry['plays'].append(p)
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback('v2_playbook_on_play_start', new_play)
self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# if the last result wasn't zero, break out of the serial batch loop
if result != 0:
break
# if the last result wasn't zero, break out of the play loop
if result != 0:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._cleanup()
if self._options.syntax:
self.display.display("No issues encountered")
return result
# FIXME: this stat summary stuff should be cleaned up and moved
# to a new method, if it even belongs here...
self._display.banner("PLAY RECAP")
hosts = sorted(self._tqm._stats.processed.keys())
for h in hosts:
t = self._tqm._stats.summarize(h)
self._display.display("%s : %s %s %s %s" % (
hostcolor(h, t),
colorize('ok', t['ok'], 'green'),
colorize('changed', t['changed'], 'yellow'),
colorize('unreachable', t['unreachable'], 'red'),
colorize('failed', t['failures'], 'red')),
screen_only=True
)
self._display.display("%s : %s %s %s %s" % (
hostcolor(h, t, False),
colorize('ok', t['ok'], None),
colorize('changed', t['changed'], None),
colorize('unreachable', t['unreachable'], None),
colorize('failed', t['failures'], None)),
log_only=True
)
self._display.display("", screen_only=True)
# END STATS STUFF
return result
def execute_playbook(play_book, host_list=[]):
host_list = host_list
# since the API is constructed for CLI it expects certain options to always be set in the context object
context.CLIARGS = ImmutableDict(connection='smart',
module_path=None,
become=None,
become_method=None,
become_user=None,
check=False,
forks=4)
sources = ','.join(host_list)
if len(host_list) == 1:
sources += ','
# initialize needed objects
loader = DataLoader(
) # Takes care of finding and reading yaml, json and ini files
passwords = dict(vault_pass='******')
# Instantiate our ResultsCollectorJSONCallback for handling results as they come in. Ansible expects this to be one of its main display outlets
results_callback = ResultsCollectorJSONCallback()
inventory = InventoryManager(loader=loader, sources=sources)
variable_manager = VariableManager(loader=loader, inventory=inventory)
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords,
stdout_callback=
results_callback, # Use our custom callback instead of the ``default`` callback plugin, which prints to stdout
)
pbex = PlaybookExecutor(playbooks=[play_book],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords)
playbook = Playbook.load(pbex._playbooks[0],
variable_manager=variable_manager,
loader=loader)
play = playbook.get_plays()[0]
# Actually run it
try:
result = tqm.run(
play
) # most interesting data for a play is actually sent to the callback's methods
finally:
# we always need to cleanup child procs and the structures we use to communicate with them
tqm.cleanup()
if loader:
loader.cleanup_all_tmp_files()
# Remove ansible tmpdir
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
results_raw = {'success': {}, 'failed': {}, 'unreachable': {}}
for host, result in results_callback.host_ok.items():
results_raw['success'][host] = result._result
for host, result in results_callback.host_failed.items():
results_raw['failed'][host] = result._result
for host, result in results_callback.host_unreachable.items():
results_raw['unreachable'][host] = result._result
return results_raw
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
self._inventory.set_playbook_basedir(
os.path.dirname(playbook_path))
i = 1
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
for play in pb.get_plays():
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the
# inventory
self._inventory.remove_restriction()
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the
# original object.
all_vars = self._variable_manager.get_vars(
loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
self._tqm._unreachable_hosts.update(
self._unreachable_hosts)
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', new_play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized
# batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(
self._tqm._failed_hosts) + len(
self._tqm._unreachable_hosts)
if new_play.any_errors_fatal and failed_hosts_count > 0:
break
elif new_play.max_fail_percentage is not None and \
(int((new_play.max_fail_percentage) / 100.0 * len(batch)) >
int((len(batch) - failed_hosts_count) / len(batch) * 100.0)):
break
elif len(batch) == failed_hosts_count:
break
# clear the failed hosts dictionaires in the TQM for
# the next batch
self._unreachable_hosts.update(
self._tqm._unreachable_hosts)
self._tqm.clear_failed_hosts()
# if the last result wasn't zero or 3 (some hosts were unreachable),
# break out of the serial batch loop
if result not in (0, 3):
break
i = i + 1 # per play
self._tqm.send_callback('v2_playbook_on_stats',
self._tqm._stats)
# if the last result wasn't zero, break out of the playbook
# file name loop
if result != 0:
break
finally:
if self._tqm is not None:
self._tqm.cleanup()
return result
options = Options(listtags=False, listtasks=False, listhosts=False, syntax=False, connection='ssh', module_path=None, forks=100, remote_user='******', private_key_file=None, ssh_common_args=None, ssh_extra_args=None, sftp_extra_args=None, scp_extra_args=None, become=True, become_method=None, become_user='******', verbosity=None, check=False)
variable_manager.extra_vars = {'hosts': 'mywebserver'} # This can accomodate various other command line arguments.`
passwords = {}
pbex = PlaybookExecutor(playbooks=[playbook_path], inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords)
print pbex._playbooks
taskData = []
conditionalData = []
print "\nParsed tasks....."
# Extract data from compiled playbook
for playbook_path in pbex._playbooks:
pb = Playbook.load(playbook_path, variable_manager=pbex._variable_manager, loader=pbex._loader)
plays = pb.get_plays()
for play in plays:
#print play.get_name()
tasklist = play.get_tasks() # play.get_tasks() returns a merged list of task blocks (pre, post, etc)
for taskBlocks in tasklist:
for task in taskBlocks:
hashedTask = hash(task)
changed = Task._get_parent_attribute(task, "changed_when")
taskData.append(hashedTask)
if changed is not None:
conditionalData.append(hashedTask)
print task, hashedTask, changed
# Build graph
def execute_playbook(play_book, host_list=[]):
host_list = host_list
# since the API is constructed for CLI it expects certain options to always be set in the context object
context.CLIARGS = ImmutableDict(
connection='paramiko',
module_path=[
'/home/koshik/Documents/projects/devops/scripts/aenv/lib/python3.6/site-packages/ansible'
],
forks=10,
become=None,
become_method=None,
become_user=None,
check=False,
diff=False)
sources = ','.join(host_list)
if len(host_list) == 1:
sources += ','
# initialize needed objects
loader = DataLoader(
) # Takes care of finding and reading yaml, json and ini files
passwords = dict(vault_pass='******')
# Instantiate our ResultsCollectorJSONCallback for handling results as they come in. Ansible expects this to be one of its main display outlets
results_callback = ResultsCollectorJSONCallback()
inventory = InventoryManager(loader=loader, sources=sources)
variable_manager = VariableManager(loader=loader, inventory=inventory)
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords,
stdout_callback=
results_callback, # Use our custom callback instead of the ``default`` callback plugin, which prints to stdout
)
pbex = PlaybookExecutor(playbooks=[play_book],
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords)
playbook = Playbook.load(pbex._playbooks[0],
variable_manager=variable_manager,
loader=loader)
play = playbook.get_plays()[0]
# Actually run it
try:
result = tqm.run(
play
) # most interesting data for a play is actually sent to the callback's methods
finally:
# we always need to cleanup child procs and the structures we use to communicate with them
tqm.cleanup()
if loader:
loader.cleanup_all_tmp_files()
# Remove ansible tmpdir
shutil.rmtree(C.DEFAULT_LOCAL_TMP, True)
if (results_callback.host_ok.items()):
print("********* SUCCESS ***********")
return results_callback.host_ok.items()
if (results_callback.host_failed.items()):
print("********* FAILED *******")
for host, result in results_callback.host_failed.items():
print('{0} >>> {1}'.format(host, result._result['msg']))
if (results_callback.host_unreachable.items()):
print("********* HOST DOWN *********")
for host, result in results_callback.host_unreachable.items():
print('{0} >>> {1}'.format(host, result._result['msg']))
def parse(self, *args, **kwargs) -> PlaybookNode:
"""
Loop through the playbook and generate the graph.
The graph is drawn following this order (https://docs.ansible.com/ansible/2.4/playbooks_reuse_roles.html#using-roles)
for each play:
add pre_tasks
add roles
if include_role_tasks
add role_tasks
add tasks
add post_tasks
:return:
"""
playbook = Playbook.load(
self.playbook_filename,
loader=self.data_loader,
variable_manager=self.variable_manager,
)
# the root node
playbook_root_node = PlaybookNode(self.playbook_filename,
raw_object=playbook)
# loop through the plays
for play in playbook.get_plays():
# the load basedir is relative to the playbook path
if play._included_path is not None:
self.data_loader.set_basedir(play._included_path)
else:
self.data_loader.set_basedir(playbook._basedir)
display.vvv(
f"Loader basedir set to {self.data_loader.get_basedir()}")
play_vars = self.variable_manager.get_vars(play)
play_hosts = [
h.get_name() for h in self.inventory_manager.get_hosts(
self.template(play.hosts, play_vars))
]
play_name = f"Play: {clean_name(play.get_name())} ({len(play_hosts)})"
play_name = self.template(play_name, play_vars)
display.v(f"Parsing {play_name}")
play_node = PlayNode(play_name, hosts=play_hosts, raw_object=play)
playbook_root_node.add_node("plays", play_node)
# loop through the pre_tasks
display.v("Parsing pre_tasks...")
for pre_task_block in play.pre_tasks:
self._include_tasks_in_blocks(
current_play=play,
parent_nodes=[play_node],
block=pre_task_block,
play_vars=play_vars,
node_type="pre_task",
)
# loop through the roles
display.v("Parsing roles...")
for role in play.get_roles():
# Don't insert tasks from ``import/include_role``, preventing duplicate graphing
if role.from_include:
continue
# the role object doesn't inherit the tags from the play. So we add it manually.
role.tags = role.tags + play.tags
if not role.evaluate_tags(only_tags=self.tags,
skip_tags=self.skip_tags,
all_vars=play_vars):
display.vv(
f"The role '{role.get_name()}' is skipped due to the tags."
)
# Go to the next role
continue
if self.group_roles_by_name:
# If we are grouping roles, we use the hash of role name as the node id
role_node_id = "role_" + hash_value(role.get_name())
else:
# Otherwise, a random id is used
role_node_id = generate_id("role_")
role_node = RoleNode(
clean_name(role.get_name()),
node_id=role_node_id,
raw_object=role,
parent=play_node,
)
# edge from play to role
play_node.add_node("roles", role_node)
if self.include_role_tasks:
# loop through the tasks of the roles
for block in role.compile(play):
self._include_tasks_in_blocks(
current_play=play,
parent_nodes=[role_node],
block=block,
play_vars=play_vars,
node_type="task",
)
# end of roles loop
# loop through the tasks
display.v("Parsing tasks...")
for task_block in play.tasks:
self._include_tasks_in_blocks(
current_play=play,
parent_nodes=[play_node],
block=task_block,
play_vars=play_vars,
node_type="task",
)
# loop through the post_tasks
display.v("Parsing post_tasks...")
for post_task_block in play.post_tasks:
self._include_tasks_in_blocks(
current_play=play,
parent_nodes=[play_node],
block=post_task_block,
play_vars=play_vars,
node_type="post_task",
)
# Summary
display.v(
f"{len(play_node.pre_tasks)} pre_task(s) added to the graph.")
display.v(f"{len(play_node.roles)} role(s) added to the play")
display.v(f"{len(play_node.tasks)} task(s) added to the play")
display.v(
f"{len(play_node.post_tasks)} post_task(s) added to the play")
# moving to the next play
return playbook_root_node
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
# FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
i = 1
plays = pb.get_plays()
display.vv(u'%d plays in %s' %
(len(plays), to_text(playbook_path)))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
# Doing this before vars_prompt to allow for using variables in prompt.
all_vars = self._variable_manager.get_vars(play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if play.vars_prompt:
for var in new_play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = boolean(var.get("private", True))
confirm = boolean(var.get("confirm", False))
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
if vname not in self._variable_manager.extra_vars:
if self._tqm:
self._tqm.send_callback(
'v2_playbook_on_vars_prompt', vname,
private, prompt, encrypt, confirm,
salt_size, salt, default)
play.vars[vname] = display.do_var_prompt(
vname, private, prompt, encrypt,
confirm, salt_size, salt, default)
else: # we are either in --list-<option> or syntax check
play.vars[vname] = default
# Post validating again in case variables were entered in the prompt.
all_vars = self._variable_manager.get_vars(play=play)
templar = Templar(loader=self._loader,
variables=all_vars)
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(new_play)
else:
self._tqm._unreachable_hosts.update(
self._unreachable_hosts)
previously_failed = len(self._tqm._failed_hosts)
previously_unreachable = len(
self._tqm._unreachable_hosts)
break_play = False
# we are actually running plays
batches = self._get_serialized_batches(new_play)
if len(batches) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', new_play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
for batch in batches:
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# break the play if the result equals the special return code
if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
result = self._tqm.RUN_FAILED_HOSTS
break_play = True
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
(previously_failed + previously_unreachable)
if len(batch) == failed_hosts_count:
break_play = True
break
# update the previous counts so they don't accumulate incorrectly
# over multiple serial batches
previously_failed += len(
self._tqm._failed_hosts) - previously_failed
previously_unreachable += len(
self._tqm._unreachable_hosts
) - previously_unreachable
# save the unreachable hosts from this batch
self._unreachable_hosts.update(
self._tqm._unreachable_hosts)
if break_play:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# send the stats callback for this playbook
if self._tqm is not None:
if C.RETRY_FILES_ENABLED:
retries = set(self._tqm._failed_hosts.keys())
retries.update(self._tqm._unreachable_hosts.keys())
retries = sorted(retries)
if len(retries) > 0:
if C.RETRY_FILES_SAVE_PATH:
basedir = C.RETRY_FILES_SAVE_PATH
elif playbook_path:
basedir = os.path.dirname(
os.path.abspath(playbook_path))
else:
basedir = '~/'
(retry_name, _) = os.path.splitext(
os.path.basename(playbook_path))
filename = os.path.join(basedir,
"%s.retry" % retry_name)
if self._generate_retry_inventory(
filename, retries):
display.display(
"\tto retry, use: --limit @%s\n" %
filename)
self._tqm.send_callback('v2_playbook_on_stats',
self._tqm._stats)
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._tqm.cleanup()
if self._loader:
self._loader.cleanup_all_tmp_files()
if self._options.syntax:
display.display("No issues encountered")
return result
return result
def get_playbook(self, *args, **kwargs):
inject_ansible_paths(self.master.ctrl)
from ansible.playbook import Play, Playbook
import ansible.errors
(options, loader, inventory, variable_manager) = self.get_ansible_variablemanager(**kwargs)
playbooks_directory = get_playbooks_directory(self.master.main_config)
playbook = kwargs.pop('playbook', None)
if playbook is None:
for instance_id in (self.uid, self.id):
playbook_path = os.path.join(playbooks_directory, '%s.yml' % instance_id)
if os.path.exists(playbook_path):
playbook = playbook_path
break
if 'playbook' in self.config:
if playbook is not None and playbook != self.config['playbook']:
log.warning("Instance '%s' has the 'playbook' option set, but there is also a playbook at the default location '%s', which differs from '%s'." % (self.config_id, playbook, self.config['playbook']))
playbook = self.config['playbook']
if playbook is not None:
log.info("Using playbook at '%s'." % playbook)
roles = kwargs.pop('roles', None)
if roles is None and 'roles' in self.config:
roles = self.config['roles']
if roles is not None and playbook is not None:
log.error("You can't use a playbook and the 'roles' options at the same time for instance '%s'." % self.config_id)
sys.exit(1)
if playbook is None and roles is None:
return None
skip_host_check = kwargs.pop('skip_host_check', False)
try:
if roles is None:
pb = Playbook.load(playbook, variable_manager=variable_manager, loader=loader)
plays = pb.get_plays()
else:
if isinstance(roles, basestring):
roles = roles.split()
data = {
'hosts': [self.uid],
'roles': roles}
plays = [Play.load(data, variable_manager=variable_manager, loader=loader)]
pb = Playbook(loader=loader)
pb._entries.extend(plays)
except ansible.errors.AnsibleError as e:
log.error("AnsibleError: %s" % e)
sys.exit(1)
for play in plays:
if play._attributes.get('remote_user') is None:
play._attributes['remote_user'] = self.config.get('user', 'root')
if self.config.get('sudo'):
play._attributes['sudo'] = self.config.get('sudo')
if not skip_host_check:
hosts = play._attributes.get('hosts', None)
if isinstance(hosts, basestring):
hosts = hosts.split(':')
if hosts is None:
hosts = {}
if self.uid not in hosts:
log.warning("The host '%s' is not in the list of hosts (%s) of '%s'.", self.uid, ','.join(hosts), playbook)
if not yesno("Do you really want to apply '%s' to the host '%s'?" % (playbook, self.uid)):
sys.exit(1)
play._attributes['hosts'] = [self.uid]
return pb
def run_playbook(self, playbook, workname, work_uuid, username, describe, yaml_content, is_checksyntax=False):
"""
:param playbooks: playbook路径的列表
:return:
"""
if is_checksyntax:
self._tqm = None
else:
# self.callback = PlayBookResultsCollector()
# self.callback = ResultCallback(work_uuid, workname, 'playbook', self.options, describe,
# yaml_content=yaml_content, username=username)
self.callback = JsonCallback(work_uuid, workname, 'playbook', self.options, describe,
yaml_content=yaml_content, username=username)
self._tqm = TaskQueueManager(inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords=self.passwords,
stdout_callback=self.callback)
check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE)
try:
pb = Playbook.load(playbook, variable_manager=self.variable_manager, loader=self.loader)
if self._tqm is not None:
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
plays = pb.get_plays()
for play in plays:
if play._included_path is not None:
self.loader.set_basedir(play._included_path)
else:
self.loader.set_basedir(pb._basedir)
self.inventory.remove_restriction()
# Allow variables to be used in vars_prompt fields.
all_vars = self.variable_manager.get_vars(play=play)
templar = Templar(loader=self.loader, variables=all_vars)
play.post_validate(templar)
if is_checksyntax or self._tqm is None:
return True
batches = self._get_serialized_batches(play)
if len(batches) == 0:
self._tqm.send_callback('v2_playbook_on_play_start', play)
self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
continue
for batch in batches:
self.inventory.restrict_to_hosts(batch)
try:
self._tqm.run(play)
except Exception as e:
print(e)
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
self._tqm.cleanup()
self.loader.cleanup_all_tmp_files()
return True
except Exception as e:
print(e)
return False
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
signal.signal(signal.SIGINT, self._cleanup)
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
self._inventory.set_playbook_basedir(
os.path.dirname(playbook_path))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
i = 1
plays = pb.get_plays()
self._display.vv('%d plays in %s' %
(len(plays), playbook_path))
for play in plays:
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
if play.vars_prompt:
for var in play.vars_prompt:
if 'name' not in var:
raise AnsibleError(
"'vars_prompt' item is missing 'name:'",
obj=play._ds)
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = var.get("private", True)
confirm = var.get("confirm", False)
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
if vname not in play.vars:
self._tqm.send_callback(
'v2_playbook_on_vars_prompt', vname,
private, prompt, encrypt, confirm,
salt_size, salt, default)
play.vars[vname] = self._do_var_prompt(
vname, private, prompt, encrypt, confirm,
salt_size, salt, default)
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(
loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(new_play)
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', new_play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# if the last result wasn't zero, break out of the serial batch loop
if result != 0:
break
# if the last result wasn't zero, break out of the play loop
if result != 0:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._cleanup()
if self._options.syntax:
self.display.display("No issues encountered")
return result
# FIXME: this stat summary stuff should be cleaned up and moved
# to a new method, if it even belongs here...
self._display.banner("PLAY RECAP")
hosts = sorted(self._tqm._stats.processed.keys())
for h in hosts:
t = self._tqm._stats.summarize(h)
self._display.display(
"%s : %s %s %s %s" %
(hostcolor(h, t), colorize('ok', t['ok'], 'green'),
colorize('changed', t['changed'], 'yellow'),
colorize('unreachable', t['unreachable'],
'red'), colorize('failed', t['failures'], 'red')),
screen_only=True)
self._display.display(
"%s : %s %s %s %s" %
(hostcolor(h, t, False), colorize('ok', t['ok'], None),
colorize('changed', t['changed'], None),
colorize('unreachable', t['unreachable'],
None), colorize('failed', t['failures'], None)),
log_only=True)
self._display.display("", screen_only=True)
# END STATS STUFF
return result
def test_noop(self):
fake_loader = DictDataLoader({
"test_play.yml":
"""
- hosts: all
gather_facts: no
tasks:
- block:
- block:
- name: task1
debug: msg='task1'
failed_when: inventory_hostname == 'host01'
- name: task2
debug: msg='task2'
rescue:
- name: rescue1
debug: msg='rescue1'
- name: rescue2
debug: msg='rescue2'
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml',
loader=fake_loader,
variable_manager=mock_var_manager)
hosts = []
for i in range(0, 2):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
mock_var_manager._fact_cache['host00'] = dict()
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
mock_options = MagicMock()
mock_options.module_path = None
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
options=mock_options,
passwords=None,
)
tqm._initialize_processes(3)
strategy = StrategyModule(tqm)
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'meta')
# debug: task1, debug: task1
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'debug')
self.assertEqual(host2_task.action, 'debug')
self.assertEqual(host1_task.name, 'task1')
self.assertEqual(host2_task.name, 'task1')
# mark the second host failed
itr.mark_host_failed(hosts[1])
# debug: task2, meta: noop
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'debug')
self.assertEqual(host2_task.action, 'meta')
self.assertEqual(host1_task.name, 'task2')
self.assertEqual(host2_task.name, '')
# meta: noop, debug: rescue1
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'debug')
self.assertEqual(host1_task.name, '')
self.assertEqual(host2_task.name, 'rescue1')
# meta: noop, debug: rescue2
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'debug')
self.assertEqual(host1_task.name, '')
self.assertEqual(host2_task.name, 'rescue2')
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'meta')
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'meta')
# end of iteration
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNone(host1_task)
self.assertIsNone(host2_task)
def run(self):
''' create and execute the single task playbook '''
super(AdHocCLI, self).run()
# only thing left should be host pattern
pattern = to_text(self.args[0], errors='surrogate_or_strict')
sshpass = None
becomepass = None
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
# dynamically load any plugins
get_all_plugin_loaders()
loader, inventory, variable_manager = self._play_prereqs(self.options)
try:
hosts = CLI.get_host_list(inventory, self.options.subset, pattern)
except AnsibleError:
if self.options.subset:
raise
else:
hosts = []
display.warning("No hosts matched, nothing to do")
if self.options.listhosts:
display.display(' hosts (%d):' % len(hosts))
for host in hosts:
display.display(' %s' % host)
return 0
if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args:
err = "No argument passed to %s module" % self.options.module_name
if pattern.endswith(".yml"):
err = err + ' (did you mean to run ansible-playbook?)'
raise AnsibleOptionsError(err)
play_ds = self._play_ds(pattern, self.options.seconds,
self.options.poll_interval)
play = Play().load(play_ds,
variable_manager=variable_manager,
loader=loader)
# used in start callback
playbook = Playbook(loader)
playbook._entries.append(play)
playbook._file_name = '__adhoc_playbook__'
if self.callback:
cb = self.callback
elif self.options.one_line:
cb = 'oneline'
# Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks'
elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
cb = C.DEFAULT_STDOUT_CALLBACK
else:
cb = 'minimal'
run_tree = False
if self.options.tree:
C.DEFAULT_CALLBACK_WHITELIST.append('tree')
C.TREE_DIR = self.options.tree
run_tree = True
# now create a task queue manager to execute the play
self._tqm = None
try:
self._tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=self.options,
passwords=passwords,
stdout_callback=cb,
run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
run_tree=run_tree,
)
self._tqm.send_callback('v2_playbook_on_start', playbook)
result = self._tqm.run(play)
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
finally:
if self._tqm:
self._tqm.cleanup()
if loader:
loader.cleanup_all_tmp_files()
return result
def test_play_iterator(self):
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
gather_facts: false
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
- block:
- debug: msg="this is a block task"
- block:
- debug: msg="this is a sub-block in a block"
rescue:
- debug: msg="this is a rescue task"
- block:
- debug: msg="this is a sub-block in a rescue"
always:
- debug: msg="this is an always task"
- block:
- debug: msg="this is a sub-block in an always"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml': """
- debug: msg="this is a role task"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
mock_var_manager._fact_cache['host00'] = dict()
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# lookup up an original task
target_task = p._entries[0].tasks[0].block[0]
task_copy = target_task.copy(exclude_block=True)
found_task = itr.get_original_task(hosts[0], task_copy)
self.assertEqual(target_task, found_task)
bad_task = Task()
found_task = itr.get_original_task(hosts[0], bad_task)
self.assertIsNone(found_task)
# pre task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# role task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNotNone(task._role)
# regular play task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNone(task._role)
# block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a block task"))
# sub-block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
# mark the host failed
itr.mark_host_failed(hosts[0])
# block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a rescue task"))
# sub-block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
# block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is an always task"))
# sub-block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# post task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
# host 0 shouldn't be in the failed hosts, as the error
# was handled by a rescue block
failed_hosts = itr.get_failed_hosts()
self.assertNotIn(hosts[0], failed_hosts)
def test_play_iterator_add_tasks(self):
fake_loader = DictDataLoader({
'test_play.yml': """
- hosts: all
gather_facts: no
tasks:
- debug: msg="dummy task"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# test the high-level add_tasks() method
s = HostState(blocks=[0,1,2])
itr._insert_tasks_into_state = MagicMock(return_value=s)
itr.add_tasks(hosts[0], [3,4,5])
self.assertEqual(itr._host_states[hosts[0].name], s)
# now actually test the lower-level method that does the work
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# iterate past first task
_, task = itr.get_next_task_for_host(hosts[0])
while(task and task.action != 'debug'):
_, task = itr.get_next_task_for_host(hosts[0])
if task is None:
raise Exception("iterated past end of play while looking for place to insert tasks")
# get the current host state and copy it so we can mutate it
s = itr.get_host_state(hosts[0])
s_copy = s.copy()
# assert with an empty task list, or if we're in a failed state, we simply return the state as-is
res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
self.assertEqual(res_state, s_copy)
s_copy.fail_state = itr.FAILED_TASKS
res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
self.assertEqual(res_state, s_copy)
# but if we've failed with a rescue/always block
mock_task = MagicMock()
s_copy.run_state = itr.ITERATING_RESCUE
res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
self.assertEqual(res_state, s_copy)
self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
itr._host_states[hosts[0].name] = res_state
(next_state, next_task) = itr.get_next_task_for_host(hosts[0], peek=True)
self.assertEqual(next_task, mock_task)
itr._host_states[hosts[0].name] = s
# test a regular insertion
s_copy = s.copy()
res_state = itr._insert_tasks_into_state(s_copy, task_list=[MagicMock()])
def test_play_iterator_add_tasks(self):
fake_loader = DictDataLoader({
'test_play.yml':
"""
- hosts: all
gather_facts: no
tasks:
- debug: msg="dummy task"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml',
loader=fake_loader,
variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# test the high-level add_tasks() method
s = HostState(blocks=[0, 1, 2])
itr._insert_tasks_into_state = MagicMock(return_value=s)
itr.add_tasks(hosts[0], [MagicMock(), MagicMock(), MagicMock()])
self.assertEqual(itr._host_states[hosts[0].name], s)
# now actually test the lower-level method that does the work
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# iterate past first task
_, task = itr.get_next_task_for_host(hosts[0])
while (task and task.action != 'debug'):
_, task = itr.get_next_task_for_host(hosts[0])
if task is None:
raise Exception(
"iterated past end of play while looking for place to insert tasks"
)
# get the current host state and copy it so we can mutate it
s = itr.get_host_state(hosts[0])
s_copy = s.copy()
# assert with an empty task list, or if we're in a failed state, we simply return the state as-is
res_state = itr._insert_tasks_into_state(s_copy, task_list=[])
self.assertEqual(res_state, s_copy)
s_copy.fail_state = itr.FAILED_TASKS
res_state = itr._insert_tasks_into_state(s_copy,
task_list=[MagicMock()])
self.assertEqual(res_state, s_copy)
# but if we've failed with a rescue/always block
mock_task = MagicMock()
s_copy.run_state = itr.ITERATING_RESCUE
res_state = itr._insert_tasks_into_state(s_copy, task_list=[mock_task])
self.assertEqual(res_state, s_copy)
self.assertIn(mock_task, res_state._blocks[res_state.cur_block].rescue)
itr._host_states[hosts[0].name] = res_state
(next_state, next_task) = itr.get_next_task_for_host(hosts[0],
peek=True)
self.assertEqual(next_task, mock_task)
itr._host_states[hosts[0].name] = s
# test a regular insertion
s_copy = s.copy()
res_state = itr._insert_tasks_into_state(s_copy,
task_list=[MagicMock()])
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
signal.signal(signal.SIGINT, self._cleanup)
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
self._inventory.set_playbook_basedir(
os.path.dirname(playbook_path))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
i = 1
plays = pb.get_plays()
self._display.vv('%d plays in %s' %
(len(plays), playbook_path))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
if play.vars_prompt:
for var in play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = var.get("private", True)
confirm = var.get("confirm", False)
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
if vname not in play.vars:
if self._tqm:
self._tqm.send_callback(
'v2_playbook_on_vars_prompt', vname,
private, prompt, encrypt, confirm,
salt_size, salt, default)
play.vars[vname] = self._do_var_prompt(
vname, private, prompt, encrypt, confirm,
salt_size, salt, default)
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(
loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(new_play)
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm._unreachable_hosts.update(
self._unreachable_hosts)
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', new_play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(
self._tqm._failed_hosts) + len(
self._tqm._unreachable_hosts)
if new_play.any_errors_fatal and failed_hosts_count > 0:
break
elif new_play.max_fail_percentage is not None and \
int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
break
elif len(batch) == failed_hosts_count:
break
# clear the failed hosts dictionaires in the TQM for the next batch
self._unreachable_hosts.update(
self._tqm._unreachable_hosts)
self._tqm.clear_failed_hosts()
# if the last result wasn't zero or 3 (some hosts were unreachable),
# break out of the serial batch loop
if result not in (0, 3):
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._cleanup()
if self._options.syntax:
self.display.display("No issues encountered")
return result
# FIXME: this stat summary stuff should be cleaned up and moved
# to a new method, if it even belongs here...
self._display.banner("PLAY RECAP")
hosts = sorted(self._tqm._stats.processed.keys())
for h in hosts:
t = self._tqm._stats.summarize(h)
self._display.display(
u"%s : %s %s %s %s" %
(hostcolor(h, t), colorize(u'ok', t['ok'], 'green'),
colorize(u'changed', t['changed'], 'yellow'),
colorize(u'unreachable', t['unreachable'],
'red'), colorize(u'failed', t['failures'], 'red')),
screen_only=True)
self._display.display(
u"%s : %s %s %s %s" %
(hostcolor(h, t, False), colorize(u'ok', t['ok'], None),
colorize(u'changed', t['changed'], None),
colorize(u'unreachable', t['unreachable'],
None), colorize(u'failed', t['failures'], None)),
log_only=True)
self._display.display("", screen_only=True)
# END STATS STUFF
return result
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
self._inventory.set_playbook_basedir(
os.path.dirname(playbook_path))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
i = 1
plays = pb.get_plays()
display.vv(u'%d plays in %s' %
(len(plays), to_unicode(playbook_path)))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
if play.vars_prompt:
for var in play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = var.get("private", True)
confirm = var.get("confirm", False)
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
if vname not in self._variable_manager.extra_vars:
if self._tqm:
self._tqm.send_callback(
'v2_playbook_on_vars_prompt', vname,
private, prompt, encrypt, confirm,
salt_size, salt, default)
play.vars[vname] = display.do_var_prompt(
vname, private, prompt, encrypt,
confirm, salt_size, salt, default)
else: # we are either in --list-<option> or syntax check
play.vars[vname] = default
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(
loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(new_play)
else:
self._tqm._unreachable_hosts.update(
self._unreachable_hosts)
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', new_play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(
self._tqm._failed_hosts) + len(
self._tqm._unreachable_hosts)
if new_play.max_fail_percentage is not None and \
int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
break
elif len(batch) == failed_hosts_count:
break
# clear the failed hosts dictionaires in the TQM for the next batch
self._unreachable_hosts.update(
self._tqm._unreachable_hosts)
self._tqm.clear_failed_hosts()
# if the last result wasn't zero or 3 (some hosts were unreachable),
# break out of the serial batch loop
if result not in (0, 3):
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# send the stats callback for this playbook
if self._tqm is not None:
if C.RETRY_FILES_ENABLED:
retries = set(self._tqm._failed_hosts.keys())
retries.update(self._tqm._unreachable_hosts.keys())
retries = sorted(retries)
if len(retries) > 0:
if C.RETRY_FILES_SAVE_PATH:
basedir = C.shell_expand(
C.RETRY_FILES_SAVE_PATH)
else:
basedir = os.path.dirname(playbook_path)
(retry_name, _) = os.path.splitext(
os.path.basename(playbook_path))
filename = os.path.join(basedir,
"%s.retry" % retry_name)
if self._generate_retry_inventory(
filename, retries):
display.display(
"\tto retry, use: --limit @%s\n" %
filename)
self._tqm.send_callback('v2_playbook_on_stats',
self._tqm._stats)
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._tqm.cleanup()
if self._options.syntax:
display.display("No issues encountered")
return result
return result
if self._inventory else [],
['--user', self._username] if self._username else [],
['--become'] if self._become else [],
['--become-user', self._become_user] if self._become_user else [],
['--forks', str(self._forks)] if self._forks else [],
[
'--ssh-extra-args',
'-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'
] if not self._modifyKnownHosts
else []['--extra-vars',
json.dumps(self._extra_vars)] if self._extra_vars else [],
[self._playbook_file.name],
]
cmd = reduce(operator.concat, cmd_builder_monoid)
Subprocess(cmd, **self._subprocess_kwargs)
################################################################
if __name__ == '__main__':
loader = DataLoader()
variable_manager = VariableManager()
playbook = Playbook(loader)
Inventory(loader,
variable_manager,
host_list={
'a': ['foo', 'bar'],
'b': ['foo', 'baz']
})
def run(self):
''' create and execute the single task playbook '''
super(AdHocCLI, self).run()
# only thing left should be host pattern
pattern = to_text(context.CLIARGS['args'],
errors='surrogate_or_strict')
sshpass = None
becomepass = None
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
# get basic objects
loader, inventory, variable_manager = self._play_prereqs()
try:
hosts = self.get_host_list(inventory, context.CLIARGS['subset'],
pattern)
except AnsibleError:
if context.CLIARGS['subset']:
raise
else:
hosts = []
display.warning("No hosts matched, nothing to do")
if context.CLIARGS['listhosts']:
display.display(' hosts (%d):' % len(hosts))
for host in hosts:
display.display(' %s' % host)
return 0
if context.CLIARGS[
'module_name'] in C.MODULE_REQUIRE_ARGS and not context.CLIARGS[
'module_args']:
err = "No argument passed to %s module" % context.CLIARGS[
'module_name']
if pattern.endswith(".yml"):
err = err + ' (did you mean to run ansible-playbook?)'
raise AnsibleOptionsError(err)
# Avoid modules that don't work with ad-hoc
if context.CLIARGS['module_name'] in ('import_playbook', ):
raise AnsibleOptionsError(
"'%s' is not a valid action for ad-hoc commands" %
context.CLIARGS['module_name'])
play_ds = self._play_ds(pattern, context.CLIARGS['seconds'],
context.CLIARGS['poll_interval'])
play = Play().load(play_ds,
variable_manager=variable_manager,
loader=loader)
# used in start callback
playbook = Playbook(loader)
playbook._entries.append(play)
playbook._file_name = '__adhoc_playbook__'
if self.callback:
cb = self.callback
elif context.CLIARGS['one_line']:
cb = 'oneline'
# Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks'
elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
cb = C.DEFAULT_STDOUT_CALLBACK
else:
cb = 'minimal'
run_tree = False
if context.CLIARGS['tree']:
C.CALLBACKS_ENABLED.append('tree')
C.TREE_DIR = context.CLIARGS['tree']
run_tree = True
# now create a task queue manager to execute the play
self._tqm = None
try:
self._tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords,
stdout_callback=cb,
run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
run_tree=run_tree,
forks=context.CLIARGS['forks'],
)
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', playbook)
result = self._tqm.run(play)
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
finally:
if self._tqm:
self._tqm.cleanup()
if loader:
loader.cleanup_all_tmp_files()
return result
def run(self):
''' create and execute the single task playbook '''
super(AdHocCLI, self).run()
# only thing left should be host pattern
pattern = to_text(self.args[0], errors='surrogate_or_strict')
sshpass = None
becomepass = None
self.normalize_become_options()
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
# dynamically load any plugins
get_all_plugin_loaders()
loader, inventory, variable_manager = self._play_prereqs(self.options)
try:
hosts = CLI.get_host_list(inventory, self.options.subset, pattern)
except AnsibleError:
if self.options.subset:
raise
else:
hosts = []
display.warning("No hosts matched, nothing to do")
if self.options.listhosts:
display.display(' hosts (%d):' % len(hosts))
for host in hosts:
display.display(' %s' % host)
return 0
if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args:
err = "No argument passed to %s module" % self.options.module_name
if pattern.endswith(".yml"):
err = err + ' (did you mean to run ansible-playbook?)'
raise AnsibleOptionsError(err)
# Avoid modules that don't work with ad-hoc
if self.options.module_name.startswith(('include', 'import_')):
raise AnsibleOptionsError("'%s' is not a valid action for ad-hoc commands" % self.options.module_name)
play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval)
play = Play().load(play_ds, variable_manager=variable_manager, loader=loader)
# used in start callback
playbook = Playbook(loader)
playbook._entries.append(play)
playbook._file_name = '__adhoc_playbook__'
if self.callback:
cb = self.callback
elif self.options.one_line:
cb = 'oneline'
# Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks'
elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
cb = C.DEFAULT_STDOUT_CALLBACK
else:
cb = 'minimal'
run_tree = False
if self.options.tree:
C.DEFAULT_CALLBACK_WHITELIST.append('tree')
C.TREE_DIR = self.options.tree
run_tree = True
# now create a task queue manager to execute the play
self._tqm = None
try:
self._tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
options=self.options,
passwords=passwords,
stdout_callback=cb,
run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
run_tree=run_tree,
)
self._tqm.send_callback('v2_playbook_on_start', playbook)
result = self._tqm.run(play)
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
finally:
if self._tqm:
self._tqm.cleanup()
if loader:
loader.cleanup_all_tmp_files()
return result
def test_playbook_executor__get_serialized_batches(self):
fake_loader = DictDataLoader({
'no_serial.yml':
'''
- hosts: all
gather_facts: no
tasks:
- debug: var=inventory_hostname
''',
'serial_int.yml':
'''
- hosts: all
gather_facts: no
serial: 2
tasks:
- debug: var=inventory_hostname
''',
'serial_pct.yml':
'''
- hosts: all
gather_facts: no
serial: 20%
tasks:
- debug: var=inventory_hostname
''',
})
mock_inventory = MagicMock()
mock_var_manager = MagicMock()
# fake out options to use the syntax CLI switch, which will ensure
# the PlaybookExecutor doesn't create a TaskQueueManager
mock_options = MagicMock()
mock_options.syntax.value = True
pbe = PlaybookExecutor(
playbooks=['no_serial.yml', 'serial_int.yml', 'serial_pct.yml'],
inventory=mock_inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
options=mock_options,
passwords=[],
)
playbook = Playbook.load(pbe._playbooks[0],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(pbe._get_serialized_batches(play), [[
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]])
playbook = Playbook.load(pbe._playbooks[1],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']])
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']])
# Test when serial percent is under 1.0
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
self.assertEqual(pbe._get_serialized_batches(play),
[['host0'], ['host1'], ['host2']])
# Test when there is a remainder for serial as a percent
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9', 'host10'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9'], ['host10']])
def _run_playbook(self, yamlfile, is_checksyntax=False):
if not is_checksyntax :
if self.options.flush_cache:
self._flush_cache()
self._loading_callback(yamlfile)
# self._unreachable_hosts = dict()
tqm = TaskQueueManager(
inventory=self.inventory,
variable_manager=self.variable_manager,
loader=self.loader,
options=self.options,
passwords=self.passwords,
stdout_callback=self.callback,
)
check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE)
else :
tqm = None
try :
pb = Playbook.load(
yamlfile,
variable_manager=self.variable_manager,
loader=self.loader
)
self.inventory.set_playbook_basedir(yamlfile)
if tqm is not None :
tqm.load_callbacks()
tqm.send_callback('v2_playbook_on_start', pb)
self.logger.debug(self.log_prefix + '任务开始执行')
plays = pb.get_plays()
for play in plays:
if play._included_path is not None:
self.loader.set_basedir(play._included_path)
else:
self.loader.set_basedir(pb._basedir)
self.inventory.remove_restriction()
all_vars = self.variable_manager.get_vars(loader=self.loader, play=play)
templar = Templar(loader=self.loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if is_checksyntax or tqm is None:
return True
# tqm._unreachable_hosts.update(self._unreachable_hosts)
batches = self._get_serialized_batches(new_play)
if len(batches) == 0:
tqm.send_callback('v2_playbook_on_play_start', new_play)
tqm.send_callback('v2_playbook_on_no_hosts_matched')
self.logger.debug(self.log_prefix + '任务开始执行，但没有匹配主机')
# 需要写日志,需要写该模块
continue
for batch in batches:
self.inventory.restrict_to_hosts(batch)
try :
tqm.run(play)
except Exception as e:
print(e)
pass
# self._unreachable_hosts.update(tqm._unreachable_hosts)
tqm.send_callback('v2_playbook_on_stats', tqm._stats)
self.logger.debug(self.log_prefix + '任务执行完比')
# self.host_list = self.inventory.get_hosts(play.hosts)
tqm.cleanup()
self.loader.cleanup_all_tmp_files()
return True
except Exception as e:
print(e)
if is_checksyntax :
return False
else :
return False
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude,
self).load_data(ds, variable_manager, loader)
all_vars = self.vars.copy()
if variable_manager:
all_vars.update(variable_manager.get_vars(loader=loader))
templar = Templar(loader=loader, variables=all_vars)
try:
forward_conditional = False
if not new_obj.evaluate_conditional(templar=templar,
all_vars=all_vars):
return None
except AnsibleError:
# conditional evaluation raised an error, so we set a flag to indicate
# we need to forward the conditionals on to the included play(s)
forward_conditional = True
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = templar.template(new_obj.include)
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name,
variable_manager=variable_manager)
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(file_name)
# Check to see if we need to forward the conditionals on to the included
# plays. If so, we can take a shortcut here and simply prepend them to
# those attached to each block (if any)
if forward_conditional:
for task_block in entry.tasks:
task_block.when = self.when[:] + task_block.when
return pb
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
self._inventory.set_playbook_basedir(os.path.dirname(playbook_path))
i = 1
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
for play in pb.get_plays():
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
self._tqm._unreachable_hosts.update(self._unreachable_hosts)
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback('v2_playbook_on_play_start', new_play)
self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts)
if new_play.any_errors_fatal and failed_hosts_count > 0:
break
elif new_play.max_fail_percentage is not None and \
int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0):
break
elif len(batch) == failed_hosts_count:
break
# clear the failed hosts dictionaires in the TQM for the next batch
self._unreachable_hosts.update(self._tqm._unreachable_hosts)
self._tqm.clear_failed_hosts()
# if the last result wasn't zero or 3 (some hosts were unreachable),
# break out of the serial batch loop
if result not in (0, 3):
break
i = i + 1 # per play
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
finally:
if self._tqm is not None:
self._cleanup()
return result
def test_play_iterator(self):
# import epdb; epdb.st()
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
gather_facts: false
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
- block:
- debug: msg="this is a block task"
- block:
- debug: msg="this is a sub-block in a block"
rescue:
- debug: msg="this is a rescue task"
- block:
- debug: msg="this is a sub-block in a rescue"
always:
- debug: msg="this is an always task"
- block:
- debug: msg="this is a sub-block in an always"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml': """
- name: role task
debug: msg="this is a role task"
- block:
- name: role block task
debug: msg="inside block in role"
always:
- name: role always task
debug: msg="always task in block in role"
- include: foo.yml
- name: role task after include
debug: msg="after include in role"
- block:
- name: starting role nested block 1
debug:
- block:
- name: role nested block 1 task 1
debug:
- name: role nested block 1 task 2
debug:
- name: role nested block 1 task 3
debug:
- name: end of role nested block 1
debug:
- name: starting role nested block 2
debug:
- block:
- name: role nested block 2 task 1
debug:
- name: role nested block 2 task 2
debug:
- name: role nested block 2 task 3
debug:
- name: end of role nested block 2
debug:
""",
'/etc/ansible/roles/test_role/tasks/foo.yml': """
- name: role included task
debug: msg="this is task in an include from a role"
"""
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
mock_var_manager._fact_cache['host00'] = dict()
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# pre task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# role task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.name, "role task")
self.assertIsNotNone(task._role)
# role block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role block task")
self.assertIsNotNone(task._role)
# role block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role always task")
self.assertIsNotNone(task._role)
# role include task
# (host_state, task) = itr.get_next_task_for_host(hosts[0])
# self.assertIsNotNone(task)
# self.assertEqual(task.action, 'debug')
# self.assertEqual(task.name, "role included task")
# self.assertIsNotNone(task._role)
# role task after include
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role task after include")
self.assertIsNotNone(task._role)
# role nested block tasks
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "starting role nested block 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 1 task 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 1 task 2")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 1 task 3")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "end of role nested block 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "starting role nested block 2")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 2 task 1")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 2 task 2")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "role nested block 2 task 3")
self.assertIsNotNone(task._role)
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.name, "end of role nested block 2")
self.assertIsNotNone(task._role)
# regular play task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNone(task._role)
# block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a block task"))
# sub-block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
# mark the host failed
itr.mark_host_failed(hosts[0])
# block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a rescue task"))
# sub-block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
# block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is an always task"))
# sub-block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# post task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
# host 0 shouldn't be in the failed hosts, as the error
# was handled by a rescue block
failed_hosts = itr.get_failed_hosts()
self.assertNotIn(hosts[0], failed_hosts)
# Object supplying methods to load JSON or YAML from file or string, into a Python data structure.
loader = DataLoader()
# https://github.com/ansible/ansible/blob/devel/lib/ansible/inventory/__init__.py
#
# Builds the inventory as a data structure (including groups), object also caches host and group vars.
# Loader supplies JSON/YAML parsing methods, variable_manager allows object to copy (cache) vars data
inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list='localhost')
# Here we simply copy back the inventory data into the central variable_manager.
variable_manager.set_inventory(inventory)
# https://github.com/ansible/ansible/blob/devel/lib/ansible/playbook/__init__.py
#
# We use the Playbook object to load and then create Play objects from YAML files. Playbook
# will load and parse the YAML data into the Playbook.entries[] list.
# Playbook.load() is a static method that calls Playbook.__init__
playbook = Playbook.load("test/site.yml", variable_manager, loader)
# Now we have the playbook loaded we can loop through the plays. A play is a list of roles
# and/or tasks/handler blocks to execute on a set of hosts.
# see definition: https://github.com/ansible/ansible/blob/devel/lib/ansible/playbook/play.py
for play in playbook.get_plays():
for block in play.compile():
if block.has_tasks():
for task in block.block:
if task._role in play.get_roles():
changed = str(task._get_parent_attribute('changed_when'))
failed = str(task._get_parent_attribute('failed_when'))
print("PLAY: %s, ROLE: %s, %s, changed_when: %s, failed_when: %s" % (play.get_name(), task._role, task, changed, failed))
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
entrylist = []
entry = {}
try:
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader)
self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
i = 1
plays = pb.get_plays()
display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path)))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
if play.vars_prompt:
for var in play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = var.get("private", True)
confirm = var.get("confirm", False)
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
if vname not in self._variable_manager.extra_vars:
if self._tqm:
self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default)
play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default)
else: # we are either in --list-<option> or syntax check
play.vars[vname] = default
# Create a temporary copy of the play here, so we can run post_validate
# on it without the templating changes affecting the original object.
all_vars = self._variable_manager.get_vars(loader=self._loader, play=play)
templar = Templar(loader=self._loader, variables=all_vars)
new_play = play.copy()
new_play.post_validate(templar)
if self._options.syntax:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(new_play)
else:
self._tqm._unreachable_hosts.update(self._unreachable_hosts)
previously_failed = len(self._tqm._failed_hosts)
previously_unreachable = len(self._tqm._unreachable_hosts)
break_play = False
# we are actually running plays
for batch in self._get_serialized_batches(new_play):
if len(batch) == 0:
self._tqm.send_callback('v2_playbook_on_play_start', new_play)
self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
break
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# break the play if the result equals the special return code
if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
result = self._tqm.RUN_FAILED_HOSTS
break_play = True
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
(previously_failed + previously_unreachable)
if len(batch) == failed_hosts_count:
break_play = True
break
# update the previous counts so they don't accumulate incorrectly
# over multiple serial batches
previously_failed += len(self._tqm._failed_hosts) - previously_failed
previously_unreachable += len(self._tqm._unreachable_hosts) - previously_unreachable
# save the unreachable hosts from this batch
self._unreachable_hosts.update(self._tqm._unreachable_hosts)
if break_play:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# send the stats callback for this playbook
if self._tqm is not None:
if C.RETRY_FILES_ENABLED:
retries = set(self._tqm._failed_hosts.keys())
retries.update(self._tqm._unreachable_hosts.keys())
retries = sorted(retries)
if len(retries) > 0:
if C.RETRY_FILES_SAVE_PATH:
basedir = C.shell_expand(C.RETRY_FILES_SAVE_PATH)
elif playbook_path:
basedir = os.path.dirname(playbook_path)
else:
basedir = '~/'
(retry_name, _) = os.path.splitext(os.path.basename(playbook_path))
filename = os.path.join(basedir, "%s.retry" % retry_name)
if self._generate_retry_inventory(filename, retries):
display.display("\tto retry, use: --limit @%s\n" % filename)
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._tqm.cleanup()
if self._loader:
self._loader.cleanup_all_tmp_files()
if self._options.syntax:
display.display("No issues encountered")
return result
return result
def test_noop(self):
fake_loader = DictDataLoader({
"test_play.yml":
"""
- hosts: localhost
gather_facts: no
tasks:
- name: simple_function
debug:
msg : 'hello world'
tags:
- functional
- name: call_simple_function
call_task:
name : "simple_function"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml',
loader=fake_loader,
variable_manager=mock_var_manager)
inventory = MagicMock()
inventory.hosts = {}
hosts = []
host = MagicMock()
host.name = host.get_name.return_value = 'localhost'
hosts.append(host)
inventory.hosts[host.name] = host
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
mock_var_manager._fact_cache['localhost'] = dict()
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
passwords=None,
forks=5,
)
tqm._initialize_processes(3)
strategy = StrategyModule(tqm)
strategy._hosts_cache = [h.name for h in hosts]
strategy._hosts_cache_all = [h.name for h in hosts]
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
def test_play_iterator_nested_blocks(self):
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
gather_facts: false
tasks:
- block:
- block:
- block:
- block:
- block:
- debug: msg="this is the first task"
- ping:
rescue:
- block:
- block:
- block:
- block:
- debug: msg="this is the rescue task"
always:
- block:
- block:
- block:
- block:
- debug: msg="this is the always task"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
# get the first task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg='this is the first task'))
# fail the host
itr.mark_host_failed(hosts[0])
# get the resuce task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg='this is the rescue task'))
# get the always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg='this is the always task'))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
self.assertEqual(task.args, dict(_raw_params='flush_handlers'))
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
def test_play_iterator(self):
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
gather_facts: false
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml': """
- debug: msg="this is a role task"
""",
})
p = Playbook.load('test_play.yml', loader=fake_loader)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# pre task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# role task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNotNone(task._role)
# regular play task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNone(task._role)
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# post task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
def test_play_iterator(self):
#import epdb; epdb.st()
fake_loader = DictDataLoader({
"test_play.yml": """
- hosts: all
gather_facts: false
roles:
- test_role
pre_tasks:
- debug: msg="this is a pre_task"
tasks:
- debug: msg="this is a regular task"
- block:
- debug: msg="this is a block task"
- block:
- debug: msg="this is a sub-block in a block"
rescue:
- debug: msg="this is a rescue task"
- block:
- debug: msg="this is a sub-block in a rescue"
always:
- debug: msg="this is an always task"
- block:
- debug: msg="this is a sub-block in an always"
post_tasks:
- debug: msg="this is a post_task"
""",
'/etc/ansible/roles/test_role/tasks/main.yml': """
- debug: msg="this is a role task"
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml', loader=fake_loader, variable_manager=mock_var_manager)
hosts = []
for i in range(0, 10):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
mock_var_manager._fact_cache['host00'] = dict()
inventory = MagicMock()
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
# lookup up an original task
target_task = p._entries[0].tasks[0].block[0]
task_copy = target_task.copy(exclude_block=True)
found_task = itr.get_original_task(hosts[0], task_copy)
self.assertEqual(target_task, found_task)
bad_task = Task()
found_task = itr.get_original_task(hosts[0], bad_task)
self.assertIsNone(found_task)
# pre task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# role task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNotNone(task._role)
# regular play task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertIsNone(task._role)
# block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a block task"))
# sub-block task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a block"))
# mark the host failed
itr.mark_host_failed(hosts[0])
# block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a rescue task"))
# sub-block rescue task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in a rescue"))
# block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is an always task"))
# sub-block always task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
self.assertEqual(task.args, dict(msg="this is a sub-block in an always"))
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# post task
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'debug')
# implicit meta: flush_handlers
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNotNone(task)
self.assertEqual(task.action, 'meta')
# end of iteration
(host_state, task) = itr.get_next_task_for_host(hosts[0])
self.assertIsNone(task)
# host 0 shouldn't be in the failed hosts, as the error
# was handled by a rescue block
failed_hosts = itr.get_failed_hosts()
self.assertNotIn(hosts[0], failed_hosts)
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from ansible.playbook import Playbook
from ansible.playbook.play import Play
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude,
self).load_data(ds, variable_manager, loader)
all_vars = self.vars.copy()
if variable_manager:
all_vars.update(variable_manager.get_vars())
templar = Templar(loader=loader, variables=all_vars)
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = templar.template(new_obj.import_playbook)
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name,
variable_manager=variable_manager,
vars=self.vars.copy())
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
# conditional includes on a playbook need a marker to skip gathering
if new_obj.when and isinstance(entry, Play):
entry._included_conditional = new_obj.when[:]
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(file_name)
# Check to see if we need to forward the conditionals on to the included
# plays. If so, we can take a shortcut here and simply prepend them to
# those attached to each block (if any)
if new_obj.when:
for task_block in (entry.pre_tasks + entry.roles +
entry.tasks + entry.post_tasks):
task_block._attributes[
'when'] = new_obj.when[:] + task_block.when[:]
return pb