def test_basic_playbook(self):
fake_loader = DictDataLoader({
"test_file.yml":
"""
- hosts: all
""",
})
p = Playbook.load("test_file.yml", loader=fake_loader)
plays = p.get_plays()
def setUp(self):
inventory = MagicMock()
variable_manager = MagicMock()
loader = MagicMock()
passwords = []
# Reset the stored command line args
co.GlobalCLIArgs._Singleton__instance = None
self._tqm = TaskQueueManager(inventory, variable_manager, loader,
passwords)
self._playbook = Playbook(loader)
# we use a MagicMock to register the result of the call we
# expect to `v2_playbook_on_call`. We don't mock out the
# method since we're testing code that uses `inspect` to
# look at that method's argspec and we want to ensure this
# test is easy to reason about.
self._register = MagicMock()
def run(self):
''' create and execute the single task playbook '''
super(AdHocCLI, self).run()
# only thing left should be host pattern
pattern = to_text(context.CLIARGS['args'],
errors='surrogate_or_strict')
sshpass = None
becomepass = None
(sshpass, becomepass) = self.ask_passwords()
passwords = {'conn_pass': sshpass, 'become_pass': becomepass}
# get basic objects
loader, inventory, variable_manager = self._play_prereqs()
try:
hosts = self.get_host_list(inventory, context.CLIARGS['subset'],
pattern)
except AssibleError:
if context.CLIARGS['subset']:
raise
else:
hosts = []
display.warning("No hosts matched, nothing to do")
if context.CLIARGS['listhosts']:
display.display(' hosts (%d):' % len(hosts))
for host in hosts:
display.display(' %s' % host)
return 0
if context.CLIARGS[
'module_name'] in C.MODULE_REQUIRE_ARGS and not context.CLIARGS[
'module_args']:
err = "No argument passed to %s module" % context.CLIARGS[
'module_name']
if pattern.endswith(".yml"):
err = err + ' (did you mean to run assible-playbook?)'
raise AssibleOptionsError(err)
# Avoid modules that don't work with ad-hoc
if context.CLIARGS['module_name'] in ('import_playbook', ):
raise AssibleOptionsError(
"'%s' is not a valid action for ad-hoc commands" %
context.CLIARGS['module_name'])
play_ds = self._play_ds(pattern, context.CLIARGS['seconds'],
context.CLIARGS['poll_interval'])
play = Play().load(play_ds,
variable_manager=variable_manager,
loader=loader)
# used in start callback
playbook = Playbook(loader)
playbook._entries.append(play)
playbook._file_name = '__adhoc_playbook__'
if self.callback:
cb = self.callback
elif context.CLIARGS['one_line']:
cb = 'oneline'
# Respect custom 'stdout_callback' only with enabled 'bin_assible_callbacks'
elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default':
cb = C.DEFAULT_STDOUT_CALLBACK
else:
cb = 'minimal'
run_tree = False
if context.CLIARGS['tree']:
C.DEFAULT_CALLBACK_WHITELIST.append('tree')
C.TREE_DIR = context.CLIARGS['tree']
run_tree = True
# now create a task queue manager to execute the play
self._tqm = None
try:
self._tqm = TaskQueueManager(
inventory=inventory,
variable_manager=variable_manager,
loader=loader,
passwords=passwords,
stdout_callback=cb,
run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS,
run_tree=run_tree,
forks=context.CLIARGS['forks'],
)
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', playbook)
result = self._tqm.run(play)
self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats)
finally:
if self._tqm:
self._tqm.cleanup()
if loader:
loader.cleanup_all_tmp_files()
return result
def load_data(self, ds, basedir, variable_manager=None, loader=None):
'''
Overrides the base load_data(), as we're actually going to return a new
Playbook() object rather than a PlaybookInclude object
'''
# import here to avoid a dependency loop
from assible.playbook import Playbook
from assible.playbook.play import Play
# first, we use the original parent method to correctly load the object
# via the load_data/preprocess_data system we normally use for other
# playbook objects
new_obj = super(PlaybookInclude,
self).load_data(ds, variable_manager, loader)
all_vars = self.vars.copy()
if variable_manager:
all_vars.update(variable_manager.get_vars())
templar = Templar(loader=loader, variables=all_vars)
# then we use the object to load a Playbook
pb = Playbook(loader=loader)
file_name = templar.template(new_obj.import_playbook)
if not os.path.isabs(file_name):
file_name = os.path.join(basedir, file_name)
pb._load_playbook_data(file_name=file_name,
variable_manager=variable_manager,
vars=self.vars.copy())
# finally, update each loaded playbook entry with any variables specified
# on the included playbook and/or any tags which may have been set
for entry in pb._entries:
# conditional includes on a playbook need a marker to skip gathering
if new_obj.when and isinstance(entry, Play):
entry._included_conditional = new_obj.when[:]
temp_vars = entry.vars.copy()
temp_vars.update(new_obj.vars)
param_tags = temp_vars.pop('tags', None)
if param_tags is not None:
entry.tags.extend(param_tags.split(','))
entry.vars = temp_vars
entry.tags = list(set(entry.tags).union(new_obj.tags))
if entry._included_path is None:
entry._included_path = os.path.dirname(file_name)
# Check to see if we need to forward the conditionals on to the included
# plays. If so, we can take a shortcut here and simply prepend them to
# those attached to each block (if any)
if new_obj.when:
for task_block in (entry.pre_tasks + entry.roles +
entry.tasks + entry.post_tasks):
task_block._attributes[
'when'] = new_obj.when[:] + task_block.when[:]
return pb
def test_get_serialized_batches(self):
fake_loader = DictDataLoader({
'no_serial.yml':
'''
- hosts: all
gather_facts: no
tasks:
- debug: var=inventory_hostname
''',
'serial_int.yml':
'''
- hosts: all
gather_facts: no
serial: 2
tasks:
- debug: var=inventory_hostname
''',
'serial_pct.yml':
'''
- hosts: all
gather_facts: no
serial: 20%
tasks:
- debug: var=inventory_hostname
''',
'serial_list.yml':
'''
- hosts: all
gather_facts: no
serial: [1, 2, 3]
tasks:
- debug: var=inventory_hostname
''',
'serial_list_mixed.yml':
'''
- hosts: all
gather_facts: no
serial: [1, "20%", -1]
tasks:
- debug: var=inventory_hostname
''',
})
mock_inventory = MagicMock()
mock_var_manager = MagicMock()
templar = Templar(loader=fake_loader)
pbe = PlaybookExecutor(
playbooks=[
'no_serial.yml', 'serial_int.yml', 'serial_pct.yml',
'serial_list.yml', 'serial_list_mixed.yml'
],
inventory=mock_inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
passwords=[],
)
playbook = Playbook.load(pbe._playbooks[0],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(pbe._get_serialized_batches(play), [[
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]])
playbook = Playbook.load(pbe._playbooks[1],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']])
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9']])
playbook = Playbook.load(pbe._playbooks[3],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0'], ['host1', 'host2'], ['host3', 'host4', 'host5'],
['host6', 'host7', 'host8'], ['host9']])
playbook = Playbook.load(pbe._playbooks[4],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0'], ['host1', 'host2'],
['host3', 'host4', 'host5', 'host6', 'host7', 'host8', 'host9']])
# Test when serial percent is under 1.0
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = ['host0', 'host1', 'host2']
self.assertEqual(pbe._get_serialized_batches(play),
[['host0'], ['host1'], ['host2']])
# Test when there is a remainder for serial as a percent
playbook = Playbook.load(pbe._playbooks[2],
variable_manager=mock_var_manager,
loader=fake_loader)
play = playbook.get_plays()[0]
play.post_validate(templar)
mock_inventory.get_hosts.return_value = [
'host0', 'host1', 'host2', 'host3', 'host4', 'host5', 'host6',
'host7', 'host8', 'host9', 'host10'
]
self.assertEqual(
pbe._get_serialized_batches(play),
[['host0', 'host1'], ['host2', 'host3'], ['host4', 'host5'],
['host6', 'host7'], ['host8', 'host9'], ['host10']])
def test_empty_playbook(self):
fake_loader = DictDataLoader({})
p = Playbook(loader=fake_loader)
def run(self):
'''
Run the given playbook, based on the settings in the play which
may limit the runs to serialized groups, etc.
'''
result = 0
entrylist = []
entry = {}
try:
# preload become/connection/shell to set config defs cached
list(connection_loader.all(class_only=True))
list(shell_loader.all(class_only=True))
list(become_loader.all(class_only=True))
for playbook_path in self._playbooks:
pb = Playbook.load(playbook_path,
variable_manager=self._variable_manager,
loader=self._loader)
# FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path)))
if self._tqm is None: # we are doing a listing
entry = {'playbook': playbook_path}
entry['plays'] = []
else:
# make sure the tqm has callbacks loaded
self._tqm.load_callbacks()
self._tqm.send_callback('v2_playbook_on_start', pb)
i = 1
plays = pb.get_plays()
display.vv(u'%d plays in %s' %
(len(plays), to_text(playbook_path)))
for play in plays:
if play._included_path is not None:
self._loader.set_basedir(play._included_path)
else:
self._loader.set_basedir(pb._basedir)
# clear any filters which may have been applied to the inventory
self._inventory.remove_restriction()
# Allow variables to be used in vars_prompt fields.
all_vars = self._variable_manager.get_vars(play=play)
templar = Templar(loader=self._loader, variables=all_vars)
setattr(play, 'vars_prompt',
templar.template(play.vars_prompt))
# FIXME: this should be a play 'sub object' like loop_control
if play.vars_prompt:
for var in play.vars_prompt:
vname = var['name']
prompt = var.get("prompt", vname)
default = var.get("default", None)
private = boolean(var.get("private", True))
confirm = boolean(var.get("confirm", False))
encrypt = var.get("encrypt", None)
salt_size = var.get("salt_size", None)
salt = var.get("salt", None)
unsafe = var.get("unsafe", None)
if vname not in self._variable_manager.extra_vars:
if self._tqm:
self._tqm.send_callback(
'v2_playbook_on_vars_prompt', vname,
private, prompt, encrypt, confirm,
salt_size, salt, default, unsafe)
play.vars[vname] = display.do_var_prompt(
vname, private, prompt, encrypt,
confirm, salt_size, salt, default,
unsafe)
else: # we are either in --list-<option> or syntax check
play.vars[vname] = default
# Post validate so any play level variables are templated
all_vars = self._variable_manager.get_vars(play=play)
templar = Templar(loader=self._loader, variables=all_vars)
play.post_validate(templar)
if context.CLIARGS['syntax']:
continue
if self._tqm is None:
# we are just doing a listing
entry['plays'].append(play)
else:
self._tqm._unreachable_hosts.update(
self._unreachable_hosts)
previously_failed = len(self._tqm._failed_hosts)
previously_unreachable = len(
self._tqm._unreachable_hosts)
break_play = False
# we are actually running plays
batches = self._get_serialized_batches(play)
if len(batches) == 0:
self._tqm.send_callback(
'v2_playbook_on_play_start', play)
self._tqm.send_callback(
'v2_playbook_on_no_hosts_matched')
for batch in batches:
# restrict the inventory to the hosts in the serialized batch
self._inventory.restrict_to_hosts(batch)
# and run it...
result = self._tqm.run(play=play)
# break the play if the result equals the special return code
if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0:
result = self._tqm.RUN_FAILED_HOSTS
break_play = True
# check the number of failures here, to see if they're above the maximum
# failure percentage allowed, or if any errors are fatal. If either of those
# conditions are met, we break out, otherwise we only break out if the entire
# batch failed
failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \
(previously_failed + previously_unreachable)
if len(batch) == failed_hosts_count:
break_play = True
break
# update the previous counts so they don't accumulate incorrectly
# over multiple serial batches
previously_failed += len(
self._tqm._failed_hosts) - previously_failed
previously_unreachable += len(
self._tqm._unreachable_hosts
) - previously_unreachable
# save the unreachable hosts from this batch
self._unreachable_hosts.update(
self._tqm._unreachable_hosts)
if break_play:
break
i = i + 1 # per play
if entry:
entrylist.append(entry) # per playbook
# send the stats callback for this playbook
if self._tqm is not None:
if C.RETRY_FILES_ENABLED:
retries = set(self._tqm._failed_hosts.keys())
retries.update(self._tqm._unreachable_hosts.keys())
retries = sorted(retries)
if len(retries) > 0:
if C.RETRY_FILES_SAVE_PATH:
basedir = C.RETRY_FILES_SAVE_PATH
elif playbook_path:
basedir = os.path.dirname(
os.path.abspath(playbook_path))
else:
basedir = '~/'
(retry_name, _) = os.path.splitext(
os.path.basename(playbook_path))
filename = os.path.join(basedir,
"%s.retry" % retry_name)
if self._generate_retry_inventory(
filename, retries):
display.display(
"\tto retry, use: --limit @%s\n" %
filename)
self._tqm.send_callback('v2_playbook_on_stats',
self._tqm._stats)
# if the last result wasn't zero, break out of the playbook file name loop
if result != 0:
break
if entrylist:
return entrylist
finally:
if self._tqm is not None:
self._tqm.cleanup()
if self._loader:
self._loader.cleanup_all_tmp_files()
if context.CLIARGS['syntax']:
display.display("No issues encountered")
return result
if context.CLIARGS['start_at_task'] and not self._tqm._start_at_done:
display.error(
"No matching task \"%s\" found."
" Note: --start-at-task can only follow static includes." %
context.CLIARGS['start_at_task'])
return result
def test_noop(self):
fake_loader = DictDataLoader({
"test_play.yml":
"""
- hosts: all
gather_facts: no
tasks:
- block:
- block:
- name: task1
debug: msg='task1'
failed_when: inventory_hostname == 'host01'
- name: task2
debug: msg='task2'
rescue:
- name: rescue1
debug: msg='rescue1'
- name: rescue2
debug: msg='rescue2'
""",
})
mock_var_manager = MagicMock()
mock_var_manager._fact_cache = dict()
mock_var_manager.get_vars.return_value = dict()
p = Playbook.load('test_play.yml',
loader=fake_loader,
variable_manager=mock_var_manager)
inventory = MagicMock()
inventory.hosts = {}
hosts = []
for i in range(0, 2):
host = MagicMock()
host.name = host.get_name.return_value = 'host%02d' % i
hosts.append(host)
inventory.hosts[host.name] = host
inventory.get_hosts.return_value = hosts
inventory.filter_hosts.return_value = hosts
mock_var_manager._fact_cache['host00'] = dict()
play_context = PlayContext(play=p._entries[0])
itr = PlayIterator(
inventory=inventory,
play=p._entries[0],
play_context=play_context,
variable_manager=mock_var_manager,
all_vars=dict(),
)
tqm = TaskQueueManager(
inventory=inventory,
variable_manager=mock_var_manager,
loader=fake_loader,
passwords=None,
forks=5,
)
tqm._initialize_processes(3)
strategy = StrategyModule(tqm)
strategy._hosts_cache = [h.name for h in hosts]
strategy._hosts_cache_all = [h.name for h in hosts]
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'meta')
# debug: task1, debug: task1
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'debug')
self.assertEqual(host2_task.action, 'debug')
self.assertEqual(host1_task.name, 'task1')
self.assertEqual(host2_task.name, 'task1')
# mark the second host failed
itr.mark_host_failed(hosts[1])
# debug: task2, meta: noop
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'debug')
self.assertEqual(host2_task.action, 'meta')
self.assertEqual(host1_task.name, 'task2')
self.assertEqual(host2_task.name, '')
# meta: noop, debug: rescue1
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'debug')
self.assertEqual(host1_task.name, '')
self.assertEqual(host2_task.name, 'rescue1')
# meta: noop, debug: rescue2
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'debug')
self.assertEqual(host1_task.name, '')
self.assertEqual(host2_task.name, 'rescue2')
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'meta')
# implicit meta: flush_handlers
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNotNone(host1_task)
self.assertIsNotNone(host2_task)
self.assertEqual(host1_task.action, 'meta')
self.assertEqual(host2_task.action, 'meta')
# end of iteration
hosts_left = strategy.get_hosts_left(itr)
hosts_tasks = strategy._get_next_task_lockstep(hosts_left, itr)
host1_task = hosts_tasks[0][1]
host2_task = hosts_tasks[1][1]
self.assertIsNone(host1_task)
self.assertIsNone(host2_task)