def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no', execution_id=None): """ :param tasks: [{'action': {'module': 'shell', 'args': 'ls'}, ...}, ] :param pattern: all, *, or others :param play_name: The play name :param gather_facts: :return: """ self.check_pattern(pattern) self.results_callback = self.get_result_callback(execution_id) cleaned_tasks = self.clean_tasks(tasks) self.set_control_master_if_need(cleaned_tasks) context.CLIARGS = ImmutableDict(self.options) play_source = dict(name=play_name, hosts=pattern, gather_facts=gather_facts, tasks=cleaned_tasks) play = Play().load( play_source, variable_manager=self.variable_manager, loader=self.loader, ) loader = DataLoader() # used in start callback playbook = Playbook(loader) playbook._entries.append(play) playbook._file_name = '__adhoc_playbook__' tqm = TaskQueueManager( inventory=self.inventory, variable_manager=self.variable_manager, loader=self.loader, stdout_callback=self.results_callback, passwords={"conn_pass": self.options.get("password", "")}) try: tqm.send_callback('v2_playbook_on_start', playbook) tqm.run(play) tqm.send_callback('v2_playbook_on_stats', tqm._stats) return self.results_callback except Exception as e: raise AnsibleError(e) finally: if tqm is not None: tqm.cleanup() shutil.rmtree(C.DEFAULT_LOCAL_TMP, True) self.results_callback.close()
def run(self, tasks, pattern, play_name='Ansible Ad-hoc', gather_facts='no'): """ :param tasks: [{'action': {'module': 'shell', 'args': 'ls'}, ...}, ] :param pattern: all, *, or others :param play_name: The play name :param gather_facts: :return: """ self.check_pattern(pattern) cleaned_tasks = self.clean_tasks(tasks) play_source = dict(name=play_name, hosts=pattern, gather_facts=gather_facts, tasks=cleaned_tasks) play = Play().load( play_source, variable_manager=self.variable_manager, loader=self.loader, ) tqm = TaskQueueManager( inventory=self.inventory, variable_manager=self.variable_manager, loader=self.loader, options=self.options, stdout_callback=self.results_callback, passwords=self.options.passwords, ) msg = ("Get matched hosts: {}".format( self.inventory.get_matched_hosts(pattern))) try: tqm.send_callback('on_playbook_start', play.name) self.results_callback.display(msg) tqm.run(play) return self.results_callback.results except Exception as e: raise AnsibleError(e) finally: tqm.send_callback('v2_playbook_on_stats', tqm._stats) tqm.send_callback('on_playbook_end', play.name) tqm.cleanup() self.loader.cleanup_all_tmp_files()
def _run_playbook(self, yamlfile, is_checksyntax=False): if not is_checksyntax : if self.options.flush_cache: self._flush_cache() self._loading_callback(yamlfile) # self._unreachable_hosts = dict() tqm = TaskQueueManager( inventory=self.inventory, variable_manager=self.variable_manager, loader=self.loader, options=self.options, passwords=self.passwords, stdout_callback=self.callback, ) check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE) else : tqm = None try : pb = Playbook.load( yamlfile, variable_manager=self.variable_manager, loader=self.loader ) self.inventory.set_playbook_basedir(yamlfile) if tqm is not None : tqm.load_callbacks() tqm.send_callback('v2_playbook_on_start', pb) self.logger.debug(self.log_prefix + '任务开始执行') plays = pb.get_plays() for play in plays: if play._included_path is not None: self.loader.set_basedir(play._included_path) else: self.loader.set_basedir(pb._basedir) self.inventory.remove_restriction() all_vars = self.variable_manager.get_vars(loader=self.loader, play=play) templar = Templar(loader=self.loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if is_checksyntax or tqm is None: return True # tqm._unreachable_hosts.update(self._unreachable_hosts) batches = self._get_serialized_batches(new_play) if len(batches) == 0: tqm.send_callback('v2_playbook_on_play_start', new_play) tqm.send_callback('v2_playbook_on_no_hosts_matched') self.logger.debug(self.log_prefix + '任务开始执行,但没有匹配主机') # 需要写日志,需要写该模块 continue for batch in batches: self.inventory.restrict_to_hosts(batch) try : tqm.run(play) except Exception as e: print(e) pass # self._unreachable_hosts.update(tqm._unreachable_hosts) tqm.send_callback('v2_playbook_on_stats', tqm._stats) self.logger.debug(self.log_prefix + '任务执行完比') # self.host_list = self.inventory.get_hosts(play.hosts) tqm.cleanup() self.loader.cleanup_all_tmp_files() return True except Exception as e: print(e) if is_checksyntax : return False else : return False
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, display, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self.passwords = passwords # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, display=display, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' signal.signal(signal.SIGINT, self._cleanup) result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir( os.path.dirname(playbook_path)) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] i = 1 plays = pb.get_plays() self._display.vv('%d plays in %s' % (len(plays), playbook_path)) for play in plays: # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: if 'name' not in var: raise AnsibleError( "'vars_prompt' item is missing 'name:'", obj=play._ds) vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = var.get("private", True) confirm = var.get("confirm", False) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in play.vars: self._tqm.send_callback( 'v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = self._do_var_prompt( vname, private, prompt, encrypt, confirm, salt_size, salt, default) # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars( loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback( 'v2_playbook_on_play_start', new_play) self._tqm.send_callback( 'v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # if the last result wasn't zero, break out of the serial batch loop if result != 0: break # if the last result wasn't zero, break out of the play loop if result != 0: break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._cleanup() if self._options.syntax: self.display.display("No issues encountered") return result # FIXME: this stat summary stuff should be cleaned up and moved # to a new method, if it even belongs here... self._display.banner("PLAY RECAP") hosts = sorted(self._tqm._stats.processed.keys()) for h in hosts: t = self._tqm._stats.summarize(h) self._display.display( "%s : %s %s %s %s" % (hostcolor(h, t), colorize('ok', t['ok'], 'green'), colorize('changed', t['changed'], 'yellow'), colorize('unreachable', t['unreachable'], 'red'), colorize('failed', t['failures'], 'red')), screen_only=True) self._display.display( "%s : %s %s %s %s" % (hostcolor(h, t, False), colorize('ok', t['ok'], None), colorize('changed', t['changed'], None), colorize('unreachable', t['unreachable'], None), colorize('failed', t['failures'], None)), log_only=True) self._display.display("", screen_only=True) # END STATS STUFF return result def _cleanup(self, signum=None, framenum=None): return self._tqm.cleanup() def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, basestring) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%", "")) serial = int((serial_pct / 100.0) * len(all_hosts)) else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches def _do_var_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): if sys.__stdin__.isatty(): if prompt and default is not None: msg = "%s [%s]: " % (prompt, default) elif prompt: msg = "%s: " % prompt else: msg = 'input for %s: ' % varname def do_prompt(prompt, private): if sys.stdout.encoding: msg = prompt.encode(sys.stdout.encoding) else: # when piping the output, or at other times when stdout # may not be the standard file descriptor, the stdout # encoding may not be set, so default to something sane msg = prompt.encode(locale.getpreferredencoding()) if private: return getpass.getpass(msg) return raw_input(msg) if confirm: while True: result = do_prompt(msg, private) second = do_prompt("confirm " + msg, private) if result == second: break self._display.display( "***** VALUES ENTERED DO NOT MATCH ****") else: result = do_prompt(msg, private) else: result = None self._display.warning( "Not prompting as we are not in interactive mode") # if result is false and default is not None if not result and default is not None: result = default if encrypt: result = do_encrypt(result, encrypt, salt_size, salt) # handle utf-8 chars result = to_unicode(result, errors='strict') return result
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path))) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) i = 1 plays = pb.get_plays() display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path))) for play in plays: if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = var.get("private", True) confirm = var.get("confirm", False) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in self._variable_manager.extra_vars: if self._tqm: self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default) else: # we are either in --list-<option> or syntax check play.vars[vname] = default # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: self._tqm._unreachable_hosts.update(self._unreachable_hosts) previously_failed = len(self._tqm._failed_hosts) previously_unreachable = len(self._tqm._unreachable_hosts) break_play = False # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # break the play if the result equals the special return code if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0: result = self._tqm.RUN_FAILED_HOSTS break_play = True # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \ (previously_failed + previously_unreachable) if len(batch) == failed_hosts_count: break_play = True break # update the previous counts so they don't accumulate incorrectly # over multiple serial batches previously_failed += len(self._tqm._failed_hosts) - previously_failed previously_unreachable += len(self._tqm._unreachable_hosts) - previously_unreachable # save the unreachable hosts from this batch self._unreachable_hosts.update(self._tqm._unreachable_hosts) if break_play: break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # send the stats callback for this playbook if self._tqm is not None: if C.RETRY_FILES_ENABLED: retries = set(self._tqm._failed_hosts.keys()) retries.update(self._tqm._unreachable_hosts.keys()) retries = sorted(retries) if len(retries) > 0: if C.RETRY_FILES_SAVE_PATH: basedir = C.shell_expand(C.RETRY_FILES_SAVE_PATH) elif playbook_path: basedir = os.path.dirname(playbook_path) else: basedir = '~/' (retry_name, _) = os.path.splitext(os.path.basename(playbook_path)) filename = os.path.join(basedir, "%s.retry" % retry_name) if self._generate_retry_inventory(filename, retries): display.display("\tto retry, use: --limit @%s\n" % filename) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._tqm.cleanup() if self._loader: self._loader.cleanup_all_tmp_files() if self._options.syntax: display.display("No issues encountered") return result return result def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) all_hosts_len = len(all_hosts) # the serial value can be listed as a scalar or a list of # scalars, so we make sure it's a list here serial_batch_list = play.serial if len(serial_batch_list) == 0: serial_batch_list = [-1] cur_item = 0 serialized_batches = [] while len(all_hosts) > 0: # get the serial value from current item in the list serial = pct_to_int(serial_batch_list[cur_item], all_hosts_len) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise grab a chunk of the hosts equal # to the current serial item size if serial <= 0: serialized_batches.append(all_hosts) break else: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) # increment the current batch list item number, and if we've hit # the end keep using the last element until we've consumed all of # the hosts in the inventory cur_item += 1 if cur_item > len(serial_batch_list) - 1: cur_item = len(serial_batch_list) - 1 return serialized_batches def _generate_retry_inventory(self, retry_path, replay_hosts): ''' Called when a playbook run fails. It generates an inventory which allows re-running on ONLY the failed hosts. This may duplicate some variable information in group_vars/host_vars but that is ok, and expected. ''' try: makedirs_safe(os.path.dirname(retry_path)) with open(retry_path, 'w') as fd: for x in replay_hosts: fd.write("%s\n" % x) except Exception as e: display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_str(e))) return False return True
class TestTaskQueueManagerCallbacks(unittest.TestCase): def setUp(self): inventory = MagicMock() variable_manager = MagicMock() loader = MagicMock() options = MagicMock() passwords = [] self._tqm = TaskQueueManager(inventory, variable_manager, loader, options, passwords) self._playbook = Playbook(loader) # we use a MagicMock to register the result of the call we # expect to `v2_playbook_on_call`. We don't mock out the # method since we're testing code that uses `inspect` to # look at that method's argspec and we want to ensure this # test is easy to reason about. self._register = MagicMock() def tearDown(self): pass def test_task_queue_manager_callbacks_v2_playbook_on_start_legacy(self): """ Assert that no exceptions are raised when sending a Playbook start callback to a legacy callback module plugin. """ register = self._register class LegacyCallbackModule(CallbackBase): """ This is a callback module with the legacy method signature for `v2_playbook_on_start`. """ CALLBACK_VERSION = 2.0 CALLBACK_TYPE = 'notification' CALLBACK_NAME = 'legacy_module' def v2_playbook_on_start(self): register(self) callback_module = LegacyCallbackModule() self._tqm._callback_plugins.append(callback_module) self._tqm.send_callback('v2_playbook_on_start', self._playbook) register.assert_called_once_with(callback_module) def test_task_queue_manager_callbacks_v2_playbook_on_start(self): """ Assert that no exceptions are raised when sending a Playbook start callback to a current callback module plugin. """ register = self._register class CallbackModule(CallbackBase): """ This is a callback module with the current method signature for `v2_playbook_on_start`. """ CALLBACK_VERSION = 2.0 CALLBACK_TYPE = 'notification' CALLBACK_NAME = 'current_module' def v2_playbook_on_start(self, playbook): register(self, playbook) callback_module = CallbackModule() self._tqm._callback_plugins.append(callback_module) self._tqm.send_callback('v2_playbook_on_start', self._playbook) register.assert_called_once_with(callback_module, self._playbook) def test_task_queue_manager_callbacks_v2_playbook_on_start_wrapped(self): """ Assert that no exceptions are raised when sending a Playbook start callback to a wrapped current callback module plugin. """ register = self._register def wrap_callback(func): """ This wrapper changes the exposed argument names for a method from the original names to (*args, **kwargs). This is used in order to validate that wrappers which change par- ameter names do not break the TQM callback system. :param func: function to decorate :return: decorated function """ def wrapper(*args, **kwargs): return func(*args, **kwargs) return wrapper class WrappedCallbackModule(CallbackBase): """ This is a callback module with the current method signature for `v2_playbook_on_start` wrapped in order to change the signature. """ CALLBACK_VERSION = 2.0 CALLBACK_TYPE = 'notification' CALLBACK_NAME = 'current_module' @wrap_callback def v2_playbook_on_start(self, playbook): register(self, playbook) callback_module = WrappedCallbackModule() self._tqm._callback_plugins.append(callback_module) self._tqm.send_callback('v2_playbook_on_start', self._playbook) register.assert_called_once_with(callback_module, self._playbook)
class IMPlaybookExecutor(PlaybookExecutor): ''' Simplified version of the PlaybookExecutor ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords, output): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) # Set out Callback as the stdout one to avoid stdout messages self._tqm._stdout_callback = AnsibleCallbacks(output) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' result = 0 try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir( os.path.dirname(playbook_path)) i = 1 # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) for play in pb.get_plays(): if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the # inventory self._inventory.remove_restriction() # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the # original object. all_vars = self._variable_manager.get_vars( loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) self._tqm._unreachable_hosts.update( self._unreachable_hosts) # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback( 'v2_playbook_on_play_start', new_play) self._tqm.send_callback( 'v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized # batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len( self._tqm._failed_hosts) + len( self._tqm._unreachable_hosts) if new_play.any_errors_fatal and failed_hosts_count > 0: break elif new_play.max_fail_percentage is not None and \ (int((new_play.max_fail_percentage) / 100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0)): break elif len(batch) == failed_hosts_count: break # clear the failed hosts dictionaires in the TQM for # the next batch self._unreachable_hosts.update( self._tqm._unreachable_hosts) self._tqm.clear_failed_hosts() # if the last result wasn't zero or 3 (some hosts were unreachable), # break out of the serial batch loop if result not in (0, 3): break i = i + 1 # per play self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook # file name loop if result != 0: break finally: if self._tqm is not None: self._tqm.cleanup() return result
class AdHocCLI(CLI): ''' is an extra-simple tool/framework/API for doing 'remote things'. this command allows you to define and run a single task 'playbook' against a set of hosts ''' def init_parser(self): ''' create an options parser for bin/ansible ''' super(AdHocCLI, self).init_parser( usage='%prog <host-pattern> [options]', desc="Define and run a single task 'playbook' against" " a set of hosts", epilog="Some modules do not make sense in Ad-Hoc (include," " meta, etc)") opt_help.add_runas_options(self.parser) opt_help.add_inventory_options(self.parser) opt_help.add_async_options(self.parser) opt_help.add_output_options(self.parser) opt_help.add_connect_options(self.parser) opt_help.add_check_options(self.parser) opt_help.add_runtask_options(self.parser) opt_help.add_vault_options(self.parser) opt_help.add_fork_options(self.parser) opt_help.add_module_options(self.parser) opt_help.add_basedir_options(self.parser) opt_help.add_tasknoplay_options(self.parser) # options unique to ansible ad-hoc self.parser.add_argument('-a', '--args', dest='module_args', help="module arguments", default=C.DEFAULT_MODULE_ARGS) self.parser.add_argument('-m', '--module-name', dest='module_name', help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME, default=C.DEFAULT_MODULE_NAME) self.parser.add_argument('args', metavar='pattern', help='host pattern') def post_process_args(self, options): '''Post process and validate options for bin/ansible ''' options = super(AdHocCLI, self).post_process_args(options) display.verbosity = options.verbosity self.validate_conflicts(options, runas_opts=True, fork_opts=True) return options def _play_ds(self, pattern, async_val, poll): check_raw = context.CLIARGS['module_name'] in C.MODULE_REQUIRE_ARGS mytask = { 'action': { 'module': context.CLIARGS['module_name'], 'args': parse_kv(context.CLIARGS['module_args'], check_raw=check_raw) }, 'timeout': context.CLIARGS['task_timeout'] } # avoid adding to tasks that don't support it, unless set, then give user an error if context.CLIARGS['module_name'] not in ('include_role', 'include_tasks') and any( frozenset( (async_val, poll))): mytask['async_val'] = async_val mytask['poll'] = poll return dict(name="Ansible Ad-Hoc", hosts=pattern, gather_facts='no', tasks=[mytask]) def run(self): ''' create and execute the single task playbook ''' super(AdHocCLI, self).run() # only thing left should be host pattern pattern = to_text(context.CLIARGS['args'], errors='surrogate_or_strict') sshpass = None becomepass = None (sshpass, becomepass) = self.ask_passwords() passwords = {'conn_pass': sshpass, 'become_pass': becomepass} # get basic objects loader, inventory, variable_manager = self._play_prereqs() try: hosts = self.get_host_list(inventory, context.CLIARGS['subset'], pattern) except AnsibleError: if context.CLIARGS['subset']: raise else: hosts = [] display.warning("No hosts matched, nothing to do") if context.CLIARGS['listhosts']: display.display(' hosts (%d):' % len(hosts)) for host in hosts: display.display(' %s' % host) return 0 if context.CLIARGS[ 'module_name'] in C.MODULE_REQUIRE_ARGS and not context.CLIARGS[ 'module_args']: err = "No argument passed to %s module" % context.CLIARGS[ 'module_name'] if pattern.endswith(".yml"): err = err + ' (did you mean to run ansible-playbook?)' raise AnsibleOptionsError(err) # Avoid modules that don't work with ad-hoc if context.CLIARGS['module_name'] in ('import_playbook', ): raise AnsibleOptionsError( "'%s' is not a valid action for ad-hoc commands" % context.CLIARGS['module_name']) play_ds = self._play_ds(pattern, context.CLIARGS['seconds'], context.CLIARGS['poll_interval']) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) # used in start callback playbook = Playbook(loader) playbook._entries.append(play) playbook._file_name = '__adhoc_playbook__' if self.callback: cb = self.callback elif context.CLIARGS['one_line']: cb = 'oneline' # Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks' elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default': cb = C.DEFAULT_STDOUT_CALLBACK else: cb = 'minimal' run_tree = False if context.CLIARGS['tree']: C.CALLBACKS_ENABLED.append('tree') C.TREE_DIR = context.CLIARGS['tree'] run_tree = True # now create a task queue manager to execute the play self._tqm = None try: self._tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, passwords=passwords, stdout_callback=cb, run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS, run_tree=run_tree, forks=context.CLIARGS['forks'], ) self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', playbook) result = self._tqm.run(play) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) finally: if self._tqm: self._tqm.cleanup() if loader: loader.cleanup_all_tmp_files() return result
class IMPlaybookExecutor(PlaybookExecutor): ''' Simplified version of the PlaybookExecutor ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords, output): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) # Set out Callback as the stdout one to avoid stdout messages self._tqm._stdout_callback = AnsibleCallbacks(output) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' result = 0 try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.dirname(playbook_path)) i = 1 # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) for play in pb.get_plays(): if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) self._tqm._unreachable_hosts.update(self._unreachable_hosts) # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) if new_play.any_errors_fatal and failed_hosts_count > 0: break elif new_play.max_fail_percentage is not None and \ int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0): break elif len(batch) == failed_hosts_count: break # clear the failed hosts dictionaires in the TQM for the next batch self._unreachable_hosts.update(self._tqm._unreachable_hosts) self._tqm.clear_failed_hosts() # if the last result wasn't zero or 3 (some hosts were unreachable), # break out of the serial batch loop if result not in (0, 3): break i = i + 1 # per play self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break finally: if self._tqm is not None: self._cleanup() return result
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.dirname(playbook_path)) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) i = 1 plays = pb.get_plays() display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path))) for play in plays: if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = var.get("private", True) confirm = var.get("confirm", False) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in self._variable_manager.extra_vars: if self._tqm: self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = display.do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default) else: # we are either in --list-<option> or syntax check play.vars[vname] = default # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: self._tqm._unreachable_hosts.update(self._unreachable_hosts) # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) if new_play.max_fail_percentage is not None and \ int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0): break elif len(batch) == failed_hosts_count: break # clear the failed hosts dictionaires in the TQM for the next batch self._unreachable_hosts.update(self._tqm._unreachable_hosts) self._tqm.clear_failed_hosts() # if the last result wasn't zero or 3 (some hosts were unreachable), # break out of the serial batch loop if result not in (0, 3): break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # send the stats callback for this playbook if self._tqm is not None: if C.RETRY_FILES_ENABLED: retries = list(set(self._tqm._failed_hosts.keys() + self._tqm._unreachable_hosts.keys())) retries.sort() if len(retries) > 0: if C.RETRY_FILES_SAVE_PATH: basedir = C.shell_expand(C.RETRY_FILES_SAVE_PATH) else: basedir = os.path.dirname(playbook_path) (retry_name, _) = os.path.splitext(os.path.basename(playbook_path)) filename = os.path.join(basedir, "%s.retry" % retry_name) if self._generate_retry_inventory(filename, retries): display.display("\tto retry, use: --limit @%s\n" % filename) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._tqm.cleanup() if self._options.syntax: display.display("No issues encountered") return result return result def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, string_types) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%","")) serial = int((serial_pct/100.0) * len(all_hosts)) else: if play.serial is None: serial = -1 else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches def _generate_retry_inventory(self, retry_path, replay_hosts): ''' Called when a playbook run fails. It generates an inventory which allows re-running on ONLY the failed hosts. This may duplicate some variable information in group_vars/host_vars but that is ok, and expected. ''' try: with open(retry_path, 'w') as fd: for x in replay_hosts: fd.write("%s\n" % x) except Exception as e: display.error("Could not create retry file '%s'. The error was: %s" % (retry_path, e)) return False return True
class AdHocCLI(CLI): ''' is an extra-simple tool/framework/API for doing 'remote things'. this command allows you to define and run a single task 'playbook' against a set of hosts ''' def parse(self): ''' create an options parser for bin/ansible ''' self.parser = CLI.base_parser( usage='%prog <host-pattern> [options]', runas_opts=True, inventory_opts=True, async_opts=True, output_opts=True, connect_opts=True, check_opts=True, runtask_opts=True, vault_opts=True, fork_opts=True, module_opts=True, basedir_opts=True, desc="Define and run a single task 'playbook' against a set of hosts", epilog="Some modules do not make sense in Ad-Hoc (include, meta, etc)", ) # options unique to ansible ad-hoc self.parser.add_option('-a', '--args', dest='module_args', help="module arguments", default=C.DEFAULT_MODULE_ARGS) self.parser.add_option('-m', '--module-name', dest='module_name', help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME, default=C.DEFAULT_MODULE_NAME) super(AdHocCLI, self).parse() if len(self.args) < 1: raise AnsibleOptionsError("Missing target hosts") elif len(self.args) > 1: raise AnsibleOptionsError("Extraneous options or arguments") display.verbosity = self.options.verbosity self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True) def _play_ds(self, pattern, async_val, poll): check_raw = self.options.module_name in ('command', 'win_command', 'shell', 'win_shell', 'script', 'raw') return dict( name="Ansible Ad-Hoc", hosts=pattern, gather_facts='no', tasks=[dict(action=dict(module=self.options.module_name, args=parse_kv(self.options.module_args, check_raw=check_raw)), async_val=async_val, poll=poll)] ) def run(self): ''' create and execute the single task playbook ''' super(AdHocCLI, self).run() # only thing left should be host pattern pattern = to_text(self.args[0], errors='surrogate_or_strict') sshpass = None becomepass = None self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = {'conn_pass': sshpass, 'become_pass': becomepass} # dynamically load any plugins get_all_plugin_loaders() loader, inventory, variable_manager = self._play_prereqs(self.options) try: hosts = CLI.get_host_list(inventory, self.options.subset, pattern) except AnsibleError: if self.options.subset: raise else: hosts = [] display.warning("No hosts matched, nothing to do") if self.options.listhosts: display.display(' hosts (%d):' % len(hosts)) for host in hosts: display.display(' %s' % host) return 0 if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args: err = "No argument passed to %s module" % self.options.module_name if pattern.endswith(".yml"): err = err + ' (did you mean to run ansible-playbook?)' raise AnsibleOptionsError(err) # Avoid modules that don't work with ad-hoc if self.options.module_name.startswith(('include', 'import_')): raise AnsibleOptionsError("'%s' is not a valid action for ad-hoc commands" % self.options.module_name) play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) # used in start callback playbook = Playbook(loader) playbook._entries.append(play) playbook._file_name = '__adhoc_playbook__' if self.callback: cb = self.callback elif self.options.one_line: cb = 'oneline' # Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks' elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default': cb = C.DEFAULT_STDOUT_CALLBACK else: cb = 'minimal' run_tree = False if self.options.tree: C.DEFAULT_CALLBACK_WHITELIST.append('tree') C.TREE_DIR = self.options.tree run_tree = True # now create a task queue manager to execute the play self._tqm = None try: self._tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, options=self.options, passwords=passwords, stdout_callback=cb, run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS, run_tree=run_tree, ) self._tqm.send_callback('v2_playbook_on_start', playbook) result = self._tqm.run(play) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) finally: if self._tqm: self._tqm.cleanup() if loader: loader.cleanup_all_tmp_files() return result
def run(self): """Run the Application""" # Since the API is constructed for CLI it expects certain options to # always be set in the context object context.CLIARGS = ImmutableDict( connection="smart", module_path=[get_path("./modules"), "/usr/share/ansible"], forks=10, become=None, become_method=None, become_user=None, check=False, diff=False, verbosity=3, ) # Initialize needed objects loader = DataLoader( ) # Takes care of finding and reading yaml, json and # ini files passwords = dict(vault_pass="******") # Instantiate our ResultCallback for handling results as they come in if self.args.measurement: results_callback = MeasurementsResultCallback( plugins=self._plugins) else: results_callback = ResultCallback(plugins=self._plugins) # Create inventory, use path to host config file as source or hosts in a # comma separated string inventory = InventoryManager(loader=loader, sources=self._sources) # Variable manager takes care of merging all the different sources to give # you a unified view of variables available in each context variable_manager = VariableManager(loader=loader, inventory=inventory) # Instantiate task queue manager, which takes care of forking and setting # up all objects to iterate over host list and tasks # IMPORTANT: This also adds library dirs paths to the module loader # IMPORTANT: and so it must be initialized before calling `Play.load()`. tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, passwords=passwords, stdout_callback= results_callback, # Use our custom callback instead of # the ``default`` callback plugin, # which prints to stdout ) # Create data structure that represents our play, including tasks, this is # basically what our YAML loader does internally. play_source = dict( name="Ansible Play", hosts="all", gather_facts="yes", tasks=self.playbook_tasks(), ) # Create play object, playbook objects use .load instead of init or new # methods, this will also automatically create the task objects from the # info provided in play_source play = Play().load(play_source, variable_manager=variable_manager, loader=loader) # Actually run it try: tqm.load_callbacks() tqm.run(play) tqm.send_callback( "v2_playbook_on_stats", tqm._stats, # pylint: disable=protected-access ) finally: # We always need to cleanup child procs and the structures we use to # communicate with them tqm.cleanup() if loader: loader.cleanup_all_tmp_files() # Remove ansible tmpdir shutil.rmtree(C.DEFAULT_LOCAL_TMP, True) # pylint: disable=no-member if tqm is not None: tqm.cleanup()
class PLAYBOOK(Base): def run(self, playbook): pass def run_playbook(self, playbook, workname, work_uuid, username, describe, yaml_content, is_checksyntax=False): """ :param playbooks: playbook路径的列表 :return: """ if is_checksyntax: self._tqm = None else: # self.callback = PlayBookResultsCollector() # self.callback = ResultCallback(work_uuid, workname, 'playbook', self.options, describe, # yaml_content=yaml_content, username=username) self.callback = JsonCallback(work_uuid, workname, 'playbook', self.options, describe, yaml_content=yaml_content, username=username) self._tqm = TaskQueueManager(inventory=self.inventory, variable_manager=self.variable_manager, loader=self.loader, options=self.options, passwords=self.passwords, stdout_callback=self.callback) check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE) try: pb = Playbook.load(playbook, variable_manager=self.variable_manager, loader=self.loader) if self._tqm is not None: self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) plays = pb.get_plays() for play in plays: if play._included_path is not None: self.loader.set_basedir(play._included_path) else: self.loader.set_basedir(pb._basedir) self.inventory.remove_restriction() # Allow variables to be used in vars_prompt fields. all_vars = self.variable_manager.get_vars(play=play) templar = Templar(loader=self.loader, variables=all_vars) play.post_validate(templar) if is_checksyntax or self._tqm is None: return True batches = self._get_serialized_batches(play) if len(batches) == 0: self._tqm.send_callback('v2_playbook_on_play_start', play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') continue for batch in batches: self.inventory.restrict_to_hosts(batch) try: self._tqm.run(play) except Exception as e: print(e) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) self._tqm.cleanup() self.loader.cleanup_all_tmp_files() return True except Exception as e: print(e) return False def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self.inventory.get_hosts(play.hosts) all_hosts_len = len(all_hosts) # the serial value can be listed as a scalar or a list of # scalars, so we make sure it's a list here serial_batch_list = play.serial if len(serial_batch_list) == 0: serial_batch_list = [-1] cur_item = 0 serialized_batches = [] while len(all_hosts) > 0: # get the serial value from current item in the list serial = pct_to_int(serial_batch_list[cur_item], all_hosts_len) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise grab a chunk of the hosts equal # to the current serial item size if serial <= 0: serialized_batches.append(all_hosts) break else: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) # increment the current batch list item number, and if we've hit # the end keep using the last element until we've consumed all of # the hosts in the inventory cur_item += 1 if cur_item > len(serial_batch_list) - 1: cur_item = len(serial_batch_list) - 1 return serialized_batches # # def get_playbook_result(self): # self.results_raw = {'skipped': {}, 'failed': {}, 'ok': {}, "status": {}, 'unreachable': {}, "changed": {}} # for host, result in self.callback.task_ok.items(): # self.results_raw['ok'][host] = result # # for host, result in self.callback.task_failed.items(): # self.results_raw['failed'][host] = result # # for host, result in self.callback.task_status.items(): # self.results_raw['status'][host] = result # # # for host, result in self.callback.task_changed.items(): # # self.results_raw['changed'][host] = result # # for host, result in self.callback.task_skipped.items(): # self.results_raw['skipped'][host] = result # # for host, result in self.callback.task_unreachable.items(): # self.results_raw['unreachable'][host] = result # return self.results_raw
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir( os.path.dirname(playbook_path)) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) i = 1 plays = pb.get_plays() display.vv(u'%d plays in %s' % (len(plays), to_unicode(playbook_path))) for play in plays: if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = var.get("private", True) confirm = var.get("confirm", False) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in self._variable_manager.extra_vars: if self._tqm: self._tqm.send_callback( 'v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = display.do_var_prompt( vname, private, prompt, encrypt, confirm, salt_size, salt, default) else: # we are either in --list-<option> or syntax check play.vars[vname] = default # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars( loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: self._tqm._unreachable_hosts.update( self._unreachable_hosts) # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback( 'v2_playbook_on_play_start', new_play) self._tqm.send_callback( 'v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len( self._tqm._failed_hosts) + len( self._tqm._unreachable_hosts) if new_play.max_fail_percentage is not None and \ int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0): break elif len(batch) == failed_hosts_count: break # clear the failed hosts dictionaires in the TQM for the next batch self._unreachable_hosts.update( self._tqm._unreachable_hosts) self._tqm.clear_failed_hosts() # if the last result wasn't zero or 3 (some hosts were unreachable), # break out of the serial batch loop if result not in (0, 3): break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # send the stats callback for this playbook if self._tqm is not None: if C.RETRY_FILES_ENABLED: retries = set(self._tqm._failed_hosts.keys()) retries.update(self._tqm._unreachable_hosts.keys()) retries = sorted(retries) if len(retries) > 0: if C.RETRY_FILES_SAVE_PATH: basedir = C.shell_expand( C.RETRY_FILES_SAVE_PATH) else: basedir = os.path.dirname(playbook_path) (retry_name, _) = os.path.splitext( os.path.basename(playbook_path)) filename = os.path.join(basedir, "%s.retry" % retry_name) if self._generate_retry_inventory( filename, retries): display.display( "\tto retry, use: --limit @%s\n" % filename) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._tqm.cleanup() if self._options.syntax: display.display("No issues encountered") return result return result def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, string_types) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%", "")) serial = int((serial_pct / 100.0) * len(all_hosts)) else: if play.serial is None: serial = -1 else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches def _generate_retry_inventory(self, retry_path, replay_hosts): ''' Called when a playbook run fails. It generates an inventory which allows re-running on ONLY the failed hosts. This may duplicate some variable information in group_vars/host_vars but that is ok, and expected. ''' try: with open(retry_path, 'w') as fd: for x in replay_hosts: fd.write("%s\n" % x) except Exception as e: display.error( "Could not create retry file '%s'. The error was: %s" % (retry_path, e)) return False return True
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, display, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self.passwords = passwords # make sure the module path (if specified) is parsed and # added to the module_loader object if options.module_path is not None: for path in options.module_path.split(os.pathsep): module_loader.add_directory(path) if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, display=display, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' signal.signal(signal.SIGINT, self._cleanup) result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] i = 1 plays = pb.get_plays() self._display.vv('%d plays in %s' % (len(plays), playbook_path)) for play in plays: self._inventory.remove_restriction() # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing pname = new_play.get_name().strip() if pname == 'PLAY: <no name specified>': pname = 'PLAY: #%d' % i p = { 'name': pname } if self._options.listhosts: p['pattern']=play.hosts p['hosts']=set(self._inventory.get_hosts(new_play.hosts)) #TODO: play tasks are really blocks, need to figure out how to get task objects from them elif self._options.listtasks: p['tasks'] = [] for task in play.get_tasks(): p['tasks'].append(task) #p['tasks'].append({'name': task.get_name().strip(), 'tags': task.tags}) elif self._options.listtags: p['tags'] = set(new_play.tags) for task in play.get_tasks(): p['tags'].update(task) #p['tags'].update(task.tags) entry['plays'].append(p) else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # if the last result wasn't zero, break out of the serial batch loop if result != 0: break # if the last result wasn't zero, break out of the play loop if result != 0: break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._cleanup() if self._options.syntax: self.display.display("No issues encountered") return result # FIXME: this stat summary stuff should be cleaned up and moved # to a new method, if it even belongs here... self._display.banner("PLAY RECAP") hosts = sorted(self._tqm._stats.processed.keys()) for h in hosts: t = self._tqm._stats.summarize(h) self._display.display("%s : %s %s %s %s" % ( hostcolor(h, t), colorize('ok', t['ok'], 'green'), colorize('changed', t['changed'], 'yellow'), colorize('unreachable', t['unreachable'], 'red'), colorize('failed', t['failures'], 'red')), screen_only=True ) self._display.display("%s : %s %s %s %s" % ( hostcolor(h, t, False), colorize('ok', t['ok'], None), colorize('changed', t['changed'], None), colorize('unreachable', t['unreachable'], None), colorize('failed', t['failures'], None)), log_only=True ) self._display.display("", screen_only=True) # END STATS STUFF return result def _cleanup(self, signum=None, framenum=None): return self._tqm.cleanup() def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, basestring) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%","")) serial = int((serial_pct/100.0) * len(all_hosts)) else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' # FIXME: passwords should not be passed in piecemeal like this, # if they're just going to be stuck in a dict later. def __init__(self, playbooks, inventory, variable_manager, loader, display, options, conn_pass, become_pass): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self.passwords = {'conn_pass': conn_pass, 'become_pass': become_pass} if options.listhosts or options.listtasks or options.listtags: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, callback='default', variable_manager=variable_manager, loader=loader, display=display, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' signal.signal(signal.SIGINT, self._cleanup) result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] i = 1 plays = pb.get_plays() self._display.vv('%d plays in %s' % (len(plays), playbook_path)) for play in plays: self._inventory.remove_restriction() # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars( loader=self._loader, play=play) new_play = play.copy() new_play.post_validate(all_vars, fail_on_undefined=False) if self._tqm is None: # we are just doing a listing pname = new_play.get_name().strip() if pname == 'PLAY: <no name specified>': pname = 'PLAY: #%d' % i p = {'name': pname} if self._options.listhosts: p['pattern'] = play.hosts p['hosts'] = set( self._inventory.get_hosts(new_play.hosts)) #TODO: play tasks are really blocks, need to figure out how to get task objects from them elif self._options.listtasks: p['tasks'] = [] for task in play.get_tasks(): p['tasks'].append(task) #p['tasks'].append({'name': task.get_name().strip(), 'tags': task.tags}) elif self._options.listtags: p['tags'] = set(new_play.tags) for task in play.get_tasks(): p['tags'].update(task) #p['tags'].update(task.tags) entry['plays'].append(p) else: # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback( 'v2_playbook_on_play_start', new_play) self._tqm.send_callback( 'v2_playbook_on_no_hosts_matched') result = 0 break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) if result != 0: break if result != 0: break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook if entrylist: return entrylist finally: if self._tqm is not None: self._cleanup() # FIXME: this stat summary stuff should be cleaned up and moved # to a new method, if it even belongs here... self._display.banner("PLAY RECAP") hosts = sorted(self._tqm._stats.processed.keys()) for h in hosts: t = self._tqm._stats.summarize(h) self._display.display( "%s : %s %s %s %s" % (hostcolor(h, t), colorize('ok', t['ok'], 'green'), colorize('changed', t['changed'], 'yellow'), colorize('unreachable', t['unreachable'], 'red'), colorize('failed', t['failures'], 'red')), screen_only=True) self._display.display( "%s : %s %s %s %s" % (hostcolor(h, t, False), colorize('ok', t['ok'], None), colorize('changed', t['changed'], None), colorize('unreachable', t['unreachable'], None), colorize('failed', t['failures'], None)), log_only=True) self._display.display("", screen_only=True) # END STATS STUFF return result def _cleanup(self, signum=None, framenum=None): return self._tqm.cleanup() def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, basestring) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%", "")) serial = int((serial_pct / 100.0) * len(all_hosts)) else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches
class AdHocCLI(CLI): ''' is an extra-simple tool/framework/API for doing 'remote things'. this command allows you to define and run a single task 'playbook' against a set of hosts ''' def parse(self): ''' create an options parser for bin/ansible ''' self.parser = CLI.base_parser( usage='%prog <host-pattern> [options]', runas_opts=True, inventory_opts=True, async_opts=True, output_opts=True, connect_opts=True, check_opts=True, runtask_opts=True, vault_opts=True, fork_opts=True, module_opts=True, basedir_opts=True, desc= "Define and run a single task 'playbook' against a set of hosts", epilog= "Some modules do not make sense in Ad-Hoc (include, meta, etc)", ) # options unique to ansible ad-hoc self.parser.add_option('-a', '--args', dest='module_args', help="module arguments", default=C.DEFAULT_MODULE_ARGS) self.parser.add_option('-m', '--module-name', dest='module_name', help="module name to execute (default=%s)" % C.DEFAULT_MODULE_NAME, default=C.DEFAULT_MODULE_NAME) super(AdHocCLI, self).parse() if len(self.args) < 1: raise AnsibleOptionsError("Missing target hosts") elif len(self.args) > 1: raise AnsibleOptionsError("Extraneous options or arguments") display.verbosity = self.options.verbosity self.validate_conflicts(runas_opts=True, vault_opts=True, fork_opts=True) def _play_ds(self, pattern, async_val, poll): check_raw = self.options.module_name in ('command', 'win_command', 'shell', 'win_shell', 'script', 'raw') return dict(name="Ansible Ad-Hoc", hosts=pattern, gather_facts='no', tasks=[ dict(action=dict(module=self.options.module_name, args=parse_kv( self.options.module_args, check_raw=check_raw)), async_val=async_val, poll=poll) ]) def run(self): ''' create and execute the single task playbook ''' super(AdHocCLI, self).run() # only thing left should be host pattern pattern = to_text(self.args[0], errors='surrogate_or_strict') sshpass = None becomepass = None self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = {'conn_pass': sshpass, 'become_pass': becomepass} # dynamically load any plugins get_all_plugin_loaders() loader, inventory, variable_manager = self._play_prereqs(self.options) try: hosts = CLI.get_host_list(inventory, self.options.subset, pattern) except AnsibleError: if self.options.subset: raise else: hosts = [] display.warning("No hosts matched, nothing to do") if self.options.listhosts: display.display(' hosts (%d):' % len(hosts)) for host in hosts: display.display(' %s' % host) return 0 if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args: err = "No argument passed to %s module" % self.options.module_name if pattern.endswith(".yml"): err = err + ' (did you mean to run ansible-playbook?)' raise AnsibleOptionsError(err) # Avoid modules that don't work with ad-hoc if self.options.module_name in ('import_playbook', ): raise AnsibleOptionsError( "'%s' is not a valid action for ad-hoc commands" % self.options.module_name) play_ds = self._play_ds(pattern, self.options.seconds, self.options.poll_interval) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) # used in start callback playbook = Playbook(loader) playbook._entries.append(play) playbook._file_name = '__adhoc_playbook__' if self.callback: cb = self.callback elif self.options.one_line: cb = 'oneline' # Respect custom 'stdout_callback' only with enabled 'bin_ansible_callbacks' elif C.DEFAULT_LOAD_CALLBACK_PLUGINS and C.DEFAULT_STDOUT_CALLBACK != 'default': cb = C.DEFAULT_STDOUT_CALLBACK else: cb = 'minimal' run_tree = False if self.options.tree: C.DEFAULT_CALLBACK_WHITELIST.append('tree') C.TREE_DIR = self.options.tree run_tree = True # now create a task queue manager to execute the play self._tqm = None try: self._tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, options=self.options, passwords=passwords, stdout_callback=cb, run_additional_callbacks=C.DEFAULT_LOAD_CALLBACK_PLUGINS, run_tree=run_tree, ) self._tqm.send_callback('v2_playbook_on_start', playbook) result = self._tqm.run(play) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) finally: if self._tqm: self._tqm.cleanup() if loader: loader.cleanup_all_tmp_files() return result
class TestTaskQueueManagerCallbacks(unittest.TestCase): def setUp(self): inventory = MagicMock() variable_manager = MagicMock() loader = MagicMock() passwords = [] # Reset the stored command line args co.GlobalCLIArgs._Singleton__instance = None self._tqm = TaskQueueManager(inventory, variable_manager, loader, passwords) self._playbook = Playbook(loader) # we use a MagicMock to register the result of the call we # expect to `v2_playbook_on_call`. We don't mock out the # method since we're testing code that uses `inspect` to # look at that method's argspec and we want to ensure this # test is easy to reason about. self._register = MagicMock() def tearDown(self): # Reset the stored command line args co.GlobalCLIArgs._Singleton__instance = None def test_task_queue_manager_callbacks_v2_playbook_on_start(self): """ Assert that no exceptions are raised when sending a Playbook start callback to a current callback module plugin. """ register = self._register class CallbackModule(CallbackBase): """ This is a callback module with the current method signature for `v2_playbook_on_start`. """ CALLBACK_VERSION = 2.0 CALLBACK_TYPE = 'notification' CALLBACK_NAME = 'current_module' def v2_playbook_on_start(self, playbook): register(self, playbook) callback_module = CallbackModule() self._tqm._callback_plugins.append(callback_module) self._tqm.send_callback('v2_playbook_on_start', self._playbook) register.assert_called_once_with(callback_module, self._playbook) def test_task_queue_manager_callbacks_v2_playbook_on_start_wrapped(self): """ Assert that no exceptions are raised when sending a Playbook start callback to a wrapped current callback module plugin. """ register = self._register def wrap_callback(func): """ This wrapper changes the exposed argument names for a method from the original names to (*args, **kwargs). This is used in order to validate that wrappers which change par- ameter names do not break the TQM callback system. :param func: function to decorate :return: decorated function """ def wrapper(*args, **kwargs): return func(*args, **kwargs) return wrapper class WrappedCallbackModule(CallbackBase): """ This is a callback module with the current method signature for `v2_playbook_on_start` wrapped in order to change the signature. """ CALLBACK_VERSION = 2.0 CALLBACK_TYPE = 'notification' CALLBACK_NAME = 'current_module' @wrap_callback def v2_playbook_on_start(self, playbook): register(self, playbook) callback_module = WrappedCallbackModule() self._tqm._callback_plugins.append(callback_module) self._tqm.send_callback('v2_playbook_on_start', self._playbook) register.assert_called_once_with(callback_module, self._playbook)
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, display, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._display = display self._options = options self.passwords = passwords self._unreachable_hosts = dict() if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, display=display, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' signal.signal(signal.SIGINT, self._cleanup) result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.dirname(playbook_path)) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] i = 1 plays = pb.get_plays() self._display.vv('%d plays in %s' % (len(plays), playbook_path)) for play in plays: if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = var.get("private", True) confirm = var.get("confirm", False) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in play.vars: if self._tqm: self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = self._do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default) # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm._unreachable_hosts.update(self._unreachable_hosts) # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) if new_play.any_errors_fatal and failed_hosts_count > 0: break elif new_play.max_fail_percentage is not None and \ int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0): break elif len(batch) == failed_hosts_count: break # clear the failed hosts dictionaires in the TQM for the next batch self._unreachable_hosts.update(self._tqm._unreachable_hosts) self._tqm.clear_failed_hosts() # if the last result wasn't zero or 3 (some hosts were unreachable), # break out of the serial batch loop if result not in (0, 3): break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._cleanup() if self._options.syntax: self.display.display("No issues encountered") return result # FIXME: this stat summary stuff should be cleaned up and moved # to a new method, if it even belongs here... self._display.banner("PLAY RECAP") hosts = sorted(self._tqm._stats.processed.keys()) for h in hosts: t = self._tqm._stats.summarize(h) self._display.display(u"%s : %s %s %s %s" % ( hostcolor(h, t), colorize(u'ok', t['ok'], 'green'), colorize(u'changed', t['changed'], 'yellow'), colorize(u'unreachable', t['unreachable'], 'red'), colorize(u'failed', t['failures'], 'red')), screen_only=True ) self._display.display(u"%s : %s %s %s %s" % ( hostcolor(h, t, False), colorize(u'ok', t['ok'], None), colorize(u'changed', t['changed'], None), colorize(u'unreachable', t['unreachable'], None), colorize(u'failed', t['failures'], None)), log_only=True ) self._display.display("", screen_only=True) # END STATS STUFF return result def _cleanup(self, signum=None, framenum=None): return self._tqm.cleanup() def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, string_types) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%","")) serial = int((serial_pct/100.0) * len(all_hosts)) else: if play.serial is None: serial = -1 else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches def _do_var_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): if sys.__stdin__.isatty(): if prompt and default is not None: msg = "%s [%s]: " % (prompt, default) elif prompt: msg = "%s: " % prompt else: msg = 'input for %s: ' % varname def do_prompt(prompt, private): if sys.stdout.encoding: msg = prompt.encode(sys.stdout.encoding) else: # when piping the output, or at other times when stdout # may not be the standard file descriptor, the stdout # encoding may not be set, so default to something sane msg = prompt.encode(locale.getpreferredencoding()) if private: return getpass.getpass(msg) return raw_input(msg) if confirm: while True: result = do_prompt(msg, private) second = do_prompt("confirm " + msg, private) if result == second: break self._display.display("***** VALUES ENTERED DO NOT MATCH ****") else: result = do_prompt(msg, private) else: result = None self._display.warning("Not prompting as we are not in interactive mode") # if result is false and default is not None if not result and default is not None: result = default if encrypt: result = do_encrypt(result, encrypt, salt_size, salt) # handle utf-8 chars result = to_unicode(result, errors='strict') return result
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' signal.signal(signal.SIGINT, self._cleanup) result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) self._inventory.set_playbook_basedir(os.path.dirname(playbook_path)) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] i = 1 plays = pb.get_plays() display.vv('%d plays in %s' % (len(plays), playbook_path)) for play in plays: if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = var.get("private", True) confirm = var.get("confirm", False) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in play.vars: if self._tqm: self._tqm.send_callback('v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) if self._options.syntax: play.vars[vname] = default else: play.vars[vname] = self._do_var_prompt(vname, private, prompt, encrypt, confirm, salt_size, salt, default) # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(loader=self._loader, play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm._unreachable_hosts.update(self._unreachable_hosts) # we are actually running plays for batch in self._get_serialized_batches(new_play): if len(batch) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') break # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) if new_play.any_errors_fatal and failed_hosts_count > 0: break elif new_play.max_fail_percentage is not None and \ int((new_play.max_fail_percentage)/100.0 * len(batch)) > int((len(batch) - failed_hosts_count) / len(batch) * 100.0): break elif len(batch) == failed_hosts_count: break # clear the failed hosts dictionaires in the TQM for the next batch self._unreachable_hosts.update(self._tqm._unreachable_hosts) self._tqm.clear_failed_hosts() # if the last result wasn't zero or 3 (some hosts were unreachable), # break out of the serial batch loop if result not in (0, 3): break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # send the stats callback for this playbook if self._tqm is not None: self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._cleanup() if self._options.syntax: display.display("No issues encountered") return result return result def _cleanup(self, signum=None, framenum=None): return self._tqm.cleanup() def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) # check to see if the serial number was specified as a percentage, # and convert it to an integer value based on the number of hosts if isinstance(play.serial, string_types) and play.serial.endswith('%'): serial_pct = int(play.serial.replace("%","")) serial = int((serial_pct/100.0) * len(all_hosts)) else: if play.serial is None: serial = -1 else: serial = int(play.serial) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise split the list of hosts into chunks # which are based on the serial size if serial <= 0: return [all_hosts] else: serialized_batches = [] while len(all_hosts) > 0: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) return serialized_batches def _do_var_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None): if sys.__stdin__.isatty(): if prompt and default is not None: msg = "%s [%s]: " % (prompt, default) elif prompt: msg = "%s: " % prompt else: msg = 'input for %s: ' % varname def do_prompt(prompt, private): if sys.stdout.encoding: msg = prompt.encode(sys.stdout.encoding) else: # when piping the output, or at other times when stdout # may not be the standard file descriptor, the stdout # encoding may not be set, so default to something sane msg = prompt.encode(locale.getpreferredencoding()) if private: return getpass.getpass(msg) return raw_input(msg) if confirm: while True: result = do_prompt(msg, private) second = do_prompt("confirm " + msg, private) if result == second: break display.display("***** VALUES ENTERED DO NOT MATCH ****") else: result = do_prompt(msg, private) else: result = None display.warning("Not prompting as we are not in interactive mode") # if result is false and default is not None if not result and default is not None: result = default if encrypt: result = do_encrypt(result, encrypt, salt_size, salt) # handle utf-8 chars result = to_unicode(result, errors='strict') return result
# actually run it tqm = None try: tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords, stdout_callback='default', ) tqm.load_callbacks() result = tqm.run(play) stats = tqm._stats print stats.summarize('localhost') tqm.send_callback("record_logs", user_id="pattabi", success=True) finally: if tqm is not None: tqm.cleanup() def main(): print "main" if __name__ == "__main__": main()
class PlaybookExecutor: ''' This is the primary class for executing playbooks, and thus the basis for bin/ansible-playbook operation. ''' def __init__(self, playbooks, inventory, variable_manager, loader, options, passwords): self._playbooks = playbooks self._inventory = inventory self._variable_manager = variable_manager self._loader = loader self._options = options self.passwords = passwords self._unreachable_hosts = dict() if options.listhosts or options.listtasks or options.listtags or options.syntax: self._tqm = None else: self._tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=self.passwords) # Note: We run this here to cache whether the default ansible ssh # executable supports control persist. Sometime in the future we may # need to enhance this to check that ansible_ssh_executable specified # in inventory is also cached. We can't do this caching at the point # where it is used (in task_executor) because that is post-fork and # therefore would be discarded after every task. check_for_controlpersist(C.ANSIBLE_SSH_EXECUTABLE) def run(self): ''' Run the given playbook, based on the settings in the play which may limit the runs to serialized groups, etc. ''' result = 0 entrylist = [] entry = {} try: for playbook_path in self._playbooks: pb = Playbook.load(playbook_path, variable_manager=self._variable_manager, loader=self._loader) # FIXME: move out of inventory self._inventory.set_playbook_basedir(os.path.realpath(os.path.dirname(playbook_path))) if self._tqm is None: # we are doing a listing entry = {'playbook': playbook_path} entry['plays'] = [] else: # make sure the tqm has callbacks loaded self._tqm.load_callbacks() self._tqm.send_callback('v2_playbook_on_start', pb) i = 1 plays = pb.get_plays() display.vv(u'%d plays in %s' % (len(plays), to_text(playbook_path))) for play in plays: if play._included_path is not None: self._loader.set_basedir(play._included_path) else: self._loader.set_basedir(pb._basedir) # clear any filters which may have been applied to the inventory self._inventory.remove_restriction() if play.vars_prompt: for var in play.vars_prompt: vname = var['name'] prompt = var.get("prompt", vname) default = var.get("default", None) private = boolean(var.get("private", True)) confirm = boolean(var.get("confirm", False)) encrypt = var.get("encrypt", None) salt_size = var.get("salt_size", None) salt = var.get("salt", None) if vname not in self._variable_manager.extra_vars: if self._tqm: self._tqm.send_callback( 'v2_playbook_on_vars_prompt', vname, private, prompt, encrypt, confirm, salt_size, salt, default) play.vars[vname] = display.do_var_prompt( vname, private, prompt, encrypt, confirm, salt_size, salt, default) else: # we are either in --list-<option> or syntax check play.vars[vname] = default # Create a temporary copy of the play here, so we can run post_validate # on it without the templating changes affecting the original object. all_vars = self._variable_manager.get_vars(play=play) templar = Templar(loader=self._loader, variables=all_vars) new_play = play.copy() new_play.post_validate(templar) if self._options.syntax: continue if self._tqm is None: # we are just doing a listing entry['plays'].append(new_play) else: self._tqm._unreachable_hosts.update( self._unreachable_hosts) previously_failed = len(self._tqm._failed_hosts) previously_unreachable = len( self._tqm._unreachable_hosts) break_play = False # we are actually running plays batches = self._get_serialized_batches(new_play) if len(batches) == 0: self._tqm.send_callback( 'v2_playbook_on_play_start', new_play) self._tqm.send_callback( 'v2_playbook_on_no_hosts_matched') for batch in batches: # restrict the inventory to the hosts in the serialized batch self._inventory.restrict_to_hosts(batch) # and run it... result = self._tqm.run(play=play) # break the play if the result equals the special return code if result & self._tqm.RUN_FAILED_BREAK_PLAY != 0: result = self._tqm.RUN_FAILED_HOSTS break_play = True # check the number of failures here, to see if they're above the maximum # failure percentage allowed, or if any errors are fatal. If either of those # conditions are met, we break out, otherwise we only break out if the entire # batch failed failed_hosts_count = len(self._tqm._failed_hosts) + len(self._tqm._unreachable_hosts) - \ (previously_failed + previously_unreachable) if len(batch) == failed_hosts_count: break_play = True break # update the previous counts so they don't accumulate incorrectly # over multiple serial batches previously_failed += len( self._tqm._failed_hosts) - previously_failed previously_unreachable += len( self._tqm._unreachable_hosts ) - previously_unreachable # save the unreachable hosts from this batch self._unreachable_hosts.update( self._tqm._unreachable_hosts) if break_play: break i = i + 1 # per play if entry: entrylist.append(entry) # per playbook # send the stats callback for this playbook if self._tqm is not None: if C.RETRY_FILES_ENABLED: retries = set(self._tqm._failed_hosts.keys()) retries.update(self._tqm._unreachable_hosts.keys()) retries = sorted(retries) if len(retries) > 0: if C.RETRY_FILES_SAVE_PATH: basedir = C.RETRY_FILES_SAVE_PATH elif playbook_path: basedir = os.path.dirname( os.path.abspath(playbook_path)) else: basedir = '~/' (retry_name, _) = os.path.splitext( os.path.basename(playbook_path)) filename = os.path.join(basedir, "%s.retry" % retry_name) if self._generate_retry_inventory( filename, retries): display.display( "\tto retry, use: --limit @%s\n" % filename) self._tqm.send_callback('v2_playbook_on_stats', self._tqm._stats) # if the last result wasn't zero, break out of the playbook file name loop if result != 0: break if entrylist: return entrylist finally: if self._tqm is not None: self._tqm.cleanup() if self._loader: self._loader.cleanup_all_tmp_files() if self._options.syntax: display.display("No issues encountered") return result return result def _get_serialized_batches(self, play): ''' Returns a list of hosts, subdivided into batches based on the serial size specified in the play. ''' # make sure we have a unique list of hosts all_hosts = self._inventory.get_hosts(play.hosts) all_hosts_len = len(all_hosts) # the serial value can be listed as a scalar or a list of # scalars, so we make sure it's a list here serial_batch_list = play.serial if len(serial_batch_list) == 0: serial_batch_list = [-1] cur_item = 0 serialized_batches = [] while len(all_hosts) > 0: # get the serial value from current item in the list serial = pct_to_int(serial_batch_list[cur_item], all_hosts_len) # if the serial count was not specified or is invalid, default to # a list of all hosts, otherwise grab a chunk of the hosts equal # to the current serial item size if serial <= 0: serialized_batches.append(all_hosts) break else: play_hosts = [] for x in range(serial): if len(all_hosts) > 0: play_hosts.append(all_hosts.pop(0)) serialized_batches.append(play_hosts) # increment the current batch list item number, and if we've hit # the end keep using the last element until we've consumed all of # the hosts in the inventory cur_item += 1 if cur_item > len(serial_batch_list) - 1: cur_item = len(serial_batch_list) - 1 return serialized_batches def _generate_retry_inventory(self, retry_path, replay_hosts): ''' Called when a playbook run fails. It generates an inventory which allows re-running on ONLY the failed hosts. This may duplicate some variable information in group_vars/host_vars but that is ok, and expected. ''' try: makedirs_safe(os.path.dirname(retry_path)) with open(retry_path, 'w') as fd: for x in replay_hosts: fd.write("%s\n" % x) except Exception as e: display.warning("Could not create retry file '%s'.\n\t%s" % (retry_path, to_native(e))) return False return True
class Play(object): def __init__(self, host_file, basedir=None, tags=None, become_method='sudo', become='no', become_user='******', check=False, forks=100, remote_user='******', verbosity=None, extra_vars={}, vault_password_file=None, connection=None): self.succeeded_hosts = [] self.failed_hosts = [] if ',' not in host_file and not os.path.isfile(host_file): host_file += ',' loader = DataLoader() if vault_password_file is not None: compat_utils.set_vault_password(loader, vault_password_file) variable_manager = compat_utils.create_variable_manager( loader, host_file) if extra_vars: variable_manager.extra_vars = extra_vars if isinstance(tags, list): tags = ','.join(tags) if connection is None: connection = 'local' if host_file == 'localhost,' else 'smart' if basedir: compat_utils.set_basedir(loader, basedir) options = Options(connection=connection, become_user=become_user, become_method=become_method, become=become, check=check, forks=forks, diff=False, tags=tags, remote_user=remote_user, verbosity=verbosity) options.private_key_file = None options.module_path = None self._tqm = TaskQueueManager(inventory=variable_manager._inventory, variable_manager=variable_manager, loader=loader, passwords=None, options=options) self.runtime_errors = None @property def ssh_key(self): return self._tqm._options.private_key_file @ssh_key.setter def ssh_key(self, key): self._tqm._options.private_key_file = os.path.abspath( os.path.expanduser(key)) @property def callback(self): return self._tqm._stdout_callback @callback.setter def callback(self, _cb): self._tqm._stdout_callback = _cb def _play(self, play): self.runtime_errors = None if self._tqm._terminated: self._tqm = TaskQueueManager( inventory=self._tqm._variable_manager._inventory, variable_manager=self._tqm._variable_manager, loader=self._tqm._loader, passwords=None, options=self._tqm._options) new_play = compat_utils.update_vars(self._tqm, play) used_hosts = self._tqm._inventory.get_hosts(new_play.hosts) if len(used_hosts) == 0: self._tqm.send_callback('v2_playbook_on_play_start', new_play) self._tqm.send_callback('v2_playbook_on_no_hosts_matched') return True try: self._tqm.run(new_play) except (AnsibleError, AnsibleParserError) as e: self.runtime_errors = e.message return False hosts = sorted(self._tqm._stats.processed.keys()) for h in hosts: t = self._tqm._stats.summarize(h) if t['unreachable'] > 0 or t['failures'] > 0: if h not in self.failed_hosts: self.failed_hosts.append(h) return False else: if h not in self.succeeded_hosts: self.succeeded_hosts.append(h) return True