def _get_action(self, task): """Get action based on task""" self._debug('_get_action...') try: action = action_loader.get(task.action, class_only=True) except KeyError: action = None return action
def plugin(): task = MagicMock(Task) task.action = "network" task._role = None connection = MagicMock() play_context = MagicMock() play_context.check_mode = False fake_loader = DictDataLoader({}) templar = Templar(loader=fake_loader) plugin = action_loader.get( "ansible.netcommon.network", task=task, connection=connection, play_context=play_context, loader=fake_loader, templar=templar, shared_loader_obj=None, ) return plugin
def run(self, iterator, play_context): ''' The "free" strategy is a bit more complex, in that it allows tasks to be sent to hosts as quickly as they can be processed. This means that some hosts may finish very quickly if run tasks result in little or no work being done versus other systems. The algorithm used here also tries to be more "fair" when iterating through hosts by remembering the last host in the list to be given a task and starting the search from there as opposed to the top of the hosts list again, which would end up favoring hosts near the beginning of the list. ''' # the last host to be given a task last_host = 0 result = self._tqm.RUN_OK # start with all workers being counted as being free workers_free = len(self._workers) work_to_do = True while work_to_do and not self._tqm._terminated: hosts_left = self.get_hosts_left(iterator) if len(hosts_left) == 0: self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') result = False break work_to_do = False # assume we have no more work to do starting_host = last_host # save current position so we know when we've looped back around and need to break # try and find an unblocked host with a task to run host_results = [] while True: host = hosts_left[last_host] display.debug("next free host: %s" % host) host_name = host.get_name() # peek at the next task for the host, to see if there's # anything to do do for this host (state, task) = iterator.get_next_task_for_host(host, peek=True) display.debug("free host state: %s" % state, host=host_name) display.debug("free host task: %s" % task, host=host_name) if host_name not in self._tqm._unreachable_hosts and task: # set the flag so the outer loop knows we've still found # some work which needs to be done work_to_do = True display.debug("this host has work to do", host=host_name) # check to see if this host is blocked (still executing a previous task) if host_name not in self._blocked_hosts or not self._blocked_hosts[host_name]: # pop the task, mark the host blocked, and queue it self._blocked_hosts[host_name] = True (state, task) = iterator.get_next_task_for_host(host) try: action = action_loader.get(task.action, class_only=True) except KeyError: # we don't care here, because the action may simply not have a # corresponding action plugin action = None display.debug("getting variables", host=host_name) task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task) self.add_tqm_variables(task_vars, play=iterator._play) templar = Templar(loader=self._loader, variables=task_vars) display.debug("done getting variables", host=host_name) try: task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating", host=host_name) except: # just ignore any errors during task name templating, # we don't care if it just shows the raw name display.debug("templating failed for some reason", host=host_name) run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) if run_once: if action and getattr(action, 'BYPASS_HOST_LOOP', False): raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " "and would instead execute for every host in the inventory list." % task.action, obj=task._ds) else: display.warning("Using run_once with the free strategy is not currently supported. This task will still be " "executed for every host in the inventory list.") # check to see if this task should be skipped, due to it being a member of a # role which has already run (and whether that role allows duplicate execution) if task._role and task._role.has_run(host): # If there is no metadata, the default behavior is to not allow duplicates, # if there is metadata, check to see if the allow_duplicates flag was set to true if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: display.debug("'%s' skipped because role has already run" % task, host=host_name) del self._blocked_hosts[host_name] continue if task.action == 'meta': self._execute_meta(task, play_context, iterator, target_host=host) self._blocked_hosts[host_name] = False else: # handle step if needed, skip meta actions as they are used internally if not self._step or self._take_step(task, host_name): if task.any_errors_fatal: display.warning("Using any_errors_fatal with the free strategy is not supported, " "as tasks are executed independently on each host") self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) self._queue_task(host, task, task_vars, play_context) # each task is counted as a worker being busy workers_free -= 1 del task_vars else: display.debug("%s is blocked, skipping for now" % host_name) # all workers have tasks to do (and the current host isn't done with the play). # loop back to starting host and break out if self._host_pinned and workers_free == 0 and work_to_do: last_host = starting_host break # move on to the next host and make sure we # haven't gone past the end of our hosts list last_host += 1 if last_host > len(hosts_left) - 1: last_host = 0 # if we've looped around back to the start, break out if last_host == starting_host: break results = self._process_pending_results(iterator) host_results.extend(results) # each result is counted as a worker being free again workers_free += len(results) self.update_active_connections(results) try: included_files = IncludedFile.process_include_results( host_results, iterator=iterator, loader=self._loader, variable_manager=self._variable_manager ) except AnsibleError as e: return self._tqm.RUN_ERROR if len(included_files) > 0: all_blocks = dict((host, []) for host in hosts_left) for included_file in included_files: display.debug("collecting new blocks for %s" % included_file) try: if included_file._is_role: new_ir = self._copy_included_file(included_file) new_blocks, handler_blocks = new_ir.get_block_list( play=iterator._play, variable_manager=self._variable_manager, loader=self._loader, ) self._tqm.update_handler_list([handler for handler_block in handler_blocks for handler in handler_block.block]) else: new_blocks = self._load_included_file(included_file, iterator=iterator) except AnsibleError as e: for host in included_file._hosts: iterator.mark_host_failed(host) display.warning(str(e)) continue for new_block in new_blocks: task_vars = self._variable_manager.get_vars(play=iterator._play, task=included_file._task) final_block = new_block.filter_tagged_tasks(play_context, task_vars) for host in hosts_left: if host in included_file._hosts: all_blocks[host].append(final_block) display.debug("done collecting new blocks for %s" % included_file) display.debug("adding all collected blocks from %d included file(s) to iterator" % len(included_files)) for host in hosts_left: iterator.add_tasks(host, all_blocks[host]) display.debug("done adding collected blocks to iterator") # pause briefly so we don't spin lock time.sleep(C.DEFAULT_INTERNAL_POLL_INTERVAL) # collect all the final results results = self._wait_on_pending_results(iterator) # run the base class run() method, which executes the cleanup function # and runs any outstanding handlers which have been triggered return super(StrategyModule, self).run(iterator, play_context, result)
def run(self, iterator, play_context): ''' The linear strategy is simple - get the next task and queue it for all hosts, then wait for the queue to drain before moving on to the next task ''' # iteratate over each task, while there is one left to run result = self._tqm.RUN_OK work_to_do = True while work_to_do and not self._tqm._terminated: try: display.debug("getting the remaining hosts for this loop") hosts_left = self.get_hosts_left(iterator) display.debug("done getting the remaining hosts for this loop") # queue up this task for each host in the inventory callback_sent = False work_to_do = False host_results = [] host_tasks = self._get_next_task_lockstep(hosts_left, iterator) # skip control skip_rest = False choose_step = True # flag set if task is set to any_errors_fatal any_errors_fatal = False results = [] for (host, task) in host_tasks: if not task: continue if self._tqm._terminated: break run_once = False work_to_do = True # test to see if the task across all hosts points to an action plugin which # sets BYPASS_HOST_LOOP to true, or if it has run_once enabled. If so, we # will only send this task to the first host in the list. try: action = action_loader.get(task.action, class_only=True) except KeyError: # we don't care here, because the action may simply not have a # corresponding action plugin action = None # check to see if this task should be skipped, due to it being a member of a # role which has already run (and whether that role allows duplicate execution) if task._role and task._role.has_run(host): # If there is no metadata, the default behavior is to not allow duplicates, # if there is metadata, check to see if the allow_duplicates flag was set to true if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: display.debug( "'%s' skipped because role has already run" % task) continue if task.action == 'meta': # for the linear strategy, we run meta tasks just once and for # all hosts currently being iterated over rather than one host results.extend( self._execute_meta(task, play_context, iterator, host)) if task.args.get('_raw_params', None) not in ('noop', 'reset_connection', 'end_host'): run_once = True if (task.any_errors_fatal or run_once) and not task.ignore_errors: any_errors_fatal = True else: # handle step if needed, skip meta actions as they are used internally if self._step and choose_step: if self._take_step(task): choose_step = False else: skip_rest = True break display.debug("getting variables") task_vars = self._variable_manager.get_vars( play=iterator._play, host=host, task=task) self.add_tqm_variables(task_vars, play=iterator._play) templar = Templar(loader=self._loader, variables=task_vars) display.debug("done getting variables") run_once = templar.template( task.run_once) or action and getattr( action, 'BYPASS_HOST_LOOP', False) if (task.any_errors_fatal or run_once) and not task.ignore_errors: any_errors_fatal = True if not callback_sent: display.debug( "sending task start callback, copying the task so we can template it temporarily" ) saved_name = task.name display.debug( "done copying, going to template now") try: task.name = to_text(templar.template( task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, # we don't care if it just shows the raw name display.debug( "templating failed for some reason") display.debug("here goes the callback...") self._tqm.send_callback( 'v2_playbook_on_task_start', task, is_conditional=False) task.name = saved_name callback_sent = True display.debug("sending task start callback") self._blocked_hosts[host.get_name()] = True self._queue_task(host, task, task_vars, play_context) del task_vars # if we're bypassing the host loop, break out now if run_once: break results += self._process_pending_results( iterator, max_passes=max(1, int(len(self._tqm._workers) * 0.1))) # go to next host/task group if skip_rest: continue display.debug( "done queuing things up, now waiting for results queue to drain" ) if self._pending_results > 0: results += self._wait_on_pending_results(iterator) host_results.extend(results) self.update_active_connections(results) try: included_files = IncludedFile.process_include_results( host_results, iterator=iterator, loader=self._loader, variable_manager=self._variable_manager) except AnsibleError as e: # this is a fatal error, so we abort here regardless of block state return self._tqm.RUN_ERROR include_failure = False if len(included_files) > 0: display.debug("we have included files to process") display.debug("generating all_blocks data") all_blocks = dict((host, []) for host in hosts_left) display.debug("done generating all_blocks data") for included_file in included_files: display.debug("processing included file: %s" % included_file._filename) # included hosts get the task list while those excluded get an equal-length # list of noop tasks, to make sure that they continue running in lock-step try: if included_file._is_role: new_ir = self._copy_included_file( included_file) new_blocks, handler_blocks = new_ir.get_block_list( play=iterator._play, variable_manager=self._variable_manager, loader=self._loader, ) self._tqm.update_handler_list([ handler for handler_block in handler_blocks for handler in handler_block.block ]) else: new_blocks = self._load_included_file( included_file, iterator=iterator) display.debug( "iterating over new_blocks loaded from include file" ) for new_block in new_blocks: task_vars = self._variable_manager.get_vars( play=iterator._play, task=new_block._parent) display.debug("filtering new block on tags") final_block = new_block.filter_tagged_tasks( play_context, task_vars) display.debug( "done filtering new block on tags") noop_block = self._prepare_and_create_noop_block_from( final_block, task._parent, iterator) for host in hosts_left: if host in included_file._hosts: all_blocks[host].append(final_block) else: all_blocks[host].append(noop_block) display.debug( "done iterating over new_blocks loaded from include file" ) except AnsibleError as e: for host in included_file._hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) display.error(to_text(e), wrap_text=False) include_failure = True continue # finally go through all of the hosts and append the # accumulated blocks to their list of tasks display.debug( "extending task lists for all hosts with included blocks" ) for host in hosts_left: iterator.add_tasks(host, all_blocks[host]) display.debug("done extending task lists") display.debug("done processing included files") display.debug("results queue empty") display.debug("checking for any_errors_fatal") failed_hosts = [] unreachable_hosts = [] for res in results: # execute_meta() does not set 'failed' in the TaskResult # so we skip checking it with the meta tasks and look just at the iterator if (res.is_failed() or res._task.action == 'meta') and iterator.is_failed(res._host): failed_hosts.append(res._host.name) elif res.is_unreachable(): unreachable_hosts.append(res._host.name) # if any_errors_fatal and we had an error, mark all hosts as failed if any_errors_fatal and (len(failed_hosts) > 0 or len(unreachable_hosts) > 0): dont_fail_states = frozenset( [iterator.ITERATING_RESCUE, iterator.ITERATING_ALWAYS]) for host in hosts_left: (s, _) = iterator.get_next_task_for_host(host, peek=True) if s.run_state not in dont_fail_states or \ s.run_state == iterator.ITERATING_RESCUE and s.fail_state & iterator.FAILED_RESCUE != 0: self._tqm._failed_hosts[host.name] = True result |= self._tqm.RUN_FAILED_BREAK_PLAY display.debug("done checking for any_errors_fatal") display.debug("checking for max_fail_percentage") if iterator._play.max_fail_percentage is not None and len( results) > 0: percentage = iterator._play.max_fail_percentage / 100.0 if (len(self._tqm._failed_hosts) / iterator.batch_size) > percentage: for host in hosts_left: # don't double-mark hosts, or the iterator will potentially # fail them out of the rescue/always states if host.name not in failed_hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) self._tqm.send_callback( 'v2_playbook_on_no_hosts_remaining') result |= self._tqm.RUN_FAILED_BREAK_PLAY display.debug('(%s failed / %s total )> %s max fail' % (len(self._tqm._failed_hosts), iterator.batch_size, percentage)) display.debug("done checking for max_fail_percentage") display.debug( "checking to see if all hosts have failed and the running result is not ok" ) if result != self._tqm.RUN_OK and len( self._tqm._failed_hosts) >= len(hosts_left): display.debug("^ not ok, so returning result now") self._tqm.send_callback( 'v2_playbook_on_no_hosts_remaining') return result display.debug("done checking to see if all hosts have failed") except (IOError, EOFError) as e: display.debug("got IOError/EOFError in task loop: %s" % e) # most likely an abort, return failed return self._tqm.RUN_UNKNOWN_ERROR # run the base class run() method, which executes the cleanup function # and runs any outstanding handlers which have been triggered return super(StrategyModule, self).run(iterator, play_context, result)
def _do_handler_run(self, handler, handler_name, iterator, play_context, notified_hosts=None): # FIXME: need to use iterator.get_failed_hosts() instead? # if not len(self.get_hosts_remaining(iterator._play)): # self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') # result = False # break saved_name = handler.name handler.name = handler_name self._tqm.send_callback('v2_playbook_on_handler_task_start', handler) handler.name = saved_name if notified_hosts is None: notified_hosts = self._notified_handlers[handler._uuid] run_once = False try: action = action_loader.get(handler.action, class_only=True) if handler.run_once or getattr(action, 'BYPASS_HOST_LOOP', False): run_once = True except KeyError: # we don't care here, because the action may simply not have a # corresponding action plugin pass host_results = [] for host in notified_hosts: if not handler.has_triggered(host) and ( not iterator.is_failed(host) or play_context.force_handlers): task_vars = self._variable_manager.get_vars( play=iterator._play, host=host, task=handler) self.add_tqm_variables(task_vars, play=iterator._play) self._queue_task(host, handler, task_vars, play_context) if run_once: break # collect the results from the handler run host_results = self._wait_on_pending_results(iterator) try: included_files = IncludedFile.process_include_results( host_results, self._tqm, iterator=iterator, inventory=self._inventory, loader=self._loader, variable_manager=self._variable_manager) except AnsibleError as e: return False result = True if len(included_files) > 0: for included_file in included_files: try: new_blocks = self._load_included_file(included_file, iterator=iterator, is_handler=True) # for every task in each block brought in by the include, add the list # of hosts which included the file to the notified_handlers dict for block in new_blocks: iterator._play.handlers.append(block) iterator.cache_block_tasks(block) for task in block.block: result = self._do_handler_run( handler=task, handler_name=None, iterator=iterator, play_context=play_context, notified_hosts=included_file._hosts[:], ) if not result: break except AnsibleError as e: for host in included_file._hosts: iterator.mark_host_failed(host) self._tqm._failed_hosts[host.name] = True display.warning(str(e)) continue # wipe the notification list self._notified_handlers[handler._uuid] = [] display.debug("done running handlers, result is: %s" % result) return result
def run(self, iterator, play_context): ''' The linear strategy is simple - get the next task and queue it for all hosts, then wait for the queue to drain before moving on to the next task ''' # iteratate over each task, while there is one left to run result = self._tqm.RUN_OK work_to_do = True while work_to_do and not self._tqm._terminated: try: display.debug("getting the remaining hosts for this loop") hosts_left = self.get_hosts_left(iterator) display.debug("done getting the remaining hosts for this loop") # queue up this task for each host in the inventory callback_sent = False work_to_do = False host_results = [] host_tasks = self._get_next_task_lockstep(hosts_left, iterator) # skip control skip_rest = False choose_step = True # flag set if task is set to any_errors_fatal any_errors_fatal = False succeeded_hosts = {} retry_task = True while retry_task: results = [] for (host, task) in host_tasks: if not task: continue if self._tqm._terminated: break run_once = False work_to_do = True # test to see if the task across all hosts points to an action plugin which # sets BYPASS_HOST_LOOP to true, or if it has run_once enabled. If so, we # will only send this task to the first host in the list. try: action = action_loader.get(task.action, class_only=True) except KeyError: # we don't care here, because the action may simply not have a # corresponding action plugin action = None # check to see if this task should be skipped, due to it being a member of a # role which has already run (and whether that role allows duplicate execution) if task._role and task._role.has_run(host): # If there is no metadata, the default behavior is to not allow duplicates, # if there is metadata, check to see if the allow_duplicates flag was set to true if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: display.debug( "'%s' skipped because role has already run" % task) continue if task.action == 'meta': # for the linear strategy, we run meta tasks just once and for # all hosts currently being iterated over rather than one host results.extend( self._execute_meta(task, play_context, iterator, host)) if task.args.get('_raw_params', None) != 'noop': run_once = True else: # handle step if needed, skip meta actions as they are used internally if self._step and choose_step: if self._take_step(task): choose_step = False else: skip_rest = True break display.debug("getting variables") task_vars = self._variable_manager.get_vars( play=iterator._play, host=host, task=task) self.add_tqm_variables(task_vars, play=iterator._play) templar = Templar(loader=self._loader, variables=task_vars) display.debug("done getting variables") run_once = templar.template( task.run_once) or action and getattr( action, 'BYPASS_HOST_LOOP', False) if (task.any_errors_fatal or run_once) and not task.ignore_errors: any_errors_fatal = True if not callback_sent: display.debug( "sending task start callback, copying the task so we can template it temporarily" ) saved_name = task.name display.debug( "done copying, going to template now") try: task.name = to_text(templar.template( task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, # we don't care if it just shows the raw name display.debug( "templating failed for some reason") pass display.debug("here goes the callback...") self._tqm.send_callback( 'v2_playbook_on_task_start', task, is_conditional=False) task.name = saved_name callback_sent = True display.debug("sending task start callback") if host not in succeeded_hosts: self._blocked_hosts[host.get_name()] = True self._queue_task(host, task, task_vars, play_context) del task_vars # if we're bypassing the host loop, break out now if run_once: break results += self._process_pending_results( iterator, max_passes=max(1, int(len(self._tqm._workers) * 0.1))) # go to next host/task group if skip_rest: continue display.debug( "done queuing things up, now waiting for results queue to drain" ) if self._pending_results > 0: results += self._wait_on_pending_results(iterator) host_results.extend(results) try: included_files = IncludedFile.process_include_results( host_results, self._tqm, iterator=iterator, inventory=self._inventory, loader=self._loader, variable_manager=self._variable_manager) except AnsibleError as e: # this is a fatal error, so we abort here regardless of block state return self._tqm.RUN_ERROR include_failure = False if len(included_files) > 0: display.debug("we have included files to process") # A noop task for use in padding dynamic includes noop_task = Task() noop_task.action = 'meta' noop_task.args['_raw_params'] = 'noop' noop_task.set_loader(iterator._play._loader) display.debug("generating all_blocks data") all_blocks = dict((host, []) for host in hosts_left) display.debug("done generating all_blocks data") for included_file in included_files: display.debug("processing included file: %s" % included_file._filename) # included hosts get the task list while those excluded get an equal-length # list of noop tasks, to make sure that they continue running in lock-step try: if included_file._is_role: new_ir = included_file._task.copy() new_ir.vars.update(included_file._args) new_blocks, handler_blocks = new_ir.get_block_list( play=iterator._play, variable_manager=self. _variable_manager, loader=self._loader, ) self._tqm.update_handler_list([ handler for handler_block in handler_blocks for handler in handler_block.block ]) else: new_blocks = self._load_included_file( included_file, iterator=iterator) display.debug( "iterating over new_blocks loaded from include file" ) for new_block in new_blocks: task_vars = self._variable_manager.get_vars( play=iterator._play, task=included_file._task, ) display.debug( "filtering new block on tags") final_block = new_block.filter_tagged_tasks( play_context, task_vars) display.debug( "done filtering new block on tags") noop_block = Block( parent_block=task._parent) noop_block.block = [ noop_task for t in new_block.block ] noop_block.always = [ noop_task for t in new_block.always ] noop_block.rescue = [ noop_task for t in new_block.rescue ] for host in hosts_left: if host in included_file._hosts: all_blocks[host].append( final_block) else: all_blocks[host].append(noop_block) display.debug( "done iterating over new_blocks loaded from include file" ) except AnsibleError as e: for host in included_file._hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) display.error(to_text(e), wrap_text=False) include_failure = True continue # finally go through all of the hosts and append the # accumulated blocks to their list of tasks display.debug( "extending task lists for all hosts with included blocks" ) for host in hosts_left: iterator.add_tasks(host, all_blocks[host]) display.debug("done extending task lists") display.debug("done processing included files") display.debug("results queue empty") display.debug("checking for any_errors_fatal") failed_hosts = [] unreachable_hosts = [] for res in results: if res.is_failed() and iterator.is_failed(res._host): failed_hosts.append(res._host.name) elif res.is_unreachable(): unreachable_hosts.append(res._host.name) else: succeeded_hosts[res._host] = res if (len(failed_hosts) == 0): # No more failures means we do not need to retry anymore. retry_task = False else: msg = 'Host failed, ignore/retry/abort/debug: %s on [%s] (i/r/a/d): ' % ( str(task), ' '.join(failed_hosts)) + '\n' resp = 'bogus' while resp.strip().lower()[:1] not in [ 'i', 'r', 'a', 'd' ]: resp = display.prompt(msg) # Currently the only difference between ignore and repeat is setting the retry_task to 'False' if resp.strip().lower() in ['i', 'ignore']: retry_task = False if resp.strip().lower() in [ 'r', 'repeat', 'i', 'ignore' ]: # We need to revert the internal failed host states if we want to ignore the errors # The internal state should be identical to that of a successful host for failed_host in failed_hosts: iterator._host_states[ failed_host].run_state = PlayIterator.ITERATING_TASKS iterator._host_states[ failed_host].fail_state = PlayIterator.FAILED_NONE # Fix child states as well if iterator._host_states[ failed_host].tasks_child_state: iterator._host_states[ failed_host].tasks_child_state.run_state = PlayIterator.ITERATING_TASKS iterator._host_states[ failed_host].tasks_child_state.fail_state = PlayIterator.FAILED_NONE self._tqm.clear_failed_hosts() elif resp.strip().lower() in ['a', 'abort']: retry_task = False for host in hosts_left: (s, _) = iterator.get_next_task_for_host( host, peek=True) if s.run_state != iterator.ITERATING_RESCUE or \ s.run_state == iterator.ITERATING_RESCUE and s.fail_state & iterator.FAILED_RESCUE != 0: self._tqm._failed_hosts[host.name] = True result |= self._tqm.RUN_FAILED_BREAK_PLAY # don't double-mark hosts, or the iterator will potentially # fail them out of the rescue/always states if host.name not in failed_hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) self._tqm.send_callback( 'v2_playbook_on_no_hosts_remaining') result |= self._tqm.RUN_FAILED_BREAK_PLAY elif resp.strip().lower() in ['d', 'debug']: msg = "Running PDB. Refer to https://pymotw.com/2/pdb/ for instruction.\n\n" msg += "Commmon commands:\n\n" msg += "Print the task's error message: print(failure_debug_message)\n" msg += 'Add a variable to a host: host_to_edit = hosts_left[0]; task._variable_manager.set_host_variable(host_to_edit,"new_var_key","new_var_value")\n' msg += 'Print task vars: print(task_vars["var_key_name"])\n' display.display(msg, color='blue', stderr=False, screen_only=False, log_only=False) pdb.set_trace() except (IOError, EOFError) as e: display.debug("got IOError/EOFError in task loop: %s" % e) # most likely an abort, return failed return self._tqm.RUN_UNKNOWN_ERROR # run the base class run() method, which executes the cleanup function # and runs any outstanding handlers which have been triggered return super(StrategyModule, self).run(iterator, play_context, result)
def run(self, iterator, play_context): ''' The linear strategy is simple - get the next task and queue it for all hosts, then wait for the queue to drain before moving on to the next task ''' # iteratate over each task, while there is one left to run result = self._tqm.RUN_OK work_to_do = True while work_to_do and not self._tqm._terminated: try: display.debug("getting the remaining hosts for this loop") hosts_left = self.get_hosts_left(iterator) display.debug("done getting the remaining hosts for this loop") # queue up this task for each host in the inventory callback_sent = False work_to_do = False host_results = [] host_tasks = self._get_next_task_lockstep(hosts_left, iterator) # skip control skip_rest = False choose_step = True # flag set if task is set to any_errors_fatal any_errors_fatal = False results = [] for (host, task) in host_tasks: if not task: continue if self._tqm._terminated: break run_once = False work_to_do = True # test to see if the task across all hosts points to an action plugin which # sets BYPASS_HOST_LOOP to true, or if it has run_once enabled. If so, we # will only send this task to the first host in the list. try: action = action_loader.get(task.action, class_only=True) except KeyError: # we don't care here, because the action may simply not have a # corresponding action plugin action = None # check to see if this task should be skipped, due to it being a member of a # role which has already run (and whether that role allows duplicate execution) if task._role and task._role.has_run(host): # If there is no metadata, the default behavior is to not allow duplicates, # if there is metadata, check to see if the allow_duplicates flag was set to true if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: display.debug("'%s' skipped because role has already run" % task) continue if task.action == 'meta': # for the linear strategy, we run meta tasks just once and for # all hosts currently being iterated over rather than one host results.extend(self._execute_meta(task, play_context, iterator, host)) if task.args.get('_raw_params', None) != 'noop': run_once = True else: # handle step if needed, skip meta actions as they are used internally if self._step and choose_step: if self._take_step(task): choose_step = False else: skip_rest = True break display.debug("getting variables") task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task) self.add_tqm_variables(task_vars, play=iterator._play) templar = Templar(loader=self._loader, variables=task_vars) display.debug("done getting variables") run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) if (task.any_errors_fatal or run_once) and not task.ignore_errors: any_errors_fatal = True if not callback_sent: display.debug("sending task start callback, copying the task so we can template it temporarily") saved_name = task.name display.debug("done copying, going to template now") try: task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating") except: # just ignore any errors during task name templating, # we don't care if it just shows the raw name display.debug("templating failed for some reason") display.debug("here goes the callback...") self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) task.name = saved_name callback_sent = True display.debug("sending task start callback") self._blocked_hosts[host.get_name()] = True self._queue_task(host, task, task_vars, play_context) del task_vars # if we're bypassing the host loop, break out now if run_once: break results += self._process_pending_results(iterator, max_passes=max(1, int(len(self._tqm._workers) * 0.1))) # go to next host/task group if skip_rest: continue display.debug("done queuing things up, now waiting for results queue to drain") if self._pending_results > 0: results += self._wait_on_pending_results(iterator) host_results.extend(results) try: included_files = IncludedFile.process_include_results( host_results, self._tqm, iterator=iterator, inventory=self._inventory, loader=self._loader, variable_manager=self._variable_manager ) except AnsibleError as e: # this is a fatal error, so we abort here regardless of block state return self._tqm.RUN_ERROR include_failure = False if len(included_files) > 0: display.debug("we have included files to process") # A noop task for use in padding dynamic includes noop_task = Task() noop_task.action = 'meta' noop_task.args['_raw_params'] = 'noop' noop_task.set_loader(iterator._play._loader) display.debug("generating all_blocks data") all_blocks = dict((host, []) for host in hosts_left) display.debug("done generating all_blocks data") for included_file in included_files: display.debug("processing included file: %s" % included_file._filename) # included hosts get the task list while those excluded get an equal-length # list of noop tasks, to make sure that they continue running in lock-step try: if included_file._is_role: new_ir = included_file._task.copy() new_ir.vars.update(included_file._args) new_blocks, handler_blocks = new_ir.get_block_list( play=iterator._play, variable_manager=self._variable_manager, loader=self._loader, ) self._tqm.update_handler_list([handler for handler_block in handler_blocks for handler in handler_block.block]) else: new_blocks = self._load_included_file(included_file, iterator=iterator) display.debug("iterating over new_blocks loaded from include file") for new_block in new_blocks: task_vars = self._variable_manager.get_vars( play=iterator._play, task=included_file._task, ) display.debug("filtering new block on tags") final_block = new_block.filter_tagged_tasks(play_context, task_vars) display.debug("done filtering new block on tags") noop_block = Block(parent_block=task._parent) noop_block.block = [noop_task for t in new_block.block] noop_block.always = [noop_task for t in new_block.always] noop_block.rescue = [noop_task for t in new_block.rescue] for host in hosts_left: if host in included_file._hosts: all_blocks[host].append(final_block) else: all_blocks[host].append(noop_block) display.debug("done iterating over new_blocks loaded from include file") except AnsibleError as e: for host in included_file._hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) display.error(to_text(e), wrap_text=False) include_failure = True continue # finally go through all of the hosts and append the # accumulated blocks to their list of tasks display.debug("extending task lists for all hosts with included blocks") for host in hosts_left: iterator.add_tasks(host, all_blocks[host]) display.debug("done extending task lists") display.debug("done processing included files") display.debug("results queue empty") display.debug("checking for any_errors_fatal") failed_hosts = [] unreachable_hosts = [] for res in results: if res.is_failed() and iterator.is_failed(res._host): failed_hosts.append(res._host.name) elif res.is_unreachable(): unreachable_hosts.append(res._host.name) # if any_errors_fatal and we had an error, mark all hosts as failed if any_errors_fatal and (len(failed_hosts) > 0 or len(unreachable_hosts) > 0): for host in hosts_left: (s, _) = iterator.get_next_task_for_host(host, peek=True) if s.run_state != iterator.ITERATING_RESCUE or \ s.run_state == iterator.ITERATING_RESCUE and s.fail_state & iterator.FAILED_RESCUE != 0: self._tqm._failed_hosts[host.name] = True result |= self._tqm.RUN_FAILED_BREAK_PLAY display.debug("done checking for any_errors_fatal") display.debug("checking for max_fail_percentage") if iterator._play.max_fail_percentage is not None and len(results) > 0: percentage = iterator._play.max_fail_percentage / 100.0 if (len(self._tqm._failed_hosts) / len(results)) > percentage: for host in hosts_left: # don't double-mark hosts, or the iterator will potentially # fail them out of the rescue/always states if host.name not in failed_hosts: self._tqm._failed_hosts[host.name] = True iterator.mark_host_failed(host) self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') result |= self._tqm.RUN_FAILED_BREAK_PLAY display.debug("done checking for max_fail_percentage") display.debug("checking to see if all hosts have failed and the running result is not ok") if result != self._tqm.RUN_OK and len(self._tqm._failed_hosts) >= len(hosts_left): display.debug("^ not ok, so returning result now") self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') return result display.debug("done checking to see if all hosts have failed") except (IOError, EOFError) as e: display.debug("got IOError/EOFError in task loop: %s" % e) # most likely an abort, return failed return self._tqm.RUN_UNKNOWN_ERROR # run the base class run() method, which executes the cleanup function # and runs any outstanding handlers which have been triggered return super(StrategyModule, self).run(iterator, play_context, result)
def run(self, iterator, play_context): ''' The "free" strategy is a bit more complex, in that it allows tasks to be sent to hosts as quickly as they can be processed. This means that some hosts may finish very quickly if run tasks result in little or no work being done versus other systems. The algorithm used here also tries to be more "fair" when iterating through hosts by remembering the last host in the list to be given a task and starting the search from there as opposed to the top of the hosts list again, which would end up favoring hosts near the beginning of the list. ''' # the last host to be given a task last_host = 0 result = self._tqm.RUN_OK work_to_do = True while work_to_do and not self._tqm._terminated: hosts_left = self.get_hosts_left(iterator) if len(hosts_left) == 0: self._tqm.send_callback('v2_playbook_on_no_hosts_remaining') result = False break work_to_do = False # assume we have no more work to do starting_host = last_host # save current position so we know when we've looped back around and need to break # try and find an unblocked host with a task to run host_results = [] while True: host = hosts_left[last_host] display.debug("next free host: %s" % host) host_name = host.get_name() # peek at the next task for the host, to see if there's # anything to do do for this host (state, task) = iterator.get_next_task_for_host(host, peek=True) display.debug("free host state: %s" % state, host=host_name) display.debug("free host task: %s" % task, host=host_name) if host_name not in self._tqm._unreachable_hosts and task: # set the flag so the outer loop knows we've still found # some work which needs to be done work_to_do = True display.debug("this host has work to do", host=host_name) # check to see if this host is blocked (still executing a previous task) if host_name not in self._blocked_hosts or not self._blocked_hosts[host_name]: # pop the task, mark the host blocked, and queue it self._blocked_hosts[host_name] = True (state, task) = iterator.get_next_task_for_host(host) try: action = action_loader.get(task.action, class_only=True) except KeyError: # we don't care here, because the action may simply not have a # corresponding action plugin action = None display.debug("getting variables", host=host_name) task_vars = self._variable_manager.get_vars(play=iterator._play, host=host, task=task) self.add_tqm_variables(task_vars, play=iterator._play) templar = Templar(loader=self._loader, variables=task_vars) display.debug("done getting variables", host=host_name) try: task.name = to_text(templar.template(task.name, fail_on_undefined=False), nonstring='empty') display.debug("done templating", host=host_name) except: # just ignore any errors during task name templating, # we don't care if it just shows the raw name display.debug("templating failed for some reason", host=host_name) run_once = templar.template(task.run_once) or action and getattr(action, 'BYPASS_HOST_LOOP', False) if run_once: if action and getattr(action, 'BYPASS_HOST_LOOP', False): raise AnsibleError("The '%s' module bypasses the host loop, which is currently not supported in the free strategy " "and would instead execute for every host in the inventory list." % task.action, obj=task._ds) else: display.warning("Using run_once with the free strategy is not currently supported. This task will still be " "executed for every host in the inventory list.") # check to see if this task should be skipped, due to it being a member of a # role which has already run (and whether that role allows duplicate execution) if task._role and task._role.has_run(host): # If there is no metadata, the default behavior is to not allow duplicates, # if there is metadata, check to see if the allow_duplicates flag was set to true if task._role._metadata is None or task._role._metadata and not task._role._metadata.allow_duplicates: display.debug("'%s' skipped because role has already run" % task, host=host_name) del self._blocked_hosts[host_name] continue if task.action == 'meta': self._execute_meta(task, play_context, iterator, target_host=host) self._blocked_hosts[host_name] = False else: # handle step if needed, skip meta actions as they are used internally if not self._step or self._take_step(task, host_name): if task.any_errors_fatal: display.warning("Using any_errors_fatal with the free strategy is not supported, " "as tasks are executed independently on each host") self._tqm.send_callback('v2_playbook_on_task_start', task, is_conditional=False) self._queue_task(host, task, task_vars, play_context) del task_vars else: display.debug("%s is blocked, skipping for now" % host_name) # move on to the next host and make sure we # haven't gone past the end of our hosts list last_host += 1 if last_host > len(hosts_left) - 1: last_host = 0 # if we've looped around back to the start, break out if last_host == starting_host: break results = self._process_pending_results(iterator) host_results.extend(results) self.update_active_connections(results) try: included_files = IncludedFile.process_include_results( host_results, iterator=iterator, loader=self._loader, variable_manager=self._variable_manager ) except AnsibleError as e: return self._tqm.RUN_ERROR if len(included_files) > 0: all_blocks = dict((host, []) for host in hosts_left) for included_file in included_files: display.debug("collecting new blocks for %s" % included_file) try: if included_file._is_role: new_ir = self._copy_included_file(included_file) new_blocks, handler_blocks = new_ir.get_block_list( play=iterator._play, variable_manager=self._variable_manager, loader=self._loader, ) self._tqm.update_handler_list([handler for handler_block in handler_blocks for handler in handler_block.block]) else: new_blocks = self._load_included_file(included_file, iterator=iterator) except AnsibleError as e: for host in included_file._hosts: iterator.mark_host_failed(host) display.warning(str(e)) continue for new_block in new_blocks: task_vars = self._variable_manager.get_vars(play=iterator._play, task=included_file._task) final_block = new_block.filter_tagged_tasks(play_context, task_vars) for host in hosts_left: if host in included_file._hosts: all_blocks[host].append(final_block) display.debug("done collecting new blocks for %s" % included_file) display.debug("adding all collected blocks from %d included file(s) to iterator" % len(included_files)) for host in hosts_left: iterator.add_tasks(host, all_blocks[host]) display.debug("done adding collected blocks to iterator") # pause briefly so we don't spin lock time.sleep(C.DEFAULT_INTERNAL_POLL_INTERVAL) # collect all the final results results = self._wait_on_pending_results(iterator) # run the base class run() method, which executes the cleanup function # and runs any outstanding handlers which have been triggered return super(StrategyModule, self).run(iterator, play_context, result)