def test_task_executor_execute(self): fake_loader = DictDataLoader({}) mock_host = MagicMock() mock_task = MagicMock() mock_task.args = dict() mock_task.retries = 0 mock_task.delay = -1 mock_task.register = 'foo' mock_task.until = None mock_task.changed_when = None mock_task.failed_when = None mock_task.post_validate.return_value = None mock_conn_info = MagicMock() mock_conn_info.post_validate.return_value = None mock_conn_info.update_vars.return_value = None mock_connection = MagicMock() mock_connection.set_host_overrides.return_value = None mock_connection._connect.return_value = None mock_action = MagicMock() shared_loader = None new_stdin = None job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX") te = TaskExecutor( host = mock_host, task = mock_task, job_vars = job_vars, connection_info = mock_conn_info, new_stdin = new_stdin, loader = fake_loader, shared_loader_obj = shared_loader, ) te._get_connection = MagicMock(return_value=mock_connection) te._get_action_handler = MagicMock(return_value=mock_action) mock_action.run.return_value = dict(ansible_facts=dict()) res = te._execute() mock_task.changed_when = "1 == 1" res = te._execute() mock_task.changed_when = None mock_task.failed_when = "1 == 1" res = te._execute() mock_task.failed_when = None mock_task.evaluate_conditional.return_value = False res = te._execute() mock_task.evaluate_conditional.return_value = True mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar') mock_task.action = 'include' res = te._execute()
def test_task_executor_get_handler_normal(self): te = TaskExecutor( host=MagicMock(), task=MagicMock(), job_vars={}, play_context=MagicMock(), new_stdin=None, loader=DictDataLoader({}), shared_loader_obj=MagicMock(), final_q=MagicMock(), ) action_loader = te._shared_loader_obj.action_loader action_loader.has_plugin.return_value = False action_loader.get.return_value = mock.sentinel.handler action_loader.__contains__.return_value = False mock_connection = MagicMock() mock_templar = MagicMock() action = 'namespace.prefix_sufix' module_prefix = action.split('.')[-1].split('_')[0] te._task.action = action handler = te._get_action_handler(mock_connection, mock_templar) self.assertIs(mock.sentinel.handler, handler) action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections), mock.call(module_prefix, collection_list=te._task.collections)]) action_loader.get.assert_called_once_with( 'normal', task=te._task, connection=mock_connection, play_context=te._play_context, loader=te._loader, templar=mock_templar, shared_loader_obj=te._shared_loader_obj, collection_list=None)
def test_task_executor_get_action_handler(self): te = TaskExecutor( host=MagicMock(), task=MagicMock(), job_vars={}, play_context=MagicMock(), new_stdin=None, loader=DictDataLoader({}), shared_loader_obj=MagicMock(), final_q=MagicMock(), ) action_loader = te._shared_loader_obj.action_loader action_loader.has_plugin.return_value = True action_loader.get.return_value = mock.sentinel.handler mock_connection = MagicMock() mock_templar = MagicMock() action = 'namespace.prefix_sufix' te._task.action = action handler = te._get_action_handler(mock_connection, mock_templar) self.assertIs(mock.sentinel.handler, handler) action_loader.has_plugin.assert_called_once_with( action, collection_list=te._task.collections) action_loader.get.assert_called_once_with( te._task.action, task=te._task, connection=mock_connection, play_context=te._play_context, loader=te._loader, templar=mock_templar, shared_loader_obj=te._shared_loader_obj, collection_list=te._task.collections)
def run(self): ''' Called when the process is started, and loops indefinitely until an error is encountered (typically an IOerror from the queue pipe being disconnected). During the loop, we attempt to pull tasks off the job queue and run them, pushing the result onto the results queue. We also remove the host from the blocked hosts list, to signify that they are ready for their next task. ''' if HAS_ATFORK: atfork() try: # execute the task and build a TaskResult from the result display.debug("running TaskExecutor() for %s/%s" % (self._host, self._task)) executor_result = TaskExecutor(self._host, self._task, self._task_vars, self._play_context, self._new_stdin, self._loader, self._shared_loader_obj, self._rslt_q).run() display.debug("done running TaskExecutor() for %s/%s" % (self._host, self._task)) self._host.vars = dict() self._host.groups = [] task_result = TaskResult(self._host, self._task, executor_result) # put the result on the result queue display.debug("sending task result") self._rslt_q.put(task_result) display.debug("done sending task result") except AnsibleConnectionFailure: self._host.vars = dict() self._host.groups = [] task_result = TaskResult(self._host, self._task, dict(unreachable=True)) self._rslt_q.put(task_result, block=False) except Exception as e: if not isinstance(e, (IOError, EOFError, KeyboardInterrupt, SystemExit)) or isinstance(e, TemplateNotFound): try: self._host.vars = dict() self._host.groups = [] task_result = TaskResult( self._host, self._task, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout='')) self._rslt_q.put(task_result, block=False) except: display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e)) display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc())) display.debug("WORKER PROCESS EXITING")
def test_task_executor_get_loop_items(self): fake_loader = DictDataLoader({}) mock_host = MagicMock() mock_task = MagicMock() mock_task.loop = 'items' mock_task.loop_args = ['a', 'b', 'c'] mock_play_context = MagicMock() mock_shared_loader = MagicMock() mock_shared_loader.lookup_loader = lookup_loader new_stdin = None job_vars = dict() te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=mock_shared_loader, ) items = te._get_loop_items() self.assertEqual(items, ['a', 'b', 'c'])
def test_task_executor_poll_async_result(self): fake_loader = DictDataLoader({}) mock_host = MagicMock() mock_task = MagicMock() mock_task. async = 3 mock_task.poll = 1 mock_play_context = MagicMock() mock_connection = MagicMock() mock_action = MagicMock() shared_loader = MagicMock() shared_loader.action_loader = action_loader new_stdin = None job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX") te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=shared_loader, ) te._connection = MagicMock() def _get(*args, **kwargs): mock_action = MagicMock() mock_action.run.return_value = dict(stdout='') return mock_action # testing with some bad values in the result passed to poll async, # and with a bad value returned from the mock action with patch.object(action_loader, 'get', _get): mock_templar = MagicMock() res = te._poll_async_result(result=dict(), templar=mock_templar) self.assertIn('failed', res) res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar) self.assertIn('failed', res) def _get(*args, **kwargs): mock_action = MagicMock() mock_action.run.return_value = dict(finished=1) return mock_action # now testing with good values with patch.object(action_loader, 'get', _get): mock_templar = MagicMock() res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar) self.assertEqual(res, dict(finished=1))
def test_task_executor_squash_items(self): items = ['a', 'b', 'c'] fake_loader = DictDataLoader({}) mock_host = MagicMock() def _evaluate_conditional(templar, variables): item = variables.get('item') if item == 'b': return False return True mock_task = MagicMock() mock_task.evaluate_conditional.side_effect = _evaluate_conditional mock_play_context = MagicMock() mock_shared_loader = None new_stdin = None job_vars = dict(pkg_mgr='yum') te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=mock_shared_loader, ) mock_task.action = 'foo' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) mock_task.action = 'yum' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, [['a', 'c']]) mock_task.action = '{{pkg_mgr}}' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, [['a', 'c']]) # Smoketests -- these won't optimize but make sure that they don't # traceback either mock_task.action = '{{unknown}}' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) items = [ dict(name='a', state='present'), dict(name='b', state='present'), dict(name='c', state='present') ] mock_task.action = 'yum' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, items)
def run(self): ''' Called when the process is started. Pushes the result onto the results queue. We also remove the host from the blocked hosts list, to signify that they are ready for their next task. ''' #import cProfile, pstats, StringIO #pr = cProfile.Profile() #pr.enable() if HAS_ATFORK: atfork() try: # execute the task and build a TaskResult from the result display.debug("running TaskExecutor() for %s/%s" % (self._host, self._task)) executor_result = TaskExecutor( self._host, self._task, self._task_vars, self._play_context, self._new_stdin, self._loader, self._shared_loader_obj, self._rslt_q ).run() display.debug("done running TaskExecutor() for %s/%s" % (self._host, self._task)) self._host.vars = dict() self._host.groups = [] task_result = TaskResult(self._host.name, self._task._uuid, executor_result) # put the result on the result queue display.debug("sending task result") self._rslt_q.put(task_result) display.debug("done sending task result") except AnsibleConnectionFailure: self._host.vars = dict() self._host.groups = [] task_result = TaskResult(self._host.name, self._task._uuid, dict(unreachable=True)) self._rslt_q.put(task_result, block=False) except Exception as e: if not isinstance(e, (IOError, EOFError, KeyboardInterrupt, SystemExit)) or isinstance(e, TemplateNotFound): try: self._host.vars = dict() self._host.groups = [] task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout='')) self._rslt_q.put(task_result, block=False) except: display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e)) display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc())) display.debug("WORKER PROCESS EXITING")
def test_task_executor_init(self): fake_loader = DictDataLoader({}) mock_host = MagicMock() mock_task = MagicMock() mock_play_context = MagicMock() mock_shared_loader = MagicMock() new_stdin = None job_vars = dict() te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=mock_shared_loader, )
def test_task_executor_run(self): fake_loader = DictDataLoader({}) mock_host = MagicMock() mock_task = MagicMock() mock_task._role._role_path = '/path/to/role/foo' mock_play_context = MagicMock() mock_shared_loader = MagicMock() mock_queue = MagicMock() new_stdin = None job_vars = dict() te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=mock_shared_loader, rslt_q=mock_queue, ) te._get_loop_items = MagicMock(return_value=None) te._execute = MagicMock(return_value=dict()) res = te.run() te._get_loop_items = MagicMock(return_value=[]) res = te.run() te._get_loop_items = MagicMock(return_value=['a', 'b', 'c']) te._run_loop = MagicMock(return_value=[ dict(item='a', changed=True), dict(item='b', failed=True), dict(item='c') ]) res = te.run() te._get_loop_items = MagicMock(side_effect=AnsibleError("")) res = te.run() self.assertIn("failed", res)
def test_task_executor_get_handler_prefix(self): te = TaskExecutor( host=MagicMock(), task=MagicMock(), job_vars={}, play_context=MagicMock(), new_stdin=None, loader=DictDataLoader({}), shared_loader_obj=MagicMock(), final_q=MagicMock(), ) context = MagicMock(resolved=False) te._shared_loader_obj.module_loader.find_plugin_with_context.return_value = context action_loader = te._shared_loader_obj.action_loader action_loader.has_plugin.side_effect = [False, True] action_loader.get.return_value = mock.sentinel.handler action_loader.__contains__.return_value = True mock_connection = MagicMock() mock_templar = MagicMock() action = 'namespace.netconf_suffix' module_prefix = action.split('_', 1)[0] te._task.action = action handler = te._get_action_handler(mock_connection, mock_templar) self.assertIs(mock.sentinel.handler, handler) action_loader.has_plugin.assert_has_calls([ mock.call(action, collection_list=te._task.collections), # called twice mock.call(module_prefix, collection_list=te._task.collections) ]) action_loader.get.assert_called_once_with( module_prefix, task=te._task, connection=mock_connection, play_context=te._play_context, loader=te._loader, templar=mock_templar, shared_loader_obj=te._shared_loader_obj, collection_list=te._task.collections)
def test_task_executor_run_clean_res(self): te = TaskExecutor(None, MagicMock(), None, None, None, None, None, None) te._get_loop_items = MagicMock(return_value=[1]) te._run_loop = MagicMock( return_value=[{ 'unsafe_bytes': AnsibleUnsafeBytes(b'{{ $bar }}'), 'unsafe_text': AnsibleUnsafeText(u'{{ $bar }}'), 'bytes': b'bytes', 'text': u'text', 'int': 1, }]) res = te.run() data = res['results'][0] self.assertIsInstance(data['unsafe_bytes'], AnsibleUnsafeText) self.assertIsInstance(data['unsafe_text'], AnsibleUnsafeText) self.assertIsInstance(data['bytes'], text_type) self.assertIsInstance(data['text'], text_type) self.assertIsInstance(data['int'], int)
def test_task_executor_run_loop(self): items = ['a', 'b', 'c'] fake_loader = DictDataLoader({}) mock_host = MagicMock() def _copy(exclude_parent=False, exclude_tasks=False): new_item = MagicMock() return new_item mock_task = MagicMock() mock_task.copy.side_effect = _copy mock_play_context = MagicMock() mock_shared_loader = MagicMock() mock_queue = MagicMock() new_stdin = None job_vars = dict() te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=mock_shared_loader, rslt_q=mock_queue, ) def _execute(variables): return dict(item=variables.get('item')) te._squash_items = MagicMock(return_value=items) te._execute = MagicMock(side_effect=_execute) res = te._run_loop(items) self.assertEqual(len(res), 3)
def worker(main_q, res_q, loader): while True: task = None try: if not main_q.empty(): (host, task, task_vars, conn_info) = main_q.get(block=False) executor_result = TaskExecutor(host, task, task_vars, conn_info, loader).run() debug("executor result: %s" % executor_result) task_result = TaskResult(host, task, executor_result) res_q.put(task_result) else: time.sleep(0.01) except Queue.Empty: pass except (IOError, EOFError, KeyboardInterrupt) as e: debug("got a breaking error: %s" % e) break except Exception as e: debug("EXCEPTION DURING WORKER PROCESSING: %s" % e) traceback.print_exc() break
def test_task_executor_squash_items(self): items = ['a', 'b', 'c'] fake_loader = DictDataLoader({}) mock_host = MagicMock() def _evaluate_conditional(templar, variables): item = variables.get('item') if item == 'b': return False return True mock_task = MagicMock() mock_task.evaluate_conditional.side_effect = _evaluate_conditional mock_conn_info = MagicMock() mock_shared_loader = None new_stdin = None job_vars = dict() te = TaskExecutor( host = mock_host, task = mock_task, job_vars = job_vars, connection_info = mock_conn_info, new_stdin = new_stdin, loader = fake_loader, shared_loader_obj = mock_shared_loader, ) mock_task.action = 'foo' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) mock_task.action = 'yum' new_items = te._squash_items(items=items, variables=job_vars) self.assertEqual(new_items, ['a,c'])
def run(self): ''' Called when the process is started, and loops indefinitely until an error is encountered (typically an IOerror from the queue pipe being disconnected). During the loop, we attempt to pull tasks off the job queue and run them, pushing the result onto the results queue. We also remove the host from the blocked hosts list, to signify that they are ready for their next task. ''' if HAS_ATFORK: atfork() while True: task = None try: (host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get() debug("there's work to be done!") debug("got a task/handler to work on: %s" % task) # because the task queue manager starts workers (forks) before the # playbook is loaded, set the basedir of the loader inherted by # this fork now so that we can find files correctly self._loader.set_basedir(basedir) # Serializing/deserializing tasks does not preserve the loader attribute, # since it is passed to the worker during the forking of the process and # would be wasteful to serialize. So we set it here on the task now, and # the task handles updating parent/child objects as needed. task.set_loader(self._loader) # execute the task and build a TaskResult from the result debug("running TaskExecutor() for %s/%s" % (host, task)) executor_result = TaskExecutor(host, task, job_vars, play_context, self._new_stdin, self._loader, shared_loader_obj).run() debug("done running TaskExecutor() for %s/%s" % (host, task)) task_result = TaskResult(host, task, executor_result) # put the result on the result queue debug("sending task result") self._rslt_q.put(task_result) debug("done sending task result") except queue.Empty: pass except AnsibleConnectionFailure: try: if task: task_result = TaskResult(host, task, dict(unreachable=True)) self._rslt_q.put(task_result, block=False) except: break except Exception as e: if isinstance( e, (IOError, EOFError, KeyboardInterrupt)) and not isinstance( e, TemplateNotFound): break else: try: if task: task_result = TaskResult( host, task, dict(failed=True, exception=traceback.format_exc(), stdout='')) self._rslt_q.put(task_result, block=False) except: debug("WORKER EXCEPTION: %s" % e) debug("WORKER EXCEPTION: %s" % traceback.format_exc()) break debug("WORKER PROCESS EXITING")
def run(self): ''' Called when the process is started, and loops indefinitely until an error is encountered (typically an IOerror from the queue pipe being disconnected). During the loop, we attempt to pull tasks off the job queue and run them, pushing the result onto the results queue. We also remove the host from the blocked hosts list, to signify that they are ready for their next task. ''' if HAS_ATFORK: atfork() while True: task = None try: if not self._main_q.empty(): debug("there's work to be done!") (host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get(block=False) debug("got a task/handler to work on: %s" % task) # because the task queue manager starts workers (forks) before the # playbook is loaded, set the basedir of the loader inherted by # this fork now so that we can find files correctly self._loader.set_basedir(basedir) # Serializing/deserializing tasks does not preserve the loader attribute, # since it is passed to the worker during the forking of the process and # would be wasteful to serialize. So we set it here on the task now, and # the task handles updating parent/child objects as needed. task.set_loader(self._loader) # apply the given task's information to the connection info, # which may override some fields already set by the play or # the options specified on the command line new_play_context = play_context.set_task_and_variable_override(task=task, variables=job_vars) # execute the task and build a TaskResult from the result debug("running TaskExecutor() for %s/%s" % (host, task)) executor_result = TaskExecutor(host, task, job_vars, new_play_context, self._new_stdin, self._loader, shared_loader_obj).run() debug("done running TaskExecutor() for %s/%s" % (host, task)) task_result = TaskResult(host, task, executor_result) # put the result on the result queue debug("sending task result") self._rslt_q.put(task_result, block=False) debug("done sending task result") else: time.sleep(0.1) except queue.Empty: pass except (IOError, EOFError, KeyboardInterrupt): break except AnsibleConnectionFailure: try: if task: task_result = TaskResult(host, task, dict(unreachable=True)) self._rslt_q.put(task_result, block=False) except: # FIXME: most likely an abort, catch those kinds of errors specifically break except Exception, e: debug("WORKER EXCEPTION: %s" % e) debug("WORKER EXCEPTION: %s" % traceback.format_exc()) try: if task: task_result = TaskResult(host, task, dict(failed=True, exception=traceback.format_exc(), stdout='')) self._rslt_q.put(task_result, block=False) except: # FIXME: most likely an abort, catch those kinds of errors specifically break
def test_task_executor_squash_items(self): items = ['a', 'b', 'c'] fake_loader = DictDataLoader({}) mock_host = MagicMock() loop_var = 'item' def _evaluate_conditional(templar, variables): item = variables.get(loop_var) if item == 'b': return False return True mock_task = MagicMock() mock_task.evaluate_conditional.side_effect = _evaluate_conditional mock_play_context = MagicMock() mock_shared_loader = None mock_queue = MagicMock() new_stdin = None job_vars = dict(pkg_mgr='yum') te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=mock_shared_loader, rslt_q=mock_queue, ) # No replacement mock_task.action = 'yum' new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) self.assertIsInstance(mock_task.args, MagicMock) mock_task.action = 'foo' mock_task.args = {'name': '{{item}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) self.assertEqual(mock_task.args, {'name': '{{item}}'}) mock_task.action = 'yum' mock_task.args = {'name': 'static'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) self.assertEqual(mock_task.args, {'name': 'static'}) mock_task.action = 'yum' mock_task.args = {'name': '{{pkg_mgr}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) self.assertEqual(mock_task.args, {'name': '{{pkg_mgr}}'}) mock_task.action = '{{unknown}}' mock_task.args = {'name': '{{item}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, ['a', 'b', 'c']) self.assertEqual(mock_task.args, {'name': '{{item}}'}) # Could do something like this to recover from bad deps in a package job_vars = dict(pkg_mgr='yum', packages=['a', 'b']) items = ['absent', 'latest'] mock_task.action = 'yum' mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, { 'name': '{{ packages }}', 'state': '{{ item }}' }) # Maybe should raise an error in this case. The user would have to specify: # - yum: name="{{ packages[item] }}" # with_items: # - ['a', 'b'] # - ['foo', 'bar'] # you can't use a list as a dict key so that would probably throw # an error later. If so, we can throw it now instead. # Squashing in this case would not be intuitive as the user is being # explicit in using each list entry as a key. job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "foo": "baz", "bar": "quux" }) items = [['a', 'b'], ['foo', 'bar']] mock_task.action = 'yum' mock_task.args = {'name': '{{ packages[item] }}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'}) # Replaces items = ['a', 'b', 'c'] mock_task.action = 'yum' mock_task.args = {'name': '{{item}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, [['a', 'c']]) self.assertEqual(mock_task.args, {'name': ['a', 'c']}) mock_task.action = '{{pkg_mgr}}' mock_task.args = {'name': '{{item}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) self.assertEqual(new_items, [['a', 'c']]) self.assertEqual(mock_task.args, {'name': ['a', 'c']}) # New loop_var mock_task.action = 'yum' mock_task.args = {'name': '{{a_loop_var_item}}'} mock_task.loop_control = {'loop_var': 'a_loop_var_item'} loop_var = 'a_loop_var_item' new_items = te._squash_items(items=items, loop_var='a_loop_var_item', variables=job_vars) self.assertEqual(new_items, [['a', 'c']]) self.assertEqual(mock_task.args, {'name': ['a', 'c']}) loop_var = 'item' # # These are presently not optimized but could be in the future. # Expected output if they were optimized is given as a comment # Please move these to a different section if they are optimized # # Squashing lists job_vars = dict(pkg_mgr='yum') items = [['a', 'b'], ['foo', 'bar']] mock_task.action = 'yum' mock_task.args = {'name': '{{ item }}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) # self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']]) # self.assertEqual(mock_task.args, {'name': ['a', 'b', 'foo', 'bar']}) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, {'name': '{{ item }}'}) # Retrieving from a dict items = ['a', 'b', 'foo'] mock_task.action = 'yum' mock_task.args = {'name': '{{ packages[item] }}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) # self.assertEqual(new_items, [['foo', 'baz']]) # self.assertEqual(mock_task.args, {'name': ['foo', 'baz']}) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'}) # Another way to retrieve from a dict job_vars = dict(pkg_mgr='yum') items = [{'package': 'foo'}, {'package': 'bar'}] mock_task.action = 'yum' mock_task.args = {'name': '{{ item["package"] }}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) # self.assertEqual(new_items, [['foo', 'bar']]) # self.assertEqual(mock_task.args, {'name': ['foo', 'bar']}) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, {'name': '{{ item["package"] }}'}) items = [ dict(name='a', state='present'), dict(name='b', state='present'), dict(name='c', state='present'), ] mock_task.action = 'yum' mock_task.args = {'name': '{{item.name}}', 'state': '{{item.state}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) # self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')]) # self.assertEqual(mock_task.args, {'name': ['a', 'b', 'c'], 'state': 'present'}) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, { 'name': '{{item.name}}', 'state': '{{item.state}}' }) items = [ dict(name='a', state='present'), dict(name='b', state='present'), dict(name='c', state='absent'), ] mock_task.action = 'yum' mock_task.args = {'name': '{{item.name}}', 'state': '{{item.state}}'} new_items = te._squash_items(items=items, loop_var='item', variables=job_vars) # self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'), # dict(name='c', state='absent')]) # self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'}) self.assertEqual(new_items, items) self.assertEqual(mock_task.args, { 'name': '{{item.name}}', 'state': '{{item.state}}' })
def test_task_executor_execute(self): fake_loader = DictDataLoader({}) mock_host = MagicMock() mock_task = MagicMock() mock_task.args = dict() mock_task.retries = 0 mock_task.delay = -1 mock_task.register = 'foo' mock_task.until = None mock_task.changed_when = None mock_task.failed_when = None mock_task.post_validate.return_value = None # mock_task.async cannot be left unset, because on Python 3 MagicMock() # > 0 raises a TypeError There are two reasons for using the value 1 # here: on Python 2 comparing MagicMock() > 0 returns True, and the # other reason is that if I specify 0 here, the test fails. ;) mock_task. async = 1 mock_play_context = MagicMock() mock_play_context.post_validate.return_value = None mock_play_context.update_vars.return_value = None mock_connection = MagicMock() mock_connection.set_host_overrides.return_value = None mock_connection._connect.return_value = None mock_action = MagicMock() shared_loader = None new_stdin = None job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX") te = TaskExecutor( host=mock_host, task=mock_task, job_vars=job_vars, play_context=mock_play_context, new_stdin=new_stdin, loader=fake_loader, shared_loader_obj=shared_loader, ) te._get_connection = MagicMock(return_value=mock_connection) te._get_action_handler = MagicMock(return_value=mock_action) mock_action.run.return_value = dict(ansible_facts=dict()) res = te._execute() mock_task.changed_when = "1 == 1" res = te._execute() mock_task.changed_when = None mock_task.failed_when = "1 == 1" res = te._execute() mock_task.failed_when = None mock_task.evaluate_conditional.return_value = False res = te._execute() mock_task.evaluate_conditional.return_value = True mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar') mock_task.action = 'include' res = te._execute()
def __execute_module(self, module_name=None, module_args=None, persist_files=False, delete_remote_tmp=None, wrap_async=False, run_on_ca_host=False, ignore_changed=False, ignore_failed=False): if run_on_ca_host: host = InventoryManager(DataLoader(), context.CLIARGS['inventory']).get_host( self._var("ca_host")) task = self._task.copy() task_data = self._var("ca_host_options") task_data.update({ 'name': "mgssl ca_host task: {0}".format(module_name), 'action': module_name, 'args': module_args, 'delegate_to': self._var("ca_host") }) task.load_data(task_data) task_vars = self._task_vars.copy() task_vars.update(self._task.get_variable_manager().get_vars( host=host, task=task)) executor_result = TaskExecutor(host, task, task_vars, self._play_context, None, self._loader, self._shared_loader_obj, None) # Dirty fix for mitogen compatibility # Mitogen somehow puts a task global connection binding object in each connection that gets created # during the lifetime of a task. That usually happens on the beginning of a task, but here, we create # a new task executor within a task and that also creates a new connection for local running tasks. # After execution the connections get closed, but the close function also closes and removes the parent # tasks binding object. Now all future connections will fail. # # Solution: Overwrite the close method and only call the necessary close methods except the one that closes # the binding object if self._is_mitogen: get_connection_method = executor_result._get_connection def get_connection(cvars, templar): c = get_connection_method(cvars, templar) c.close = lambda: (c._put_connection(), None) return c executor_result._get_connection = get_connection ret = executor_result.run() # Reset the close method if self._is_mitogen: executor_result._get_connection = get_connection_method else: if self._shared_loader_obj.action_loader.has_plugin( module_name, None): task = self._task.copy() task.load_data({ 'action': module_name, 'args': module_args, }) handler = self._shared_loader_obj.action_loader.get( module_name, task=task, connection=self._connection, play_context=self._play_context. set_task_and_variable_override(task, {}, self._templar), loader=self._loader, templar=self._templar, shared_loader_obj=self._shared_loader_obj, collection_list=None) ret = handler.run(None, self._task_vars) else: ret = self._execute_module(module_name, module_args, None, self._task_vars, persist_files, delete_remote_tmp, wrap_async) self._check_result(module_name, run_on_ca_host, ret, ignore_changed, ignore_failed) return ret
def _run(self): ''' Called when the process is started. Pushes the result onto the results queue. We also remove the host from the blocked hosts list, to signify that they are ready for their next task. ''' # import cProfile, pstats, StringIO # pr = cProfile.Profile() # pr.enable() # Set the queue on Display so calls to Display.display are proxied over the queue display.set_queue(self._final_q) try: # execute the task and build a TaskResult from the result display.debug("running TaskExecutor() for %s/%s" % (self._host, self._task)) executor_result = TaskExecutor(self._host, self._task, self._task_vars, self._play_context, self._new_stdin, self._loader, self._shared_loader_obj, self._final_q).run() display.debug("done running TaskExecutor() for %s/%s [%s]" % (self._host, self._task, self._task._uuid)) self._host.vars = dict() self._host.groups = [] # put the result on the result queue display.debug("sending task result for task %s" % self._task._uuid) self._final_q.send_task_result( self._host.name, self._task._uuid, executor_result, task_fields=self._task.dump_attrs(), ) display.debug("done sending task result for task %s" % self._task._uuid) except AnsibleConnectionFailure: self._host.vars = dict() self._host.groups = [] self._final_q.send_task_result( self._host.name, self._task._uuid, dict(unreachable=True), task_fields=self._task.dump_attrs(), ) except Exception as e: if not isinstance(e, (IOError, EOFError, KeyboardInterrupt, SystemExit)) or isinstance(e, TemplateNotFound): try: self._host.vars = dict() self._host.groups = [] self._final_q.send_task_result( self._host.name, self._task._uuid, dict(failed=True, exception=to_text(traceback.format_exc()), stdout=''), task_fields=self._task.dump_attrs(), ) except Exception: display.debug(u"WORKER EXCEPTION: %s" % to_text(e)) display.debug(u"WORKER TRACEBACK: %s" % to_text(traceback.format_exc())) finally: self._clean_up() display.debug("WORKER PROCESS EXITING")
self.send(result) except: return False return True def run(self, (host, task, basedir, job_vars, play_context, shared_loader_obj)): debug("there's work to be done!") debug("got a task/handler to work on: %s" % task) self._loader.set_basedir(basedir) task.set_loader(self._loader) debug("running TaskExecutor() for %s/%s" % (host, task)) executor = TaskExecutor(host, task, job_vars, play_context, self._new_stdin, self._loader, shared_loader_obj) self._TaskExecutorPool.apply_async( executor.run, callback=(lambda result, host=host, task=task: self.send_result( result, host, task))) return True def send_result(self, result, host, task): debug("done running TaskExecutor() for %s/%s" % (host, task)) res = TaskResult(host, task, result) self.send(res) class Result(IOGenerator): def exception(self, record, (type, value, trace)):