def test_task_executor_get_loop_items(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.loop = 'items'
mock_task.loop_args = ['a', 'b', 'c']
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
mock_shared_loader.lookup_loader = lookup_loader
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
)
items = te._get_loop_items()
self.assertEqual(items, ['a', 'b', 'c'])
def test_task_executor_get_action_handler(self):
te = TaskExecutor(
host=MagicMock(),
task=MagicMock(),
job_vars={},
play_context=MagicMock(),
new_stdin=None,
loader=DictDataLoader({}),
shared_loader_obj=MagicMock(),
final_q=MagicMock(),
)
action_loader = te._shared_loader_obj.action_loader
action_loader.has_plugin.return_value = True
action_loader.get.return_value = mock.sentinel.handler
mock_connection = MagicMock()
mock_templar = MagicMock()
action = 'namespace.prefix_sufix'
te._task.action = action
handler = te._get_action_handler(mock_connection, mock_templar)
self.assertIs(mock.sentinel.handler, handler)
action_loader.has_plugin.assert_called_once_with(
action, collection_list=te._task.collections)
action_loader.get.assert_called_once_with(
te._task.action, task=te._task, connection=mock_connection,
play_context=te._play_context, loader=te._loader,
templar=mock_templar, shared_loader_obj=te._shared_loader_obj,
collection_list=te._task.collections)
def test_task_executor_get_loop_items(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.loop = "items"
mock_task.loop_args = ["a", "b", "c"]
mock_conn_info = MagicMock()
mock_shared_loader = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
connection_info=mock_conn_info,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
items = te._get_loop_items()
self.assertEqual(items, ["a", "b", "c"])
def test_task_executor_get_loop_items(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.loop = 'items'
mock_task.loop_args = ['a', 'b', 'c']
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
mock_shared_loader.lookup_loader = lookup_loader
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
items = te._get_loop_items()
self.assertEqual(items, ['a', 'b', 'c'])
def test_task_executor_get_handler_normal(self):
te = TaskExecutor(
host=MagicMock(),
task=MagicMock(),
job_vars={},
play_context=MagicMock(),
new_stdin=None,
loader=DictDataLoader({}),
shared_loader_obj=MagicMock(),
final_q=MagicMock(),
)
action_loader = te._shared_loader_obj.action_loader
action_loader.has_plugin.return_value = False
action_loader.get.return_value = mock.sentinel.handler
action_loader.__contains__.return_value = False
mock_connection = MagicMock()
mock_templar = MagicMock()
action = 'namespace.prefix_sufix'
module_prefix = action.split('.')[-1].split('_')[0]
te._task.action = action
handler = te._get_action_handler(mock_connection, mock_templar)
self.assertIs(mock.sentinel.handler, handler)
action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections),
mock.call(module_prefix, collection_list=te._task.collections)])
action_loader.get.assert_called_once_with(
'normal', task=te._task, connection=mock_connection,
play_context=te._play_context, loader=te._loader,
templar=mock_templar, shared_loader_obj=te._shared_loader_obj,
collection_list=None)
def test_task_executor_poll_async_result(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.async = 0.1
mock_task.poll = 0.05
mock_play_context = MagicMock()
mock_connection = MagicMock()
mock_action = MagicMock()
mock_queue = MagicMock()
shared_loader = MagicMock()
shared_loader.action_loader = action_loader
new_stdin = None
job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = shared_loader,
rslt_q = mock_queue,
)
te._connection = MagicMock()
def _get(*args, **kwargs):
mock_action = MagicMock()
mock_action.run.return_value = dict(stdout='')
return mock_action
# testing with some bad values in the result passed to poll async,
# and with a bad value returned from the mock action
with patch.object(action_loader, 'get', _get):
mock_templar = MagicMock()
res = te._poll_async_result(result=dict(), templar=mock_templar)
self.assertIn('failed', res)
res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar)
self.assertIn('failed', res)
def _get(*args, **kwargs):
mock_action = MagicMock()
mock_action.run.return_value = dict(finished=1)
return mock_action
# now testing with good values
with patch.object(action_loader, 'get', _get):
mock_templar = MagicMock()
res = te._poll_async_result(result=dict(ansible_job_id=1), templar=mock_templar)
self.assertEqual(res, dict(finished=1))
def test_task_executor_execute(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.args = dict()
mock_task.retries = 0
mock_task.delay = -1
mock_task.register = 'foo'
mock_task.until = None
mock_task.changed_when = None
mock_task.failed_when = None
mock_task.post_validate.return_value = None
mock_conn_info = MagicMock()
mock_conn_info.post_validate.return_value = None
mock_conn_info.update_vars.return_value = None
mock_connection = MagicMock()
mock_connection.set_host_overrides.return_value = None
mock_connection._connect.return_value = None
mock_action = MagicMock()
shared_loader = None
new_stdin = None
job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
connection_info = mock_conn_info,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = shared_loader,
)
te._get_connection = MagicMock(return_value=mock_connection)
te._get_action_handler = MagicMock(return_value=mock_action)
mock_action.run.return_value = dict(ansible_facts=dict())
res = te._execute()
mock_task.changed_when = "1 == 1"
res = te._execute()
mock_task.changed_when = None
mock_task.failed_when = "1 == 1"
res = te._execute()
mock_task.failed_when = None
mock_task.evaluate_conditional.return_value = False
res = te._execute()
mock_task.evaluate_conditional.return_value = True
mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar')
mock_task.action = 'include'
res = te._execute()
def test_task_executor_execute(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.args = dict()
mock_task.retries = 0
mock_task.delay = -1
mock_task.register = "foo"
mock_task.until = None
mock_task.changed_when = None
mock_task.failed_when = None
mock_task.post_validate.return_value = None
mock_conn_info = MagicMock()
mock_conn_info.post_validate.return_value = None
mock_conn_info.update_vars.return_value = None
mock_connection = MagicMock()
mock_connection.set_host_overrides.return_value = None
mock_connection._connect.return_value = None
mock_action = MagicMock()
shared_loader = None
new_stdin = None
job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
connection_info=mock_conn_info,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=shared_loader,
)
te._get_connection = MagicMock(return_value=mock_connection)
te._get_action_handler = MagicMock(return_value=mock_action)
mock_action.run.return_value = dict(ansible_facts=dict())
res = te._execute()
mock_task.changed_when = "1 == 1"
res = te._execute()
mock_task.changed_when = None
mock_task.failed_when = "1 == 1"
res = te._execute()
mock_task.failed_when = None
mock_task.evaluate_conditional.return_value = False
res = te._execute()
mock_task.evaluate_conditional.return_value = True
mock_task.args = dict(_raw_params="foo.yml", a="foo", b="bar")
mock_task.action = "include"
res = te._execute()
def run(self):
'''
Called when the process is started, and loops indefinitely
until an error is encountered (typically an IOerror from the
queue pipe being disconnected). During the loop, we attempt
to pull tasks off the job queue and run them, pushing the result
onto the results queue. We also remove the host from the blocked
hosts list, to signify that they are ready for their next task.
'''
if HAS_ATFORK:
atfork()
try:
# execute the task and build a TaskResult from the result
display.debug("running TaskExecutor() for %s/%s" %
(self._host, self._task))
executor_result = TaskExecutor(self._host, self._task,
self._task_vars, self._play_context,
self._new_stdin, self._loader,
self._shared_loader_obj,
self._rslt_q).run()
display.debug("done running TaskExecutor() for %s/%s" %
(self._host, self._task))
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(self._host, self._task, executor_result)
# put the result on the result queue
display.debug("sending task result")
self._rslt_q.put(task_result)
display.debug("done sending task result")
except AnsibleConnectionFailure:
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(self._host, self._task,
dict(unreachable=True))
self._rslt_q.put(task_result, block=False)
except Exception as e:
if not isinstance(e,
(IOError, EOFError, KeyboardInterrupt,
SystemExit)) or isinstance(e, TemplateNotFound):
try:
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(
self._host, self._task,
dict(failed=True,
exception=to_unicode(traceback.format_exc()),
stdout=''))
self._rslt_q.put(task_result, block=False)
except:
display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e))
display.debug(u"WORKER TRACEBACK: %s" %
to_unicode(traceback.format_exc()))
display.debug("WORKER PROCESS EXITING")
def test_task_executor_squash_items(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _evaluate_conditional(templar, variables):
item = variables.get('item')
if item == 'b':
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_play_context = MagicMock()
mock_shared_loader = None
new_stdin = None
job_vars = dict(pkg_mgr='yum')
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
mock_task.action = 'foo'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
mock_task.action = '{{pkg_mgr}}'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
# Smoketests -- these won't optimize but make sure that they don't
# traceback either
mock_task.action = '{{unknown}}'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
items = [
dict(name='a', state='present'),
dict(name='b', state='present'),
dict(name='c', state='present')
]
mock_task.action = 'yum'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, items)
def run(self):
'''
Called when the process is started. Pushes the result onto the
results queue. We also remove the host from the blocked hosts list, to
signify that they are ready for their next task.
'''
#import cProfile, pstats, StringIO
#pr = cProfile.Profile()
#pr.enable()
if HAS_ATFORK:
atfork()
try:
# execute the task and build a TaskResult from the result
display.debug("running TaskExecutor() for %s/%s" % (self._host, self._task))
executor_result = TaskExecutor(
self._host,
self._task,
self._task_vars,
self._play_context,
self._new_stdin,
self._loader,
self._shared_loader_obj,
self._rslt_q
).run()
display.debug("done running TaskExecutor() for %s/%s" % (self._host, self._task))
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(self._host.name, self._task._uuid, executor_result)
# put the result on the result queue
display.debug("sending task result")
self._rslt_q.put(task_result)
display.debug("done sending task result")
except AnsibleConnectionFailure:
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(self._host.name, self._task._uuid, dict(unreachable=True))
self._rslt_q.put(task_result, block=False)
except Exception as e:
if not isinstance(e, (IOError, EOFError, KeyboardInterrupt, SystemExit)) or isinstance(e, TemplateNotFound):
try:
self._host.vars = dict()
self._host.groups = []
task_result = TaskResult(self._host.name, self._task._uuid, dict(failed=True, exception=to_unicode(traceback.format_exc()), stdout=''))
self._rslt_q.put(task_result, block=False)
except:
display.debug(u"WORKER EXCEPTION: %s" % to_unicode(e))
display.debug(u"WORKER TRACEBACK: %s" % to_unicode(traceback.format_exc()))
display.debug("WORKER PROCESS EXITING")
def test_task_executor_run_clean_res(self):
te = TaskExecutor(None, MagicMock(), None, None, None, None, None,
None)
te._get_loop_items = MagicMock(return_value=[1])
te._run_loop = MagicMock(
return_value=[{
'unsafe_bytes': AnsibleUnsafeBytes(b'{{ $bar }}'),
'unsafe_text': AnsibleUnsafeText(u'{{ $bar }}'),
'bytes': b'bytes',
'text': u'text',
'int': 1,
}])
res = te.run()
data = res['results'][0]
self.assertIsInstance(data['unsafe_bytes'], AnsibleUnsafeText)
self.assertIsInstance(data['unsafe_text'], AnsibleUnsafeText)
self.assertIsInstance(data['bytes'], text_type)
self.assertIsInstance(data['text'], text_type)
self.assertIsInstance(data['int'], int)
def test_task_executor_get_handler_prefix(self):
te = TaskExecutor(
host=MagicMock(),
task=MagicMock(),
job_vars={},
play_context=MagicMock(),
new_stdin=None,
loader=DictDataLoader({}),
shared_loader_obj=MagicMock(),
final_q=MagicMock(),
)
context = MagicMock(resolved=False)
te._shared_loader_obj.module_loader.find_plugin_with_context.return_value = context
action_loader = te._shared_loader_obj.action_loader
action_loader.has_plugin.side_effect = [False, True]
action_loader.get.return_value = mock.sentinel.handler
action_loader.__contains__.return_value = True
mock_connection = MagicMock()
mock_templar = MagicMock()
action = 'namespace.netconf_suffix'
module_prefix = action.split('_', 1)[0]
te._task.action = action
handler = te._get_action_handler(mock_connection, mock_templar)
self.assertIs(mock.sentinel.handler, handler)
action_loader.has_plugin.assert_has_calls([
mock.call(action,
collection_list=te._task.collections), # called twice
mock.call(module_prefix, collection_list=te._task.collections)
])
action_loader.get.assert_called_once_with(
module_prefix,
task=te._task,
connection=mock_connection,
play_context=te._play_context,
loader=te._loader,
templar=mock_templar,
shared_loader_obj=te._shared_loader_obj,
collection_list=te._task.collections)
def test_task_executor_run_loop(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _copy(exclude_parent=False, exclude_tasks=False):
new_item = MagicMock()
return new_item
mock_task = MagicMock()
mock_task.copy.side_effect = _copy
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
mock_queue = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
rslt_q=mock_queue,
)
def _execute(variables):
return dict(item=variables.get('item'))
te._squash_items = MagicMock(return_value=items)
te._execute = MagicMock(side_effect=_execute)
res = te._run_loop(items)
self.assertEqual(len(res), 3)
def test_task_executor_run_loop(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _copy():
new_item = MagicMock()
return new_item
mock_task = MagicMock()
mock_task.copy.side_effect = _copy
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
mock_queue = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
rslt_q = mock_queue,
)
def _execute(variables):
return dict(item=variables.get('item'))
te._squash_items = MagicMock(return_value=items)
te._execute = MagicMock(side_effect=_execute)
res = te._run_loop(items)
self.assertEqual(len(res), 3)
def test_task_executor_squash_items(self):
items = ["a", "b", "c"]
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _evaluate_conditional(templar, variables):
item = variables.get("item")
if item == "b":
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_conn_info = MagicMock()
mock_shared_loader = None
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
connection_info=mock_conn_info,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
mock_task.action = "foo"
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ["a", "b", "c"])
mock_task.action = "yum"
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ["a,c"])
def test_task_executor_squash_items(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _evaluate_conditional(templar, variables):
item = variables.get('item')
if item == 'b':
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_play_context = MagicMock()
mock_shared_loader = None
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
)
mock_task.action = 'foo'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ['a,c'])
def test_task_executor_squash_items(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _evaluate_conditional(templar, variables):
item = variables.get('item')
if item == 'b':
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_conn_info = MagicMock()
mock_shared_loader = None
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
connection_info = mock_conn_info,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
)
mock_task.action = 'foo'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
new_items = te._squash_items(items=items, variables=job_vars)
self.assertEqual(new_items, ['a,c'])
def test_task_executor_run_loop(self):
items = ["a", "b", "c"]
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _copy():
new_item = MagicMock()
return new_item
mock_task = MagicMock()
mock_task.copy.side_effect = _copy
mock_conn_info = MagicMock()
mock_shared_loader = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
connection_info=mock_conn_info,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
def _execute(variables):
return dict(item=variables.get("item"))
te._squash_items = MagicMock(return_value=items)
te._execute = MagicMock(side_effect=_execute)
res = te._run_loop(items)
self.assertEqual(len(res), 3)
def test_task_executor_poll_async_result(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task. async = 3
mock_task.poll = 1
mock_play_context = MagicMock()
mock_connection = MagicMock()
mock_action = MagicMock()
shared_loader = MagicMock()
shared_loader.action_loader = action_loader
new_stdin = None
job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=shared_loader,
)
te._connection = MagicMock()
def _get(*args, **kwargs):
mock_action = MagicMock()
mock_action.run.return_value = dict(stdout='')
return mock_action
# testing with some bad values in the result passed to poll async,
# and with a bad value returned from the mock action
with patch.object(action_loader, 'get', _get):
mock_templar = MagicMock()
res = te._poll_async_result(result=dict(), templar=mock_templar)
self.assertIn('failed', res)
res = te._poll_async_result(result=dict(ansible_job_id=1),
templar=mock_templar)
self.assertIn('failed', res)
def _get(*args, **kwargs):
mock_action = MagicMock()
mock_action.run.return_value = dict(finished=1)
return mock_action
# now testing with good values
with patch.object(action_loader, 'get', _get):
mock_templar = MagicMock()
res = te._poll_async_result(result=dict(ansible_job_id=1),
templar=mock_templar)
self.assertEqual(res, dict(finished=1))
def test_task_executor_init(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
def worker(main_q, res_q, loader):
while True:
task = None
try:
if not main_q.empty():
(host, task, task_vars, conn_info) = main_q.get(block=False)
executor_result = TaskExecutor(host, task, task_vars, conn_info, loader).run()
debug("executor result: %s" % executor_result)
task_result = TaskResult(host, task, executor_result)
res_q.put(task_result)
else:
time.sleep(0.01)
except Queue.Empty:
pass
except (IOError, EOFError, KeyboardInterrupt) as e:
debug("got a breaking error: %s" % e)
break
except Exception as e:
debug("EXCEPTION DURING WORKER PROCESSING: %s" % e)
traceback.print_exc()
break
def test_task_executor_squash_items(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
def _evaluate_conditional(templar, variables):
item = variables.get('item')
if item == 'b':
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_play_context = MagicMock()
mock_shared_loader = None
mock_queue = MagicMock()
new_stdin = None
job_vars = dict(pkg_mgr='yum')
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
rslt_q = mock_queue,
)
#
# No replacement
#
mock_task.action = 'yum'
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'foo'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
mock_task.args={'name': 'static'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
mock_task.args={'name': '{{pkg_mgr}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
#
# Replaces
#
mock_task.action = 'yum'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, [['a','c']])
mock_task.action = '{{pkg_mgr}}'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
#
# Smoketests -- these won't optimize but make sure that they don't
# traceback either
#
mock_task.action = '{{unknown}}'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
items = [dict(name='a', state='present'),
dict(name='b', state='present'),
dict(name='c', state='present')]
mock_task.action = 'yum'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, items)
def test_task_executor_execute(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.args = dict()
mock_task.retries = 0
mock_task.delay = -1
mock_task.register = 'foo'
mock_task.until = None
mock_task.changed_when = None
mock_task.failed_when = None
mock_task.post_validate.return_value = None
# mock_task.async cannot be left unset, because on Python 3 MagicMock()
# > 0 raises a TypeError There are two reasons for using the value 1
# here: on Python 2 comparing MagicMock() > 0 returns True, and the
# other reason is that if I specify 0 here, the test fails. ;)
mock_task. async = 1
mock_play_context = MagicMock()
mock_play_context.post_validate.return_value = None
mock_play_context.update_vars.return_value = None
mock_connection = MagicMock()
mock_connection.set_host_overrides.return_value = None
mock_connection._connect.return_value = None
mock_action = MagicMock()
shared_loader = None
new_stdin = None
job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=shared_loader,
)
te._get_connection = MagicMock(return_value=mock_connection)
te._get_action_handler = MagicMock(return_value=mock_action)
mock_action.run.return_value = dict(ansible_facts=dict())
res = te._execute()
mock_task.changed_when = "1 == 1"
res = te._execute()
mock_task.changed_when = None
mock_task.failed_when = "1 == 1"
res = te._execute()
mock_task.failed_when = None
mock_task.evaluate_conditional.return_value = False
res = te._execute()
mock_task.evaluate_conditional.return_value = True
mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar')
mock_task.action = 'include'
res = te._execute()
def test_task_executor_run(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task._role._role_path = "/path/to/role/foo"
mock_conn_info = MagicMock()
mock_shared_loader = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
connection_info=mock_conn_info,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
te._get_loop_items = MagicMock(return_value=None)
te._execute = MagicMock(return_value=dict())
res = te.run()
te._get_loop_items = MagicMock(return_value=[])
res = te.run()
te._get_loop_items = MagicMock(return_value=["a", "b", "c"])
te._run_loop = MagicMock(
return_value=[dict(item="a", changed=True), dict(item="b", failed=True), dict(item="c")]
)
res = te.run()
te._get_loop_items = MagicMock(side_effect=AnsibleError(""))
res = te.run()
self.assertIn("failed", res)
def run(self):
'''
Called when the process is started, and loops indefinitely
until an error is encountered (typically an IOerror from the
queue pipe being disconnected). During the loop, we attempt
to pull tasks off the job queue and run them, pushing the result
onto the results queue. We also remove the host from the blocked
hosts list, to signify that they are ready for their next task.
'''
if HAS_ATFORK:
atfork()
while True:
task = None
try:
(host, task, basedir, job_vars, play_context,
shared_loader_obj) = self._main_q.get()
debug("there's work to be done!")
debug("got a task/handler to work on: %s" % task)
# because the task queue manager starts workers (forks) before the
# playbook is loaded, set the basedir of the loader inherted by
# this fork now so that we can find files correctly
self._loader.set_basedir(basedir)
# Serializing/deserializing tasks does not preserve the loader attribute,
# since it is passed to the worker during the forking of the process and
# would be wasteful to serialize. So we set it here on the task now, and
# the task handles updating parent/child objects as needed.
task.set_loader(self._loader)
# execute the task and build a TaskResult from the result
debug("running TaskExecutor() for %s/%s" % (host, task))
executor_result = TaskExecutor(host, task, job_vars,
play_context, self._new_stdin,
self._loader,
shared_loader_obj).run()
debug("done running TaskExecutor() for %s/%s" % (host, task))
task_result = TaskResult(host, task, executor_result)
# put the result on the result queue
debug("sending task result")
self._rslt_q.put(task_result)
debug("done sending task result")
except queue.Empty:
pass
except AnsibleConnectionFailure:
try:
if task:
task_result = TaskResult(host, task,
dict(unreachable=True))
self._rslt_q.put(task_result, block=False)
except:
break
except Exception as e:
if isinstance(
e,
(IOError, EOFError, KeyboardInterrupt)) and not isinstance(
e, TemplateNotFound):
break
else:
try:
if task:
task_result = TaskResult(
host, task,
dict(failed=True,
exception=traceback.format_exc(),
stdout=''))
self._rslt_q.put(task_result, block=False)
except:
debug("WORKER EXCEPTION: %s" % e)
debug("WORKER EXCEPTION: %s" % traceback.format_exc())
break
debug("WORKER PROCESS EXITING")
def test_task_executor_squash_items(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
loop_var = 'item'
def _evaluate_conditional(templar, variables):
item = variables.get(loop_var)
if item == 'b':
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_play_context = MagicMock()
mock_shared_loader = None
mock_queue = MagicMock()
new_stdin = None
job_vars = dict(pkg_mgr='yum')
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
rslt_q = mock_queue,
)
#
# No replacement
#
mock_task.action = 'yum'
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'foo'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
mock_task.args={'name': 'static'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = 'yum'
mock_task.args={'name': '{{pkg_mgr}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
mock_task.action = '{{unknown}}'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
# Maybe should raise an error in this case. The user would have to specify:
# - yum: name="{{ packages[item] }}"
# with_items:
# - ['a', 'b']
# - ['foo', 'bar']
# you can't use a list as a dict key so that would probably throw
# an error later. If so, we can throw it now instead.
# Squashing in this case would not be intuitive as the user is being
# explicit in using each list entry as a key.
job_vars = dict(pkg_mgr='yum', packages={ "a": "foo", "b": "bar", "foo": "baz", "bar": "quux" })
items = [['a', 'b'], ['foo', 'bar']]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ packages[item] }}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, items)
#
# Replaces
#
items = ['a', 'b', 'c']
mock_task.action = 'yum'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, [['a','c']])
mock_task.action = '{{pkg_mgr}}'
mock_task.args={'name': '{{item}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
# New loop_var
mock_task.action = 'yum'
mock_task.args = {'name': '{{a_loop_var_item}}'}
mock_task.loop_control = {'loop_var': 'a_loop_var_item'}
loop_var = 'a_loop_var_item'
new_items = te._squash_items(items=items, loop_var='a_loop_var_item', variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
loop_var = 'item'
#
# These are presently not optimized but could be in the future.
# Expected output if they were optimized is given as a comment
# Please move these to a different section if they are optimized
#
# Squashing lists
job_vars = dict(pkg_mgr='yum')
items = [['a', 'b'], ['foo', 'bar']]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ item }}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
#self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']])
self.assertEqual(new_items, items)
# Retrieving from a dict
items = ['a', 'b', 'foo']
mock_task.action = 'yum'
mock_task.args = {'name': '{{ packages[item] }}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
#self.assertEqual(new_items, [['foo', 'baz']])
self.assertEqual(new_items, items)
# Another way to retrieve from a dict
job_vars = dict(pkg_mgr='yum')
items = [{'package': 'foo'}, {'package': 'bar'}]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ item["package"] }}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
#self.assertEqual(new_items, [['foo', 'bar']])
self.assertEqual(new_items, items)
items = [dict(name='a', state='present'),
dict(name='b', state='present'),
dict(name='c', state='present')]
mock_task.action = 'yum'
mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, items)
#self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')])
items = [dict(name='a', state='present'),
dict(name='b', state='present'),
dict(name='c', state='absent')]
mock_task.action = 'yum'
mock_task.args={'name': '{{item.name}}', 'state': '{{item.state}}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, items)
#self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'),
# dict(name='c', state='absent')])
# Could do something like this to recover from bad deps in a package
job_vars = dict(pkg_mgr='yum', packages=['a', 'b'])
items = [ 'absent', 'latest' ]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'}
new_items = te._squash_items(items=items, loop_var='item', variables=job_vars)
self.assertEqual(new_items, items)
def run(self):
'''
Called when the process is started, and loops indefinitely
until an error is encountered (typically an IOerror from the
queue pipe being disconnected). During the loop, we attempt
to pull tasks off the job queue and run them, pushing the result
onto the results queue. We also remove the host from the blocked
hosts list, to signify that they are ready for their next task.
'''
if HAS_ATFORK:
atfork()
while True:
task = None
try:
if not self._main_q.empty():
debug("there's work to be done!")
(host, task, basedir, job_vars, play_context, shared_loader_obj) = self._main_q.get(block=False)
debug("got a task/handler to work on: %s" % task)
# because the task queue manager starts workers (forks) before the
# playbook is loaded, set the basedir of the loader inherted by
# this fork now so that we can find files correctly
self._loader.set_basedir(basedir)
# Serializing/deserializing tasks does not preserve the loader attribute,
# since it is passed to the worker during the forking of the process and
# would be wasteful to serialize. So we set it here on the task now, and
# the task handles updating parent/child objects as needed.
task.set_loader(self._loader)
# apply the given task's information to the connection info,
# which may override some fields already set by the play or
# the options specified on the command line
new_play_context = play_context.set_task_and_variable_override(task=task, variables=job_vars)
# execute the task and build a TaskResult from the result
debug("running TaskExecutor() for %s/%s" % (host, task))
executor_result = TaskExecutor(host, task, job_vars, new_play_context, self._new_stdin, self._loader, shared_loader_obj).run()
debug("done running TaskExecutor() for %s/%s" % (host, task))
task_result = TaskResult(host, task, executor_result)
# put the result on the result queue
debug("sending task result")
self._rslt_q.put(task_result, block=False)
debug("done sending task result")
else:
time.sleep(0.1)
except queue.Empty:
pass
except (IOError, EOFError, KeyboardInterrupt):
break
except AnsibleConnectionFailure:
try:
if task:
task_result = TaskResult(host, task, dict(unreachable=True))
self._rslt_q.put(task_result, block=False)
except:
# FIXME: most likely an abort, catch those kinds of errors specifically
break
except Exception, e:
debug("WORKER EXCEPTION: %s" % e)
debug("WORKER EXCEPTION: %s" % traceback.format_exc())
try:
if task:
task_result = TaskResult(host, task, dict(failed=True, exception=traceback.format_exc(), stdout=''))
self._rslt_q.put(task_result, block=False)
except:
# FIXME: most likely an abort, catch those kinds of errors specifically
break
def test_task_executor_squash_items(self):
items = ['a', 'b', 'c']
fake_loader = DictDataLoader({})
mock_host = MagicMock()
loop_var = 'item'
def _evaluate_conditional(templar, variables):
item = variables.get(loop_var)
if item == 'b':
return False
return True
mock_task = MagicMock()
mock_task.evaluate_conditional.side_effect = _evaluate_conditional
mock_play_context = MagicMock()
mock_shared_loader = None
mock_queue = MagicMock()
new_stdin = None
job_vars = dict(pkg_mgr='yum')
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
rslt_q=mock_queue,
)
# No replacement
mock_task.action = 'yum'
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
self.assertIsInstance(mock_task.args, MagicMock)
mock_task.action = 'foo'
mock_task.args = {'name': '{{item}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
self.assertEqual(mock_task.args, {'name': '{{item}}'})
mock_task.action = 'yum'
mock_task.args = {'name': 'static'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
self.assertEqual(mock_task.args, {'name': 'static'})
mock_task.action = 'yum'
mock_task.args = {'name': '{{pkg_mgr}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
self.assertEqual(mock_task.args, {'name': '{{pkg_mgr}}'})
mock_task.action = '{{unknown}}'
mock_task.args = {'name': '{{item}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, ['a', 'b', 'c'])
self.assertEqual(mock_task.args, {'name': '{{item}}'})
# Could do something like this to recover from bad deps in a package
job_vars = dict(pkg_mgr='yum', packages=['a', 'b'])
items = ['absent', 'latest']
mock_task.action = 'yum'
mock_task.args = {'name': '{{ packages }}', 'state': '{{ item }}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {
'name': '{{ packages }}',
'state': '{{ item }}'
})
# Maybe should raise an error in this case. The user would have to specify:
# - yum: name="{{ packages[item] }}"
# with_items:
# - ['a', 'b']
# - ['foo', 'bar']
# you can't use a list as a dict key so that would probably throw
# an error later. If so, we can throw it now instead.
# Squashing in this case would not be intuitive as the user is being
# explicit in using each list entry as a key.
job_vars = dict(pkg_mgr='yum',
packages={
"a": "foo",
"b": "bar",
"foo": "baz",
"bar": "quux"
})
items = [['a', 'b'], ['foo', 'bar']]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ packages[item] }}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'})
# Replaces
items = ['a', 'b', 'c']
mock_task.action = 'yum'
mock_task.args = {'name': '{{item}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
self.assertEqual(mock_task.args, {'name': ['a', 'c']})
mock_task.action = '{{pkg_mgr}}'
mock_task.args = {'name': '{{item}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
self.assertEqual(mock_task.args, {'name': ['a', 'c']})
# New loop_var
mock_task.action = 'yum'
mock_task.args = {'name': '{{a_loop_var_item}}'}
mock_task.loop_control = {'loop_var': 'a_loop_var_item'}
loop_var = 'a_loop_var_item'
new_items = te._squash_items(items=items,
loop_var='a_loop_var_item',
variables=job_vars)
self.assertEqual(new_items, [['a', 'c']])
self.assertEqual(mock_task.args, {'name': ['a', 'c']})
loop_var = 'item'
#
# These are presently not optimized but could be in the future.
# Expected output if they were optimized is given as a comment
# Please move these to a different section if they are optimized
#
# Squashing lists
job_vars = dict(pkg_mgr='yum')
items = [['a', 'b'], ['foo', 'bar']]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ item }}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
# self.assertEqual(new_items, [['a', 'b', 'foo', 'bar']])
# self.assertEqual(mock_task.args, {'name': ['a', 'b', 'foo', 'bar']})
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {'name': '{{ item }}'})
# Retrieving from a dict
items = ['a', 'b', 'foo']
mock_task.action = 'yum'
mock_task.args = {'name': '{{ packages[item] }}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
# self.assertEqual(new_items, [['foo', 'baz']])
# self.assertEqual(mock_task.args, {'name': ['foo', 'baz']})
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {'name': '{{ packages[item] }}'})
# Another way to retrieve from a dict
job_vars = dict(pkg_mgr='yum')
items = [{'package': 'foo'}, {'package': 'bar'}]
mock_task.action = 'yum'
mock_task.args = {'name': '{{ item["package"] }}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
# self.assertEqual(new_items, [['foo', 'bar']])
# self.assertEqual(mock_task.args, {'name': ['foo', 'bar']})
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {'name': '{{ item["package"] }}'})
items = [
dict(name='a', state='present'),
dict(name='b', state='present'),
dict(name='c', state='present'),
]
mock_task.action = 'yum'
mock_task.args = {'name': '{{item.name}}', 'state': '{{item.state}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
# self.assertEqual(new_items, [dict(name=['a', 'b', 'c'], state='present')])
# self.assertEqual(mock_task.args, {'name': ['a', 'b', 'c'], 'state': 'present'})
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {
'name': '{{item.name}}',
'state': '{{item.state}}'
})
items = [
dict(name='a', state='present'),
dict(name='b', state='present'),
dict(name='c', state='absent'),
]
mock_task.action = 'yum'
mock_task.args = {'name': '{{item.name}}', 'state': '{{item.state}}'}
new_items = te._squash_items(items=items,
loop_var='item',
variables=job_vars)
# self.assertEqual(new_items, [dict(name=['a', 'b'], state='present'),
# dict(name='c', state='absent')])
# self.assertEqual(mock_task.args, {'name': '{{item.name}}', 'state': '{{item.state}}'})
self.assertEqual(new_items, items)
self.assertEqual(mock_task.args, {
'name': '{{item.name}}',
'state': '{{item.state}}'
})
def _run(self):
'''
Called when the process is started. Pushes the result onto the
results queue. We also remove the host from the blocked hosts list, to
signify that they are ready for their next task.
'''
# import cProfile, pstats, StringIO
# pr = cProfile.Profile()
# pr.enable()
# Set the queue on Display so calls to Display.display are proxied over the queue
display.set_queue(self._final_q)
try:
# execute the task and build a TaskResult from the result
display.debug("running TaskExecutor() for %s/%s" %
(self._host, self._task))
executor_result = TaskExecutor(self._host, self._task,
self._task_vars, self._play_context,
self._new_stdin, self._loader,
self._shared_loader_obj,
self._final_q).run()
display.debug("done running TaskExecutor() for %s/%s [%s]" %
(self._host, self._task, self._task._uuid))
self._host.vars = dict()
self._host.groups = []
# put the result on the result queue
display.debug("sending task result for task %s" % self._task._uuid)
self._final_q.send_task_result(
self._host.name,
self._task._uuid,
executor_result,
task_fields=self._task.dump_attrs(),
)
display.debug("done sending task result for task %s" %
self._task._uuid)
except AnsibleConnectionFailure:
self._host.vars = dict()
self._host.groups = []
self._final_q.send_task_result(
self._host.name,
self._task._uuid,
dict(unreachable=True),
task_fields=self._task.dump_attrs(),
)
except Exception as e:
if not isinstance(e,
(IOError, EOFError, KeyboardInterrupt,
SystemExit)) or isinstance(e, TemplateNotFound):
try:
self._host.vars = dict()
self._host.groups = []
self._final_q.send_task_result(
self._host.name,
self._task._uuid,
dict(failed=True,
exception=to_text(traceback.format_exc()),
stdout=''),
task_fields=self._task.dump_attrs(),
)
except Exception:
display.debug(u"WORKER EXCEPTION: %s" % to_text(e))
display.debug(u"WORKER TRACEBACK: %s" %
to_text(traceback.format_exc()))
finally:
self._clean_up()
display.debug("WORKER PROCESS EXITING")
def __execute_module(self,
module_name=None,
module_args=None,
persist_files=False,
delete_remote_tmp=None,
wrap_async=False,
run_on_ca_host=False,
ignore_changed=False,
ignore_failed=False):
if run_on_ca_host:
host = InventoryManager(DataLoader(),
context.CLIARGS['inventory']).get_host(
self._var("ca_host"))
task = self._task.copy()
task_data = self._var("ca_host_options")
task_data.update({
'name':
"mgssl ca_host task: {0}".format(module_name),
'action':
module_name,
'args':
module_args,
'delegate_to':
self._var("ca_host")
})
task.load_data(task_data)
task_vars = self._task_vars.copy()
task_vars.update(self._task.get_variable_manager().get_vars(
host=host, task=task))
executor_result = TaskExecutor(host, task, task_vars,
self._play_context, None,
self._loader,
self._shared_loader_obj, None)
# Dirty fix for mitogen compatibility
# Mitogen somehow puts a task global connection binding object in each connection that gets created
# during the lifetime of a task. That usually happens on the beginning of a task, but here, we create
# a new task executor within a task and that also creates a new connection for local running tasks.
# After execution the connections get closed, but the close function also closes and removes the parent
# tasks binding object. Now all future connections will fail.
#
# Solution: Overwrite the close method and only call the necessary close methods except the one that closes
# the binding object
if self._is_mitogen:
get_connection_method = executor_result._get_connection
def get_connection(cvars, templar):
c = get_connection_method(cvars, templar)
c.close = lambda: (c._put_connection(), None)
return c
executor_result._get_connection = get_connection
ret = executor_result.run()
# Reset the close method
if self._is_mitogen:
executor_result._get_connection = get_connection_method
else:
if self._shared_loader_obj.action_loader.has_plugin(
module_name, None):
task = self._task.copy()
task.load_data({
'action': module_name,
'args': module_args,
})
handler = self._shared_loader_obj.action_loader.get(
module_name,
task=task,
connection=self._connection,
play_context=self._play_context.
set_task_and_variable_override(task, {}, self._templar),
loader=self._loader,
templar=self._templar,
shared_loader_obj=self._shared_loader_obj,
collection_list=None)
ret = handler.run(None, self._task_vars)
else:
ret = self._execute_module(module_name, module_args, None,
self._task_vars, persist_files,
delete_remote_tmp, wrap_async)
self._check_result(module_name, run_on_ca_host, ret, ignore_changed,
ignore_failed)
return ret
def test_task_executor_execute(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task.args = dict()
mock_task.retries = 0
mock_task.delay = -1
mock_task.register = 'foo'
mock_task.until = None
mock_task.changed_when = None
mock_task.failed_when = None
mock_task.post_validate.return_value = None
# mock_task.async cannot be left unset, because on Python 3 MagicMock()
# > 0 raises a TypeError There are two reasons for using the value 1
# here: on Python 2 comparing MagicMock() > 0 returns True, and the
# other reason is that if I specify 0 here, the test fails. ;)
mock_task.async = 1
mock_play_context = MagicMock()
mock_play_context.post_validate.return_value = None
mock_play_context.update_vars.return_value = None
mock_connection = MagicMock()
mock_connection.set_host_overrides.return_value = None
mock_connection._connect.return_value = None
mock_action = MagicMock()
shared_loader = None
new_stdin = None
job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX")
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = shared_loader,
)
te._get_connection = MagicMock(return_value=mock_connection)
te._get_action_handler = MagicMock(return_value=mock_action)
mock_action.run.return_value = dict(ansible_facts=dict())
res = te._execute()
mock_task.changed_when = "1 == 1"
res = te._execute()
mock_task.changed_when = None
mock_task.failed_when = "1 == 1"
res = te._execute()
mock_task.failed_when = None
mock_task.evaluate_conditional.return_value = False
res = te._execute()
mock_task.evaluate_conditional.return_value = True
mock_task.args = dict(_raw_params='foo.yml', a='foo', b='bar')
mock_task.action = 'include'
res = te._execute()
def test_task_executor_run(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task._role._role_path = '/path/to/role/foo'
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host = mock_host,
task = mock_task,
job_vars = job_vars,
play_context = mock_play_context,
new_stdin = new_stdin,
loader = fake_loader,
shared_loader_obj = mock_shared_loader,
)
te._get_loop_items = MagicMock(return_value=None)
te._execute = MagicMock(return_value=dict())
res = te.run()
te._get_loop_items = MagicMock(return_value=[])
res = te.run()
te._get_loop_items = MagicMock(return_value=['a','b','c'])
te._run_loop = MagicMock(return_value=[dict(item='a', changed=True), dict(item='b', failed=True), dict(item='c')])
res = te.run()
te._get_loop_items = MagicMock(side_effect=AnsibleError(""))
res = te.run()
self.assertIn("failed", res)
def test_task_executor_run(self):
fake_loader = DictDataLoader({})
mock_host = MagicMock()
mock_task = MagicMock()
mock_task._role._role_path = '/path/to/role/foo'
mock_play_context = MagicMock()
mock_shared_loader = MagicMock()
new_stdin = None
job_vars = dict()
te = TaskExecutor(
host=mock_host,
task=mock_task,
job_vars=job_vars,
play_context=mock_play_context,
new_stdin=new_stdin,
loader=fake_loader,
shared_loader_obj=mock_shared_loader,
)
te._get_loop_items = MagicMock(return_value=None)
te._execute = MagicMock(return_value=dict())
res = te.run()
te._get_loop_items = MagicMock(return_value=[])
res = te.run()
te._get_loop_items = MagicMock(return_value=['a', 'b', 'c'])
te._run_loop = MagicMock(return_value=[
dict(item='a', changed=True),
dict(item='b', failed=True),
dict(item='c')
])
res = te.run()
te._get_loop_items = MagicMock(side_effect=AnsibleError(""))
res = te.run()
self.assertIn("failed", res)
self.send(result)
except:
return False
return True
def run(self, (host, task, basedir, job_vars, play_context,
shared_loader_obj)):
debug("there's work to be done!")
debug("got a task/handler to work on: %s" % task)
self._loader.set_basedir(basedir)
task.set_loader(self._loader)
debug("running TaskExecutor() for %s/%s" % (host, task))
executor = TaskExecutor(host, task, job_vars, play_context,
self._new_stdin, self._loader,
shared_loader_obj)
self._TaskExecutorPool.apply_async(
executor.run,
callback=(lambda result, host=host, task=task: self.send_result(
result, host, task)))
return True
def send_result(self, result, host, task):
debug("done running TaskExecutor() for %s/%s" % (host, task))
res = TaskResult(host, task, result)
self.send(res)
class Result(IOGenerator):
def exception(self, record, (type, value, trace)):
