def test_watchdog_1(self, kill_process_with_children_mock):
"""
Tests whether watchdog works
"""
subproc_mock = self.subprocess32_mockup()
executor = PythonExecutor("/tmp", AmbariConfig())
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
PYTHON_TIMEOUT_SECONDS = 0.1
kill_process_with_children_mock.side_effect = lambda pid: subproc_mock.terminate(
)
def launch_python_subprocess32_method(command, tmpout, tmperr):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess32 = launch_python_subprocess32_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = None
callback_method = MagicMock()
thread = Thread(target=executor.run_file,
args=("fake_puppetFile", ["arg1", "arg2"], tmpoutfile,
tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout,
callback_method, '1'))
thread.start()
time.sleep(0.1)
subproc_mock.finished_event.wait()
self.assertEquals(subproc_mock.was_terminated, True,
"subprocess32 should be terminated due to timeout")
self.assertTrue(callback_method.called)
def test_python_command(self):
executor = PythonExecutor("/tmp", AmbariConfig())
command = executor.python_command("script", ["script_param1"])
self.assertEqual(3, len(command))
self.assertTrue("python" in command[0].lower())
self.assertEquals("script", command[1])
self.assertEquals("script_param1", command[2])
def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
FileCache_mock,
dump_command_to_json_mock,
get_py_executor_mock):
FileCache_mock.return_value = None
get_custom_actions_base_dir_mock.return_value = "some path"
_, script = tempfile.mkstemp()
command = {
'role' : 'any',
'commandParams': {
'script_type': 'PYTHON',
'script': 'some_custom_action.py',
'command_timeout': '600',
'jdk_location' : 'some_location'
},
'taskId' : '13',
'roleCommand': 'ACTIONEXECUTE',
'commandType': 'BACKGROUND_EXECUTION_COMMAND',
'__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
MagicMock(), MagicMock())
}
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
import TestActionQueue
pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
TestActionQueue.patch_output_file(pyex)
pyex.condenseOutput = MagicMock()
get_py_executor_mock.return_value = pyex
orchestrator.dump_command_to_json = MagicMock()
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 777)
def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
FileCache_mock,
dump_command_to_json_mock,
get_py_executor_mock):
FileCache_mock.return_value = None
get_custom_actions_base_dir_mock.return_value = "some path"
_, script = tempfile.mkstemp()
command = {
'role' : 'any',
'commandParams': {
'script_type': 'PYTHON',
'script': 'some_custom_action.py',
'command_timeout': '600',
'jdk_location' : 'some_location'
},
'taskId' : '13',
'roleCommand': 'ACTIONEXECUTE',
'commandType': 'BACKGROUND_EXECUTION_COMMAND',
'__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
MagicMock(), MagicMock())
}
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
import TestActionQueue
pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
TestActionQueue.patch_output_file(pyex)
pyex.condenseOutput = MagicMock()
get_py_executor_mock.return_value = pyex
orchestrator.dump_command_to_json = MagicMock()
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 777)
def test_execute_python_executor(self, read_stack_version_mock,
resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir,
actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock(
)
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {
'condensed_result':
copy.copy(process_condensed_result),
'handle':
copy.copy(handle),
'command_status':
actionQueue.commandStatuses.get_command_status(
handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(
actionQueue.on_background_command_complete_callback, None,
command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'],
ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(
self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0]['stdout'], 'process_out')
def __test_execute_python_executor(self, resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
initializer_module = InitializerModule()
initializer_module.init()
initializer_module.config = cfg
initializer_module.metadata_cache.cache_update({CLUSTER_ID:{'clusterLevelParams':{}}}, 'abc')
initializer_module.configurations_cache.cache_update({CLUSTER_ID:{}}, 'abc')
initializer_module.host_level_params_cache.cache_update({CLUSTER_ID:{}}, 'abc')
CustomServiceOrchestrator.runCommand = default_run_command
actionQueue = ActionQueue(initializer_module)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock()
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {'condensed_result' : copy.copy(process_condensed_result),
'handle' : copy.copy(handle),
'command_status' : actionQueue.commandStatuses.get_command_status(handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(actionQueue.on_background_command_complete_callback,
None, command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty();
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'], ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(reports), 1)
self.assertEqual(reports[0]['stdout'], 'process_out')
def test_is_successfull(self):
executor = PythonExecutor("/tmp", AmbariConfig())
executor.python_process_has_been_killed = False
self.assertTrue(executor.isSuccessfull(0))
self.assertFalse(executor.isSuccessfull(1))
executor.python_process_has_been_killed = True
self.assertFalse(executor.isSuccessfull(0))
self.assertFalse(executor.isSuccessfull(1))
def test_watchdog_2(self):
"""
Tries to catch false positive watchdog invocations
"""
subproc_mock = self.subprocess32_mockup()
executor = PythonExecutor("/tmp", AmbariConfig())
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
PYTHON_TIMEOUT_SECONDS = 5
def launch_python_subprocess32_method(command, tmpout, tmperr):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess32 = launch_python_subprocess32_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = 0
callback_method = MagicMock()
thread = Thread(target=executor.run_file,
args=("fake_puppetFile", ["arg1", "arg2"], tmpoutfile,
tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout,
callback_method, "1-1"))
thread.start()
time.sleep(0.1)
subproc_mock.should_finish_event.set()
subproc_mock.finished_event.wait()
self.assertEquals(
subproc_mock.was_terminated, False,
"subprocess32 should not be terminated before timeout")
self.assertEquals(
subproc_mock.returncode, 0,
"subprocess32 should not be terminated before timeout")
self.assertTrue(callback_method.called)
def test_execution_results(self):
subproc_mock = self.subprocess32_mockup()
executor = PythonExecutor("/tmp", AmbariConfig())
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
tmp_file = tempfile.NamedTemporaryFile(
) # the structured out file should be preserved across calls to the hooks and script.
tmpstructuredoutfile = tmp_file.name
tmp_file.close()
PYTHON_TIMEOUT_SECONDS = 5
def launch_python_subprocess32_method(command, tmpout, tmperr):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess32 = launch_python_subprocess32_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = 0
subproc_mock.should_finish_event.set()
callback_method = MagicMock()
result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile,
tmperrfile, PYTHON_TIMEOUT_SECONDS,
tmpstructuredoutfile, callback_method,
"1-1")
self.assertEquals(result, {
'exitcode': 0,
'stderr': '',
'stdout': '',
'structuredOut': {}
})
self.assertTrue(callback_method.called)
def test_back_up_log_file_if_exists(self, rename_mock, isfile_mock):
# Test case when previous log file is absent
isfile_mock.return_value = False
log_file = "/var/lib/ambari-agent/data/output-13.txt"
executor = PythonExecutor("/tmp", AmbariConfig())
executor.back_up_log_file_if_exists(log_file)
self.assertEquals(isfile_mock.called, True)
self.assertEquals(rename_mock.called, False)
isfile_mock.reset_mock()
# Test case when 3 previous log files are absent
isfile_mock.side_effect = [True, True, True, False]
log_file = "/var/lib/ambari-agent/data/output-13.txt"
executor = PythonExecutor("/tmp", AmbariConfig())
executor.back_up_log_file_if_exists(log_file)
self.assertEquals(isfile_mock.called, True)
self.assertEquals(rename_mock.call_args_list[0][0][0],
"/var/lib/ambari-agent/data/output-13.txt")
self.assertEquals(rename_mock.call_args_list[0][0][1],
"/var/lib/ambari-agent/data/output-13.txt.2")
pass
def test_cancel_backgound_command(self, read_stack_version_mock,
resolve_hook_script_path_mock,
resolve_script_path_mock, FileCache_mock,
kill_process_with_children_mock,
get_py_executor_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
# shell.kill_process_with_children = MagicMock()
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f(a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir,
actionQueue.customServiceOrchestrator.config)
TestActionQueue.patch_output_file(pyex)
pyex.prepare_process_result = MagicMock()
get_py_executor_mock.return_value = pyex
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited'] = ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(
actionQueue.on_background_command_complete_callback,
command_complete_w, None)
execute_command = copy.deepcopy(
TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19, '')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
def test_cancel_backgound_command(self, resolve_hook_script_path_mock,
resolve_script_path_mock, FileCache_mock, kill_process_with_children_mock,
get_py_executor_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f (a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
TestActionQueue.patch_output_file(pyex)
pyex.prepare_process_result = MagicMock()
get_py_executor_mock.return_value = pyex
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited']= ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19,'reason')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)