def test_execute_status_command(self, CustomServiceOrchestrator_mock, build_mock, execute_command_mock, requestComponentStatus_mock, read_stack_version_mock, determine_command_format_version_mock, status_update_callback): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) build_mock.return_value = "dummy report" # Check execution ov V1 status command determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V1 actionQueue.execute_status_command(self.status_command) report = actionQueue.result() expected = 'dummy report' self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0], expected) self.assertFalse(requestComponentStatus_mock.called) # Check execution ov V2 status command requestComponentStatus_mock.reset_mock() determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V2 actionQueue.execute_status_command(self.status_command) report = actionQueue.result() expected = 'dummy report' self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0], expected) self.assertTrue(requestComponentStatus_mock.called)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock, get_mock, process_command_mock): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() config = MagicMock() actionQueue = ActionQueue(config, dummy_controller) actionQueue.start() time.sleep(0.1) actionQueue.stop() actionQueue.join() self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.') self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command(self, CustomServiceOrchestrator_mock, execute_command_mock, requestComponentStatus_mock, status_update_callback): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'} actionQueue.execute_status_command(self.status_command) report = actionQueue.result() expected = 'dummy report' self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["status"], expected) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertTrue(requestComponentStatus_mock.called)
def test_determine_command_format_version(self, CustomServiceOrchestrator_mock): CustomServiceOrchestrator_mock.return_value = None v1_command = {'commandParams': {'schema_version': '1.0'}} v2_command = {'commandParams': {'schema_version': '2.0'}} current_command = { # Absent 'commandParams' section } dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) self.assertEqual( actionQueue.determine_command_format_version(v1_command), ActionQueue.COMMAND_FORMAT_V1) self.assertEqual( actionQueue.determine_command_format_version(v2_command), ActionQueue.COMMAND_FORMAT_V2) self.assertEqual( actionQueue.determine_command_format_version(current_command), ActionQueue.COMMAND_FORMAT_V1)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock, get_mock, process_command_mock): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() config = MagicMock() actionQueue = ActionQueue(config, dummy_controller) actionQueue.start() time.sleep(3) actionQueue.stop() actionQueue.join() self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.') self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock, execute_command_mock, runCommand_mock, status_update_callback): csoMocks = [MagicMock()] CustomServiceOrchestrator_mock.side_effect = csoMocks csoMocks[0].status_commands_stdout = None csoMocks[0].status_commands_stderr = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) runCommand_mock.return_value = {'configurations': {}} actionQueue.execute_status_command(self.status_command_with_config) report = actionQueue.result() self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertEqual(report['componentStatus'][0]["configurations"], {}) self.assertFalse(runCommand_mock.called)
def test_execute_status_command_expect_config( self, CustomServiceOrchestrator_mock, execute_command_mock, runCommand_mock, status_update_callback): csoMocks = [MagicMock()] CustomServiceOrchestrator_mock.side_effect = csoMocks csoMocks[0].status_commands_stdout = None csoMocks[0].status_commands_stderr = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) runCommand_mock.return_value = {'configurations': {}} actionQueue.execute_status_command(self.status_command_with_config) report = actionQueue.result() self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertEqual(report['componentStatus'][0]["configurations"], {}) self.assertFalse(runCommand_mock.called)
def test_process_command(self, execute_command_mock, print_exc_mock): dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) execution_command = { 'commandType': ActionQueue.EXECUTION_COMMAND, } wrong_command = { 'commandType': "SOME_WRONG_COMMAND", } # Try wrong command actionQueue.process_command(wrong_command) self.assertFalse(execute_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() print_exc_mock.reset_mock() # Try normal execution actionQueue.process_command(execution_command) self.assertTrue(execute_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() print_exc_mock.reset_mock() execute_command_mock.reset_mock() print_exc_mock.reset_mock() # Try exception to check proper logging def side_effect(self): raise Exception("TerribleException") execute_command_mock.side_effect = side_effect actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called) print_exc_mock.reset_mock() actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called)
def test_determine_command_format_version(self, CustomServiceOrchestrator_mock): CustomServiceOrchestrator_mock.return_value = None v1_command = { 'commandParams': { 'schema_version': '1.0' } } v2_command = { 'commandParams': { 'schema_version': '2.0' } } current_command = { # Absent 'commandParams' section } dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) self.assertEqual(actionQueue.determine_command_format_version(v1_command), ActionQueue.COMMAND_FORMAT_V1) self.assertEqual(actionQueue.determine_command_format_version(v2_command), ActionQueue.COMMAND_FORMAT_V2) self.assertEqual(actionQueue.determine_command_format_version(current_command), ActionQueue.COMMAND_FORMAT_V1)
def test_build(self): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) heartbeat = Heartbeat(actionQueue, config) result = heartbeat.build({}, 100) print "Heartbeat: " + str(result) self.assertEquals(result['hostname'] != '', True, "hostname should not be empty") self.assertEquals(result['responseId'], 100) self.assertEquals('componentStatus' not in result, True, "Heartbeat should contain componentStatus") self.assertEquals(result['reports'] is not None, True, "Heartbeat should contain reports") self.assertEquals(result['timestamp'] >= 1353679373880L, True) self.assertEquals(len(result['nodeStatus']), 2) self.assertEquals(result['nodeStatus']['cause'], "NONE") self.assertEquals(result['nodeStatus']['status'], "HEALTHY") # result may or may NOT have an agentEnv structure in it self.assertEquals((len(result) is 4) or (len(result) is 5), True) self.assertEquals(not heartbeat.reports, True, "Heartbeat should not contain task in progress")
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock, resolve_script_path_mock): tempdir = tempfile.gettempdir() config = MagicMock() config.get.return_value = "something" config.getResolvedPath.return_value = tempdir config.getWorkRootPath.return_value = tempdir config.getLogPath.return_value = tempdir # Make file read calls visible def open_side_effect(file, mode): if mode == 'r': file_mock = MagicMock() file_mock.read.return_value = "Read from " + str(file) return file_mock else: return self.original_open(file, mode) open_mock.side_effect = open_side_effect json_load_mock.return_value = '' resolve_script_path_mock.return_value = "abc.py" dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) unfreeze_flag = threading.Event() python_execution_result_dict = { 'stdout': 'out', 'stderr': 'stderr', 'structuredOut': '' } def side_effect(py_file, script_params, tmpoutfile, tmperrfile, timeout, tmpstrucoutfile, override_output_files, environment_vars): unfreeze_flag.wait() return python_execution_result_dict def patched_aq_execute_command(command): # We have to perform patching for separate thread in the same thread with patch.object(PythonExecutor, "run_file") as runCommand_mock: runCommand_mock.side_effect = side_effect actionQueue.execute_command(command) ### Test install/start/stop command ### ## Test successful execution with configuration tags python_execution_result_dict['status'] = 'COMPLETE' python_execution_result_dict['exitcode'] = 0 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() # check in progress report # wait until ready while True: time.sleep(0.1) report = actionQueue.result() if len(report['reports']) != 0: break expected = { 'status': 'IN_PROGRESS', 'stderr': 'Read from {0}/errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}/output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitCode': 777 } self.assertEqual(report['reports'][0], expected) # Continue command execution unfreeze_flag.set() # wait until ready while report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report configname = os.path.join(tempdir, 'command-3.json') expected = { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitCode': 0 } self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) self.assertTrue(os.path.isfile(configname)) # Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE) self.assertEqual(status_update_callback_mock.call_count, 2) os.remove(configname) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0) ## Test failed execution python_execution_result_dict['status'] = 'FAILED' python_execution_result_dict['exitcode'] = 13 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() unfreeze_flag.set() # check in progress report # wait until ready report = actionQueue.result() while len(report['reports']) == 0 or \ report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report expected = { 'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitCode': 13 } self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0)
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock, resolve_script_path_mock): tempdir = tempfile.gettempdir() config = MagicMock() config.get.return_value = "something" config.getResolvedPath.return_value = tempdir config.getWorkRootPath.return_value = tempdir config.getLogPath.return_value = tempdir # Make file read calls visible def open_side_effect(file, mode): if mode == 'r': file_mock = MagicMock() file_mock.read.return_value = "Read from " + str(file) return file_mock else: return self.original_open(file, mode) open_mock.side_effect = open_side_effect json_load_mock.return_value = '' resolve_script_path_mock.return_value = "abc.py" dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) unfreeze_flag = threading.Event() python_execution_result_dict = { 'stdout': 'out', 'stderr': 'stderr', 'structuredOut': '' } def side_effect(py_file, script_params, tmpoutfile, tmperrfile, timeout, tmpstrucoutfile, override_output_files, environment_vars): unfreeze_flag.wait() return python_execution_result_dict def patched_aq_execute_command(command): # We have to perform patching for separate thread in the same thread with patch.object(PythonExecutor, "run_file") as runCommand_mock: runCommand_mock.side_effect = side_effect actionQueue.execute_command(command) ### Test install/start/stop command ### ## Test successful execution with configuration tags python_execution_result_dict['status'] = 'COMPLETE' python_execution_result_dict['exitcode'] = 0 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() # check in progress report # wait until ready while True: time.sleep(0.1) report = actionQueue.result() if len(report['reports']) != 0: break expected = {'status': 'IN_PROGRESS', 'stderr': 'Read from {0}/errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}/output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitCode': 777} self.assertEqual(report['reports'][0], expected) # Continue command execution unfreeze_flag.set() # wait until ready while report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report configname = os.path.join(tempdir, 'command-3.json') expected = {'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': {'global': {'tag': 'v1'}}, 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitCode': 0} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) self.assertTrue(os.path.isfile(configname)) # Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE) self.assertEqual(status_update_callback_mock.call_count, 2) os.remove(configname) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0) ## Test failed execution python_execution_result_dict['status'] = 'FAILED' python_execution_result_dict['exitcode'] = 13 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() unfreeze_flag.set() # check in progress report # wait until ready report = actionQueue.result() while len(report['reports']) == 0 or \ report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report expected = {'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitCode': 13} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0)
def test_build_long_result(self, result_mock): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) result_mock.return_value = { 'reports': [{ 'status': 'IN_PROGRESS', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitCode': 777 }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName', 'roleCommand': 'UPGRADE', 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17, 'taskId': 'taskId', 'exitCode': 0 }, { 'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitCode': 13 }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitCode': 0 }], 'componentStatus': [ { 'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult': True }, { 'status': 'UNHEALTHY', 'componentName': 'NAMENODE', 'reportResult': True }, { 'status': 'UNHEALTHY', 'componentName': 'HBASE_MASTER', 'reportResult': False }, ], } heartbeat = Heartbeat(actionQueue, config) hb = heartbeat.build({}, 10) hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' expected = { 'nodeStatus': { 'status': 'HEALTHY', 'cause': 'NONE' }, 'timestamp': 'timestamp', 'hostname': 'hostname', 'responseId': 10, 'reports': [{ 'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'taskId': 3, 'exitCode': 777 }, { 'status': 'COMPLETED', 'roleCommand': 'UPGRADE', 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17, 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName', 'taskId': 'taskId', 'exitCode': 0 }, { 'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitCode': 13 }, { 'status': 'COMPLETED', 'stdout': 'out', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'taskId': 3, 'exitCode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr' }], 'componentStatus': [{ 'status': 'HEALTHY', 'componentName': 'DATANODE' }, { 'status': 'UNHEALTHY', 'componentName': 'NAMENODE' }] } self.assertEquals(hb, expected)