def initialize_strategy(self): self.initializer = Initializer(self.world) self.actionQueue = ActionQueue(self.world) self.__weather_map = WeatherMap(self.world.weather_by_cell_x_y, Constant.WEATHER_MAP_CELL_SIZE) self.__terrain_map = TerrainMap(self.world.terrain_by_cell_x_y, Constant.TERRAIN_MAP_CELL_SIZE)
def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get( 'server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config)
def test_process_command2(self, execute_status_command_mock, execute_command_mock, print_exc_mock): dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger) execution_command = { 'commandType': ActionQueue.EXECUTION_COMMAND, } status_command = { 'commandType': ActionQueue.STATUS_COMMAND, } wrong_command = { 'commandType': "SOME_WRONG_COMMAND", } # Try wrong command actionQueue.process_command(wrong_command) self.assertFalse(execute_command_mock.called) self.assertFalse(execute_status_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() execute_status_command_mock.reset_mock() print_exc_mock.reset_mock() # Try normal execution actionQueue.process_command(execution_command) self.assertTrue(execute_command_mock.called) self.assertFalse(execute_status_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() execute_status_command_mock.reset_mock() print_exc_mock.reset_mock() actionQueue.process_command(status_command) self.assertFalse(execute_command_mock.called) self.assertTrue(execute_status_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() execute_status_command_mock.reset_mock() print_exc_mock.reset_mock() # Try exception to check proper logging def side_effect(self): raise Exception("TerribleException") execute_command_mock.side_effect = side_effect actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called) print_exc_mock.reset_mock() execute_status_command_mock.side_effect = side_effect actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock, get_mock, process_command_mock): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() config = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) actionQueue.start() time.sleep(3) actionQueue.stop() actionQueue.join() self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.') self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command(self, CustomServiceOrchestrator_mock, execute_command_mock, requestComponentStatus_mock, status_update_callback): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger) requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'} actionQueue.execute_status_command(self.status_command) report = actionQueue.result() expected = 'dummy report' self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["status"], expected) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertTrue(requestComponentStatus_mock.called)
def test_build_result3(self, result_mock): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) result_mock.return_value = { 'reports': [{ 'status': 'COMPLETED', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': False }], 'componentStatus': [] } heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger) commandResult = {} hb = heartbeat.build(commandResult, 10) hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' expected = { 'package': '', 'nodeStatus': { 'status': 'HEALTHY', 'cause': 'NONE' }, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', 'responseId': 10, 'reports': [] } self.assertEqual.__self__.maxDiff = None self.assertEquals(hb, expected) self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock, execute_command_mock, runCommand_mock, status_update_callback): csoMocks = [MagicMock()] CustomServiceOrchestrator_mock.side_effect = csoMocks csoMocks[0].status_commands_stdout = None csoMocks[0].status_commands_stderr = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger) runCommand_mock.return_value = {'configurations': {}} actionQueue.execute_status_command(self.status_command_with_config) report = actionQueue.result() self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertEqual(report['componentStatus'][0]["configurations"], {}) self.assertFalse(runCommand_mock.called)
def test_build(self): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) heartbeat = Heartbeat(actionQueue, config) result = heartbeat.build({}, 100) print "Heartbeat: " + str(result) self.assertEquals(result['hostname'] != '', True, "hostname should not be empty") self.assertEquals(result['responseId'], 100) self.assertEquals('componentStatus' not in result, True, "Heartbeat should contain componentStatus") self.assertEquals(result['reports'] is not None, True, "Heartbeat should contain reports") self.assertEquals(result['timestamp'] >= 1353679373880L, True) self.assertEquals(len(result['nodeStatus']), 2) self.assertEquals(result['nodeStatus']['cause'], "NONE") self.assertEquals(result['nodeStatus']['status'], "HEALTHY") # result may or may NOT have an agentEnv structure in it self.assertEquals((len(result) is 6) or (len(result) is 7), True) self.assertEquals(not heartbeat.reports, True, "Heartbeat should not contain task in progress")
def main(argv=None): actionQueue = ActionQueue(AgentConfig.getConfig()) heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build('3', 3))
def main(argv=None): actionQueue = ActionQueue(AmbariConfig.config) heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build('3', 3))
def test_build_long_result(self, result_mock): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) result_mock.return_value = { 'reports': [{ 'status': 'IN_PROGRESS', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': True }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName', 'roleCommand': 'UPGRADE', 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17, 'taskId': 'taskId', 'exitcode': 0, 'reportResult': True }, { 'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 13, 'reportResult': True }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 0, 'reportResult': True }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 0, 'reportResult': False }], 'componentStatus': [ { 'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult': True }, { 'status': 'UNHEALTHY', 'componentName': 'NAMENODE', 'reportResult': True }, { 'status': 'UNHEALTHY', 'componentName': 'HBASE_MASTER', 'reportResult': False }, ], } heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger) # State.STARTED results in agentState to be set to 4 (enum order) hb = heartbeat.build({}, 10) hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' expected = { 'nodeStatus': { 'status': 'HEALTHY', 'cause': 'NONE' }, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', 'responseId': 10, 'reports': [{ 'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'taskId': 3, 'exitcode': 777 }, { 'status': 'COMPLETED', 'roleCommand': 'UPGRADE', 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17, 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName', 'taskId': 'taskId', 'exitcode': 0 }, { 'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitcode': 13 }, { 'status': 'COMPLETED', 'stdout': 'out', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'taskId': 3, 'exitcode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr' }], 'componentStatus': [{ 'status': 'HEALTHY', 'componentName': 'DATANODE' }, { 'status': 'UNHEALTHY', 'componentName': 'NAMENODE' }] } self.assertEqual.__self__.maxDiff = None self.assertEquals(hb, expected)
def main(argv=None): actionQueue = ActionQueue() heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build())
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock, resolve_script_path_mock): self.assertEqual.__self__.maxDiff = None tempdir = tempfile.gettempdir() config = MagicMock() config.get.return_value = "something" config.getResolvedPath.return_value = tempdir config.getWorkRootPath.return_value = tempdir config.getLogPath.return_value = tempdir # Make file read calls visible def open_side_effect(file, mode): if mode == 'r': file_mock = MagicMock() file_mock.read.return_value = "Read from " + str(file) return file_mock else: return self.original_open(file, mode) open_mock.side_effect = open_side_effect json_load_mock.return_value = '' resolve_script_path_mock.return_value = "abc.py" dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) unfreeze_flag = threading.Event() python_execution_result_dict = { 'stdout': 'out', 'stderr': 'stderr', 'structuredOut': '' } def side_effect(py_file, script_params, tmpoutfile, tmperrfile, timeout, tmpstrucoutfile, loglevel, override_output_files, environment_vars): unfreeze_flag.wait() return python_execution_result_dict def patched_aq_execute_command(command): # We have to perform patching for separate thread in the same thread with patch.object(PythonExecutor, "run_file") as runCommand_mock: runCommand_mock.side_effect = side_effect actionQueue.execute_command(command) ### Test install/start/stop command ### ## Test successful execution with configuration tags python_execution_result_dict['status'] = 'COMPLETE' python_execution_result_dict['exitcode'] = 0 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() # check in progress report # wait until ready while True: time.sleep(0.1) report = actionQueue.result() if len(report['reports']) != 0: break expected = {'status': 'IN_PROGRESS', 'stderr': 'Read from {0}/errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}/output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': True} if IS_WINDOWS: expected = {'status': 'IN_PROGRESS', 'stderr': 'Read from {0}\\errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}\\output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': True} self.assertEqual(report['reports'][0], expected) # Continue command execution unfreeze_flag.set() # wait until ready while report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report configname = os.path.join(tempdir, 'command-3.json') expected = {'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': {'global': {'tag': 'v1'}}, 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitcode': 0, 'allocatedPorts': {}, 'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}, 'reportResult': True} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) self.assertTrue(os.path.isfile(configname)) # Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE) self.assertEqual(status_update_callback_mock.call_count, 2) os.remove(configname) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0) ## Test failed execution python_execution_result_dict['status'] = 'FAILED' python_execution_result_dict['exitcode'] = 13 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() unfreeze_flag.set() # check in progress report # wait until ready report = actionQueue.result() while len(report['reports']) == 0 or \ report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report expected = {'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitcode': 13, 'reportResult': True} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0)
def start(self): self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register() self.heartbeat = Heartbeat(self.actionQueue) pass