def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get( 'server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config)
def initialize_strategy(self): self.initializer = Initializer(self.world) self.actionQueue = ActionQueue(self.world) self.__weather_map = WeatherMap(self.world.weather_by_cell_x_y, Constant.WEATHER_MAP_CELL_SIZE) self.__terrain_map = TerrainMap(self.world.terrain_by_cell_x_y, Constant.TERRAIN_MAP_CELL_SIZE)
class Controller(threading.Thread): def __init__(self, config): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = socket.gethostname() server_secured_url = 'https://' + config.get( 'server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/register/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/heartbeat/' + self.hostname self.netutil = NetUtil() def start(self): self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register() self.heartbeat = Heartbeat(self.actionQueue) pass def __del__(self): logger.info("Server connection disconnected.") pass def registerWithServer(self): retry = False firstTime = True registered = False id = -1 ret = {} while registered == False: try: data = json.dumps(self.register.build(id)) logger.info("Registering with the server " + pprint.pformat(data)) req = urllib2.Request(self.registerUrl, data, {'Content-Type': 'application/json'}) stream = security.secured_url_open(req) response = stream.read() stream.close() ret = json.loads(response) logger.info("Registered with the server with " + pprint.pformat(ret)) registered = True pass except Exception, err: delay = self.netutil.CONNECT_SERVER_RETRY_INTERVAL_SEC logger.info("Unable to connect to: " + self.registerUrl, exc_info=True) """ Sleeping for {0} seconds and then retrying again """.format( delay) time.sleep(delay) pass pass return ret
def perform_current_step(self, action_queue: ActionQueue, updated_vehicle_x_y): # Двигаем юниты на 1 линию if self.current_step == self.STEP_MOVE_TO_INITIAL_POSITIONS: if not self.__check_step_started(): command_builder = InitializerStep0_MoveToInitialPositions( self.__me, self.__weather_map, self.__terrain_map, self.__all_vehicles) commands = command_builder.get_command_list() if len(commands) == 0: if self.__check_interrupt_condition(): self.__set_step(self.STEP_INTERRUPT_CONSOLIDATE) else: self.__set_step() return for command in commands: action_queue.push(command) self.__mark_step_started() if self.__check_move_finished(updated_vehicle_x_y): self.__mark_move_started(False) self.__mark_step_started(False) # Расширяем каждый столбец юнитов в 3 раза elif self.current_step == self.STEP_UNCONSOLIDATE: self.__perform_casual_step(InitializerStep1_Unconsolidate, action_queue, updated_vehicle_x_y) # Делаем вертикальный сдвиг на разное смещение разных типов юнитов elif self.current_step == self.STEP_SHIFT_VERTICAL: self.__perform_casual_step(InitializerStep2_ShiftVertical, action_queue, updated_vehicle_x_y) # Сдвигаем юнитов в одну колонку elif self.current_step == self.STEP_SHIFT_TOGETHER: self.__perform_casual_step(InitializerStep3_ShiftTogether, action_queue, updated_vehicle_x_y) # Сдвигаем группы по 3 через 3 elif self.current_step == self.STEP_SHIFT_HORIZONTAL: self.__perform_casual_step(InitializerStep4_ShiftHorizontal, action_queue, updated_vehicle_x_y) # Формируем длинные ряды elif self.current_step == self.STEP_MAKE_LONG_ROWS: self.__perform_casual_step(InitializerStep5_MakeLongRows, action_queue, updated_vehicle_x_y) # Поворачиваемся elif self.current_step == self.STEP_ROTATE: self.__perform_casual_step(InitializerStep6_Rotate, action_queue, updated_vehicle_x_y) elif self.current_step == self.STEP_INTERRUPT_CONSOLIDATE: self.__perform_casual_step(InitializerStep8_InterruptConsolidate, action_queue, updated_vehicle_x_y) elif self.current_step == self.STEP_INTERRUPT_SHIFT_AIR_UNITS: self.__perform_casual_step(InitializerStep9_InterruptShiftAirUnits, action_queue, updated_vehicle_x_y)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock, get_mock, process_command_mock): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() config = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) actionQueue.start() time.sleep(3) actionQueue.stop() actionQueue.join() self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.') self.assertTrue(process_command_mock.call_count > 1)
def __perform_casual_step(self, command_builder_class, action_queue: ActionQueue, updated_vehicle_x_y): if not self.__check_step_started(): command_builder = command_builder_class(self.__me, self.__weather_map, self.__terrain_map, self.__all_vehicles) commands = command_builder.get_command_list() if len(commands) == 0: self.__set_step() return for command in commands: action_queue.push(command) self.__mark_step_started() elif self.__check_move_finished(updated_vehicle_x_y): self.__set_step()
def test_execute_status_command(self, CustomServiceOrchestrator_mock, execute_command_mock, requestComponentStatus_mock, status_update_callback): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'} actionQueue.execute_status_command(self.status_command) report = actionQueue.result() expected = 'dummy report' self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["status"], expected) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertTrue(requestComponentStatus_mock.called)
def test_execute_status_command(self, CustomServiceOrchestrator_mock, execute_command_mock, requestComponentStatus_mock, status_update_callback): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger) requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'} actionQueue.execute_status_command(self.status_command) report = actionQueue.result() expected = 'dummy report' self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["status"], expected) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertTrue(requestComponentStatus_mock.called)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock, get_mock, process_command_mock): CustomServiceOrchestrator_mock.return_value = None dummy_controller = MagicMock() config = MagicMock() actionQueue = ActionQueue(config, dummy_controller) actionQueue.start() time.sleep(3) actionQueue.stop() actionQueue.join() self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.') self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock, execute_command_mock, runCommand_mock, status_update_callback): csoMocks = [MagicMock()] CustomServiceOrchestrator_mock.side_effect = csoMocks csoMocks[0].status_commands_stdout = None csoMocks[0].status_commands_stderr = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger) runCommand_mock.return_value = {'configurations': {}} actionQueue.execute_status_command(self.status_command_with_config) report = actionQueue.result() self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertEqual(report['componentStatus'][0]["configurations"], {}) self.assertFalse(runCommand_mock.called)
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock, execute_command_mock, runCommand_mock, status_update_callback): csoMocks = [MagicMock()] CustomServiceOrchestrator_mock.side_effect = csoMocks csoMocks[0].status_commands_stdout = None csoMocks[0].status_commands_stderr = None dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) runCommand_mock.return_value = {'configurations': {}} actionQueue.execute_status_command(self.status_command_with_config) report = actionQueue.result() self.assertEqual(len(report['componentStatus']), 1) self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER") self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO") self.assertEqual(report['componentStatus'][0]["clusterName"], "c1") self.assertEqual(report['componentStatus'][0]["configurations"], {}) self.assertFalse(runCommand_mock.called)
def __init__(self, parent, nao): super(ActionModel, self).__init__(parent) self._list = ActionQueue(nao) self._list.cleared.connect(self.on__list_cleared) self._list.dequeued.connect(self.on__list_dequeued) self._list.enqueued.connect(self.on__list_enqueued) self._list.removed.connect(self.on__list_dequeued) self._timer = ThreadTimer() self._timer.start() self._timer.addToThread(self._list) self._timer.timeElapsed.connect(self._list.execute)
def test_process_command(self, execute_command_mock, print_exc_mock): dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger) execution_command = { 'commandType': ActionQueue.EXECUTION_COMMAND, } wrong_command = { 'commandType': "SOME_WRONG_COMMAND", } # Try wrong command actionQueue.process_command(wrong_command) self.assertFalse(execute_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() print_exc_mock.reset_mock() # Try normal execution actionQueue.process_command(execution_command) self.assertTrue(execute_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() print_exc_mock.reset_mock() execute_command_mock.reset_mock() print_exc_mock.reset_mock() # Try exception to check proper logging def side_effect(self): raise Exception("TerribleException") execute_command_mock.side_effect = side_effect actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called) print_exc_mock.reset_mock() actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called)
def test_process_command(self, execute_command_mock, print_exc_mock): dummy_controller = MagicMock() actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller) execution_command = { 'commandType': ActionQueue.EXECUTION_COMMAND, } wrong_command = { 'commandType': "SOME_WRONG_COMMAND", } # Try wrong command actionQueue.process_command(wrong_command) self.assertFalse(execute_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() print_exc_mock.reset_mock() # Try normal execution actionQueue.process_command(execution_command) self.assertTrue(execute_command_mock.called) self.assertFalse(print_exc_mock.called) execute_command_mock.reset_mock() print_exc_mock.reset_mock() execute_command_mock.reset_mock() print_exc_mock.reset_mock() # Try exception to check proper logging def side_effect(self): raise Exception("TerribleException") execute_command_mock.side_effect = side_effect actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called) print_exc_mock.reset_mock() actionQueue.process_command(execution_command) self.assertTrue(print_exc_mock.called)
def test_build_result3(self, result_mock): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) result_mock.return_value = { 'reports': [{ 'status': 'COMPLETED', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': False }], 'componentStatus': [] } heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger) commandResult = {} hb = heartbeat.build(commandResult, 10) hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' expected = { 'package': '', 'nodeStatus': { 'status': 'HEALTHY', 'cause': 'NONE' }, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', 'responseId': 10, 'reports': [] } self.assertEqual.__self__.maxDiff = None self.assertEquals(hb, expected) self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get('server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config)
def test_build(self): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) heartbeat = Heartbeat(actionQueue, config) result = heartbeat.build({}, 100) print "Heartbeat: " + str(result) self.assertEquals(result['hostname'] != '', True, "hostname should not be empty") self.assertEquals(result['responseId'], 100) self.assertEquals('componentStatus' not in result, True, "Heartbeat should contain componentStatus") self.assertEquals(result['reports'] is not None, True, "Heartbeat should contain reports") self.assertEquals(result['timestamp'] >= 1353679373880L, True) self.assertEquals(len(result['nodeStatus']), 2) self.assertEquals(result['nodeStatus']['cause'], "NONE") self.assertEquals(result['nodeStatus']['status'], "HEALTHY") # result may or may NOT have an agentEnv structure in it self.assertEquals((len(result) is 6) or (len(result) is 7), True) self.assertEquals(not heartbeat.reports, True, "Heartbeat should not contain task in progress")
def start(self): self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.heartbeat = Heartbeat(self.actionQueue)
class Controller(threading.Thread): def __init__(self, config): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config #Disabled security until we have fix for AMBARI-157 #if(config.get('controller', 'user')!=None and config.get('controller', 'password')!=None): # self.credential = { 'user' : config.get('controller', 'user'), # 'password' : config.get('controller', 'password') # } self.url = config.get('controller', 'url') + '/agent/controller/heartbeat/' + socket.gethostname() def start(self): self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.heartbeat = Heartbeat(self.actionQueue) def __del__(self): logger.info("Controller connection disconnected.") def run(self): id='-1' if self.credential!=None: auth_handler = urllib2.HTTPBasicAuthHandler() auth_handler.add_password(realm="Controller", uri=self.url, user=self.credential['user'], passwd=self.credential['password']) opener = urllib2.build_opener(auth_handler) urllib2.install_opener(opener) retry=False firstTime=True while True: try: if retry==False: data = json.dumps(self.heartbeat.build(id)) logger.debug(data) req = urllib2.Request(self.url, data, {'Content-Type': 'application/json'}) f = urllib2.urlopen(req) response = f.read() f.close() data = json.loads(response) id=int(data['responseId']) self.actionQueue.put(data) if retry==True or firstTime==True: logger.info("Controller connection established") firstTime=False retry=False except Exception, err: retry=True if "code" in err: logger.error(err.code) else: logger.error("Unable to connect to: "+self.url,exc_info=True) if self.actionQueue.isIdle(): time.sleep(30) else: time.sleep(1)
def test_build_long_result(self, result_mock): config = AgentConfig("", "") config.set('agent', 'prefix', 'tmp') dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) result_mock.return_value = { 'reports': [{ 'status': 'IN_PROGRESS', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': True }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName', 'roleCommand': 'UPGRADE', 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17, 'taskId': 'taskId', 'exitcode': 0, 'reportResult': True }, { 'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 13, 'reportResult': True }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 0, 'reportResult': True }, { 'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'taskId': 3, 'exitcode': 0, 'reportResult': False }], 'componentStatus': [ { 'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult': True }, { 'status': 'UNHEALTHY', 'componentName': 'NAMENODE', 'reportResult': True }, { 'status': 'UNHEALTHY', 'componentName': 'HBASE_MASTER', 'reportResult': False }, ], } heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger) # State.STARTED results in agentState to be set to 4 (enum order) hb = heartbeat.build({}, 10) hb['hostname'] = 'hostname' hb['timestamp'] = 'timestamp' hb['fqdn'] = 'fqdn' expected = { 'nodeStatus': { 'status': 'HEALTHY', 'cause': 'NONE' }, 'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn', 'responseId': 10, 'reports': [{ 'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'Read from /tmp/errors-3.txt', 'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc', 'taskId': 3, 'exitcode': 777 }, { 'status': 'COMPLETED', 'roleCommand': 'UPGRADE', 'serviceName': 'serviceName', 'role': 'role', 'actionId': 17, 'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName', 'taskId': 'taskId', 'exitcode': 0 }, { 'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitcode': 13 }, { 'status': 'COMPLETED', 'stdout': 'out', 'configurationTags': { 'global': { 'tag': 'v1' } }, 'taskId': 3, 'exitcode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc', 'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr' }], 'componentStatus': [{ 'status': 'HEALTHY', 'componentName': 'DATANODE' }, { 'status': 'UNHEALTHY', 'componentName': 'NAMENODE' }] } self.assertEqual.__self__.maxDiff = None self.assertEquals(hb, expected)
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock, resolve_script_path_mock): tempdir = tempfile.gettempdir() config = MagicMock() config.get.return_value = "something" config.getResolvedPath.return_value = tempdir config.getWorkRootPath.return_value = tempdir config.getLogPath.return_value = tempdir # Make file read calls visible def open_side_effect(file, mode): if mode == 'r': file_mock = MagicMock() file_mock.read.return_value = "Read from " + str(file) return file_mock else: return self.original_open(file, mode) open_mock.side_effect = open_side_effect json_load_mock.return_value = '' resolve_script_path_mock.return_value = "abc.py" dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller) unfreeze_flag = threading.Event() python_execution_result_dict = { 'stdout': 'out', 'stderr': 'stderr', 'structuredOut': '' } def side_effect(py_file, script_params, tmpoutfile, tmperrfile, timeout, tmpstrucoutfile, loglevel, override_output_files, environment_vars): unfreeze_flag.wait() return python_execution_result_dict def patched_aq_execute_command(command): # We have to perform patching for separate thread in the same thread with patch.object(PythonExecutor, "run_file") as runCommand_mock: runCommand_mock.side_effect = side_effect actionQueue.execute_command(command) ### Test install/start/stop command ### ## Test successful execution with configuration tags python_execution_result_dict['status'] = 'COMPLETE' python_execution_result_dict['exitcode'] = 0 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() # check in progress report # wait until ready while True: time.sleep(0.1) report = actionQueue.result() if len(report['reports']) != 0: break expected = {'status': 'IN_PROGRESS', 'stderr': 'Read from {0}/errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}/output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777} self.assertEqual(report['reports'][0], expected) # Continue command execution unfreeze_flag.set() # wait until ready while report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report configname = os.path.join(tempdir, 'command-3.json') expected = {'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': {'global': {'tag': 'v1'}}, 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitcode': 0, 'allocatedPorts': {}, 'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) self.assertTrue(os.path.isfile(configname)) # Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE) self.assertEqual(status_update_callback_mock.call_count, 2) os.remove(configname) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0) ## Test failed execution python_execution_result_dict['status'] = 'FAILED' python_execution_result_dict['exitcode'] = 13 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() unfreeze_flag.set() # check in progress report # wait until ready report = actionQueue.result() while len(report['reports']) == 0 or \ report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report expected = {'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitcode': 13} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0)
def start(self): self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register() self.heartbeat = Heartbeat(self.actionQueue) pass
def main(argv=None): actionQueue = ActionQueue() heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build())
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock, resolve_script_path_mock): self.assertEqual.__self__.maxDiff = None tempdir = tempfile.gettempdir() config = MagicMock() config.get.return_value = "something" config.getResolvedPath.return_value = tempdir config.getWorkRootPath.return_value = tempdir config.getLogPath.return_value = tempdir # Make file read calls visible def open_side_effect(file, mode): if mode == 'r': file_mock = MagicMock() file_mock.read.return_value = "Read from " + str(file) return file_mock else: return self.original_open(file, mode) open_mock.side_effect = open_side_effect json_load_mock.return_value = '' resolve_script_path_mock.return_value = "abc.py" dummy_controller = MagicMock() actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger) unfreeze_flag = threading.Event() python_execution_result_dict = { 'stdout': 'out', 'stderr': 'stderr', 'structuredOut': '' } def side_effect(py_file, script_params, tmpoutfile, tmperrfile, timeout, tmpstrucoutfile, loglevel, override_output_files, environment_vars): unfreeze_flag.wait() return python_execution_result_dict def patched_aq_execute_command(command): # We have to perform patching for separate thread in the same thread with patch.object(PythonExecutor, "run_file") as runCommand_mock: runCommand_mock.side_effect = side_effect actionQueue.execute_command(command) ### Test install/start/stop command ### ## Test successful execution with configuration tags python_execution_result_dict['status'] = 'COMPLETE' python_execution_result_dict['exitcode'] = 0 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() # check in progress report # wait until ready while True: time.sleep(0.1) report = actionQueue.result() if len(report['reports']) != 0: break expected = {'status': 'IN_PROGRESS', 'stderr': 'Read from {0}/errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}/output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': True} if IS_WINDOWS: expected = {'status': 'IN_PROGRESS', 'stderr': 'Read from {0}\\errors-3.txt'.format(tempdir), 'stdout': 'Read from {0}\\output-3.txt'.format(tempdir), 'structuredOut': '', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'exitcode': 777, 'reportResult': True} self.assertEqual(report['reports'][0], expected) # Continue command execution unfreeze_flag.set() # wait until ready while report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report configname = os.path.join(tempdir, 'command-3.json') expected = {'status': 'COMPLETED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'configurationTags': {'global': {'tag': 'v1'}}, 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitcode': 0, 'allocatedPorts': {}, 'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}, 'reportResult': True} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) self.assertTrue(os.path.isfile(configname)) # Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE) self.assertEqual(status_update_callback_mock.call_count, 2) os.remove(configname) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0) ## Test failed execution python_execution_result_dict['status'] = 'FAILED' python_execution_result_dict['exitcode'] = 13 # We call method in a separate thread execution_thread = Thread(target=patched_aq_execute_command, args=(self.datanode_install_command, )) execution_thread.start() unfreeze_flag.set() # check in progress report # wait until ready report = actionQueue.result() while len(report['reports']) == 0 or \ report['reports'][0]['status'] == 'IN_PROGRESS': time.sleep(0.1) report = actionQueue.result() # check report expected = {'status': 'FAILED', 'stderr': 'stderr', 'stdout': 'out', 'clusterName': u'cc', 'roleCommand': u'INSTALL', 'serviceName': u'HBASE', 'role': u'HBASE_MASTER', 'actionId': '1-1', 'taskId': 3, 'structuredOut': '', 'exitcode': 13, 'reportResult': True} self.assertEqual(len(report['reports']), 1) self.assertEqual(report['reports'][0], expected) # now should not have reports (read complete/failed reports are deleted) report = actionQueue.result() self.assertEqual(len(report['reports']), 0)
class MyStrategy: game = None world = None me = None __weather_map = None __terrain_map = None vehicleById = {} updateTickByVehicleId = {} updatedVehicleXY = {} allVehicles = [] initializer = None actionQueue = None strategy = None _sos_mode = False def initialize_strategy(self): self.initializer = Initializer(self.world) self.actionQueue = ActionQueue(self.world) self.__weather_map = WeatherMap(self.world.weather_by_cell_x_y, Constant.WEATHER_MAP_CELL_SIZE) self.__terrain_map = TerrainMap(self.world.terrain_by_cell_x_y, Constant.TERRAIN_MAP_CELL_SIZE) def initialize_tick(self): self.updatedVehicleXY = {} for vehicle in self.world.new_vehicles: self.vehicleById[vehicle.id] = vehicle self.updateTickByVehicleId[vehicle.id] = self.world.tick_index for vehicleUpdate in self.world.vehicle_updates: vehicleId = vehicleUpdate.id if vehicleUpdate.durability == 0: self.vehicleById.pop(vehicleId) self.updateTickByVehicleId.pop(vehicleId) else: x_old, y_old = self.vehicleById[vehicleId].x, self.vehicleById[ vehicleId].y self.vehicleById[vehicleId].update(vehicleUpdate) x_new, y_new = self.vehicleById[vehicleId].x, self.vehicleById[ vehicleId].y if self.vehicleById[vehicleId].player_id == self.me.id and ( x_old != x_new or y_old != y_new): self.updatedVehicleXY[vehicleId] = (x_new - x_old, y_new - y_old) self.updateTickByVehicleId[vehicleId] = self.world.tick_index self.allVehicles = list(self.vehicleById.values()) def execute_action(self, move: Move, action: dict): for field, value in action.items(): setattr(move, field, value) return True def execute_delayed_action(self, move: Move, world: World): action = self.actionQueue.pop(world) if action is None: return False return self.execute_action(move, action) def move(self, me: Player, world: World, game: Game, move: Move): self.game = game self.world = world self.me = me if world.tick_index == 0: self.initialize_strategy() self.initialize_tick() if self.strategy is None: if self.initializer.current_step not in [ Initializer.STEP_STOP, Initializer.STEP_INTERRUPT_STOP ]: self.initializer.prepare_step(self.me, self.world, self.allVehicles) self.initializer.perform_current_step(self.actionQueue, self.updatedVehicleXY) elif self.initializer.current_step == Initializer.STEP_INTERRUPT_STOP: self.strategy = NaiveStrategy(self.actionQueue, self.world, self.__weather_map, self.__terrain_map) else: self.strategy = NaiveStrategy(self.actionQueue, self.world, self.__weather_map, self.__terrain_map) if self.strategy: #if False: if not self._sos_mode: if self.__save_our_souls(world): return else: if self.__comeback_after_strike(world): return if me.remaining_action_cooldown_ticks > 0: return if self.execute_delayed_action(move, world): return self.determine_following_actions() self.execute_delayed_action(move, world) def determine_following_actions(self): if self.strategy is None: return self.strategy.initialize_tick(self.game, self.world, self.me, self.allVehicles, self.updateTickByVehicleId) self.strategy.determine_following_actions(self.updatedVehicleXY) def __save_our_souls(self, world: World): opponent = world.get_opponent_player() next_nuclear_strike_tick = opponent.next_nuclear_strike_tick_index if next_nuclear_strike_tick == -1: return False f = Formation(self.allVehicles, self.me, ownership=Ownership.ALLY) x = opponent.next_nuclear_strike_x y = opponent.next_nuclear_strike_y next_nuclear_strike_vehicle_id = opponent.next_nuclear_strike_vehicle_id if next_nuclear_strike_vehicle_id == -1: return False if next_nuclear_strike_vehicle_id not in self.vehicleById: return False is_aerial = self.vehicleById[next_nuclear_strike_vehicle_id].aerial kill_factor = f.calc_nuclear_kill_factor(x, y) if kill_factor['total_damage'] < 700 and kill_factor[ 'survived'] > kill_factor['killed'] * 5 and not is_aerial: return False from_x, from_y = self.vehicleById[ next_nuclear_strike_vehicle_id].x, self.vehicleById[ next_nuclear_strike_vehicle_id].y sos_vector = (y - from_y, x - from_x) sos_vector_norm = 120 / math.hypot(sos_vector[0], sos_vector[1]) sos_vector = (sos_vector[0] * sos_vector_norm, sos_vector[1] * sos_vector_norm) topleft = f.find_topleft() bottomright = f.find_bottomright() if topleft[0] + sos_vector[0] < 0 or topleft[1] + sos_vector[1] < 0: sos_vector = (-sos_vector[0], -sos_vector[1]) elif bottomright[0] + sos_vector[0] >= self.world.height or bottomright[ 1] + sos_vector[1] >= self.world.width: sos_vector = (-sos_vector[0], -sos_vector[1]) self.actionQueue.clear() self.actionQueue.push(Action.move(sos_vector[1], sos_vector[0], 0.3)) self._sos_mode = True return True def __comeback_after_strike(self, world: World): opponent = world.get_opponent_player() next_nuclear_strike_tick = opponent.next_nuclear_strike_tick_index if next_nuclear_strike_tick != -1: return False if not self.actionQueue.is_action_in_queue(ActionType.MOVE): self.actionQueue.push(Action.move(0, 0)) else: self.actionQueue.clear() self._sos_mode = False return True
def main(argv=None): actionQueue = ActionQueue(AmbariConfig.config) heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build('3', 3))
class Controller(threading.Thread): def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get( 'server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config) def __del__(self): logger.info("Server connection disconnected.") pass def unregisterWithServer(self): id = -1 ret = self.retriedRequest(self.unregisterUrl, lambda: self.unregister.build(id)) if ret["response"] == "OK": logger.info("Unregistered with the server") print("Unregistered with the server") else: logMsg = "Error while unregistering with server: %s" % ret[ "errorMessage"] logger.info(logMsg) print(logMsg) return ret def registerWithServer(self): id = -1 ret = self.retriedRequest(self.registerUrl, lambda: self.register.build(id)) logger.info("Registered with the server") print("Registered with the server") return ret def retriedRequest(self, requestUrl, payloadBuilder): successfulRequest = False id = -1 ret = {} while not successfulRequest: try: data = json.dumps(payloadBuilder()) logger.info("Sending request to url " + requestUrl + " " + pprint.pformat(data)) response = self.sendRequest(requestUrl, data) ret = json.loads(response) logger.info("Request successful " + pprint.pformat(ret)) self.responseId = int(ret['responseId']) successfulRequest = True if 'statusCommands' in ret.keys(): logger.info("Got status commands on registration " + pprint.pformat(ret['statusCommands'])) self.addToQueue(ret['statusCommands']) pass else: self.hasMappedComponents = False pass except ssl.SSLError: self.repeatRegistration = False return except Exception, err: # try a reconnect only after a certain amount of random time delay = randint(0, self.range) logger.info("Unable to connect to: " + requestUrl, exc_info=True) """ Sleeping for {0} seconds and then retrying again """.format( delay) time.sleep(delay) pass pass return ret
class Controller(threading.Thread): def __init__(self, config, range=30): threading.Thread.__init__(self) logger.debug('Initializing Controller RPC thread.') self.lock = threading.Lock() self.safeMode = True self.credential = None self.config = config self.hostname = hostname.hostname() server_secured_url = 'https://' + config.get('server', 'hostname') + ':' + config.get('server', 'secured_url_port') self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname self.netutil = NetUtil() self.responseId = -1 self.repeatRegistration = False self.cachedconnect = None self.range = range self.hasMappedComponents = True self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.register = Register(self.config) self.unregister = Unregister(self.config) self.heartbeat = Heartbeat(self.actionQueue, self.config) def __del__(self): logger.info("Server connection disconnected.") pass def unregisterWithServer(self): id = -1 ret = self.retriedRequest(self.unregisterUrl, lambda: self.unregister.build(id)) if ret["response"] == "OK": logger.info("Unregistered with the server") print("Unregistered with the server") else: logMsg = "Error while unregistering with server: %s" % ret["errorMessage"] logger.info(logMsg) print(logMsg) return ret def registerWithServer(self): id = -1 ret = self.retriedRequest(self.registerUrl, lambda: self.register.build(id)) logger.info("Registered with the server") print("Registered with the server") return ret def retriedRequest(self, requestUrl, payloadBuilder): successfulRequest=False id = -1 ret = {} while not successfulRequest: try: data = json.dumps(payloadBuilder()) logger.info("Sending request to url " + requestUrl + " " + pprint.pformat(data)) response = self.sendRequest(requestUrl, data) ret = json.loads(response) logger.info("Request successful " + pprint.pformat(ret)) self.responseId= int(ret['responseId']) successfulRequest = True if 'statusCommands' in ret.keys(): logger.info("Got status commands on registration " + pprint.pformat(ret['statusCommands']) ) self.addToQueue(ret['statusCommands']) pass else: self.hasMappedComponents = False pass except ssl.SSLError: self.repeatRegistration=False return except Exception, err: # try a reconnect only after a certain amount of random time delay = randint(0, self.range) logger.info("Unable to connect to: " + requestUrl, exc_info = True) """ Sleeping for {0} seconds and then retrying again """.format(delay) time.sleep(delay) pass pass return ret
def main(argv=None): actionQueue = ActionQueue(AgentConfig.getConfig()) heartbeat = Heartbeat(actionQueue) print json.dumps(heartbeat.build('3', 3))
class ActionModel(QtCore.QAbstractTableModel): def __init__(self, parent, nao): super(ActionModel, self).__init__(parent) self._list = ActionQueue(nao) self._list.cleared.connect(self.on__list_cleared) self._list.dequeued.connect(self.on__list_dequeued) self._list.enqueued.connect(self.on__list_enqueued) self._list.removed.connect(self.on__list_dequeued) self._timer = ThreadTimer() self._timer.start() self._timer.addToThread(self._list) self._timer.timeElapsed.connect(self._list.execute) #END __init__() def dispose(self): self._timer.stop() #END dispose() def addActions(self, actions): self._list.enqueue(actions) #END addActions() def clearActions(self): self._list.clear() #END clearActions() def isRunning(self): return self._list.isRunning() #END isRunning() def removeAction(self, index): self._list.remove(index) #END removeAction() def setAutorun(self, value): self._list.setAutorun(value) #END setAutorun() def setRunning(self, value): self._list.setRunning(value) #END setRunning() # noinspection PyUnusedLocal def columnCount(self, parent): return 2 #END columnCount() def data(self, index, role): if role == QtCore.Qt.DisplayRole: item = self._list.get(index.row()) if index.column() == 0: return item.actionToString() else: return item.paramToString() #END if #END if return QtCore.QVariant() #END data() def headerData(self, section, orientation, role): if role == QtCore.Qt.DisplayRole: if orientation == QtCore.Qt.Horizontal: if section == 0: return "Action" else: return "Parameter" else: return section #END if #END if return QtCore.QVariant() #END headerData() # noinspection PyUnusedLocal def rowCount(self, parent): return self._list.length() #END rowCount() def on__list_cleared(self, length): parent = QtCore.QModelIndex() self.beginRemoveRows(parent, 0, length - 1) self.endRemoveRows() #END on__list_cleared() def on__list_dequeued(self, index): parent = QtCore.QModelIndex() self.beginRemoveRows(parent, index, index) self.endRemoveRows() #END on__list_dequeued() def on__list_enqueued(self, beginIndex, endIndex): parent = QtCore.QModelIndex() self.beginInsertRows(parent, beginIndex, endIndex) self.endInsertRows() #END on__list_enqueued() #END ActionModel.py