def __init__(self, config, range=30):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
self.hostname = hostname.hostname()
server_secured_url = 'https://' + config.get(
'server', 'hostname') + ':' + config.get('server',
'secured_url_port')
self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname
self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname
self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname
self.netutil = NetUtil()
self.responseId = -1
self.repeatRegistration = False
self.cachedconnect = None
self.range = range
self.hasMappedComponents = True
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register(self.config)
self.unregister = Unregister(self.config)
self.heartbeat = Heartbeat(self.actionQueue, self.config)
def initialize_strategy(self):
self.initializer = Initializer(self.world)
self.actionQueue = ActionQueue(self.world)
self.__weather_map = WeatherMap(self.world.weather_by_cell_x_y,
Constant.WEATHER_MAP_CELL_SIZE)
self.__terrain_map = TerrainMap(self.world.terrain_by_cell_x_y,
Constant.TERRAIN_MAP_CELL_SIZE)
class Controller(threading.Thread):
def __init__(self, config):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
self.hostname = socket.gethostname()
server_secured_url = 'https://' + config.get(
'server', 'hostname') + ':' + config.get('server',
'secured_url_port')
self.registerUrl = server_secured_url + '/agent/register/' + self.hostname
self.heartbeatUrl = server_secured_url + '/agent/heartbeat/' + self.hostname
self.netutil = NetUtil()
def start(self):
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register()
self.heartbeat = Heartbeat(self.actionQueue)
pass
def __del__(self):
logger.info("Server connection disconnected.")
pass
def registerWithServer(self):
retry = False
firstTime = True
registered = False
id = -1
ret = {}
while registered == False:
try:
data = json.dumps(self.register.build(id))
logger.info("Registering with the server " +
pprint.pformat(data))
req = urllib2.Request(self.registerUrl, data,
{'Content-Type': 'application/json'})
stream = security.secured_url_open(req)
response = stream.read()
stream.close()
ret = json.loads(response)
logger.info("Registered with the server with " +
pprint.pformat(ret))
registered = True
pass
except Exception, err:
delay = self.netutil.CONNECT_SERVER_RETRY_INTERVAL_SEC
logger.info("Unable to connect to: " + self.registerUrl,
exc_info=True)
""" Sleeping for {0} seconds and then retrying again """.format(
delay)
time.sleep(delay)
pass
pass
return ret
def perform_current_step(self, action_queue: ActionQueue,
updated_vehicle_x_y):
# Двигаем юниты на 1 линию
if self.current_step == self.STEP_MOVE_TO_INITIAL_POSITIONS:
if not self.__check_step_started():
command_builder = InitializerStep0_MoveToInitialPositions(
self.__me, self.__weather_map, self.__terrain_map,
self.__all_vehicles)
commands = command_builder.get_command_list()
if len(commands) == 0:
if self.__check_interrupt_condition():
self.__set_step(self.STEP_INTERRUPT_CONSOLIDATE)
else:
self.__set_step()
return
for command in commands:
action_queue.push(command)
self.__mark_step_started()
if self.__check_move_finished(updated_vehicle_x_y):
self.__mark_move_started(False)
self.__mark_step_started(False)
# Расширяем каждый столбец юнитов в 3 раза
elif self.current_step == self.STEP_UNCONSOLIDATE:
self.__perform_casual_step(InitializerStep1_Unconsolidate,
action_queue, updated_vehicle_x_y)
# Делаем вертикальный сдвиг на разное смещение разных типов юнитов
elif self.current_step == self.STEP_SHIFT_VERTICAL:
self.__perform_casual_step(InitializerStep2_ShiftVertical,
action_queue, updated_vehicle_x_y)
# Сдвигаем юнитов в одну колонку
elif self.current_step == self.STEP_SHIFT_TOGETHER:
self.__perform_casual_step(InitializerStep3_ShiftTogether,
action_queue, updated_vehicle_x_y)
# Сдвигаем группы по 3 через 3
elif self.current_step == self.STEP_SHIFT_HORIZONTAL:
self.__perform_casual_step(InitializerStep4_ShiftHorizontal,
action_queue, updated_vehicle_x_y)
# Формируем длинные ряды
elif self.current_step == self.STEP_MAKE_LONG_ROWS:
self.__perform_casual_step(InitializerStep5_MakeLongRows,
action_queue, updated_vehicle_x_y)
# Поворачиваемся
elif self.current_step == self.STEP_ROTATE:
self.__perform_casual_step(InitializerStep6_Rotate, action_queue,
updated_vehicle_x_y)
elif self.current_step == self.STEP_INTERRUPT_CONSOLIDATE:
self.__perform_casual_step(InitializerStep8_InterruptConsolidate,
action_queue, updated_vehicle_x_y)
elif self.current_step == self.STEP_INTERRUPT_SHIFT_AIR_UNITS:
self.__perform_casual_step(InitializerStep9_InterruptShiftAirUnits,
action_queue, updated_vehicle_x_y)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock,
get_mock, process_command_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
actionQueue.start()
time.sleep(3)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
self.assertTrue(process_command_mock.call_count > 1)
def __perform_casual_step(self, command_builder_class,
action_queue: ActionQueue, updated_vehicle_x_y):
if not self.__check_step_started():
command_builder = command_builder_class(self.__me,
self.__weather_map,
self.__terrain_map,
self.__all_vehicles)
commands = command_builder.get_command_list()
if len(commands) == 0:
self.__set_step()
return
for command in commands:
action_queue.push(command)
self.__mark_step_started()
elif self.__check_move_finished(updated_vehicle_x_y):
self.__set_step()
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
execute_command_mock,
requestComponentStatus_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'}
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["status"], expected)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertTrue(requestComponentStatus_mock.called)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
execute_command_mock,
requestComponentStatus_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'}
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["status"], expected)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertTrue(requestComponentStatus_mock.called)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock,
get_mock, process_command_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.start()
time.sleep(3)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock,
execute_command_mock,
runCommand_mock,
status_update_callback):
csoMocks = [MagicMock()]
CustomServiceOrchestrator_mock.side_effect = csoMocks
csoMocks[0].status_commands_stdout = None
csoMocks[0].status_commands_stderr = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
runCommand_mock.return_value = {'configurations': {}}
actionQueue.execute_status_command(self.status_command_with_config)
report = actionQueue.result()
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertEqual(report['componentStatus'][0]["configurations"], {})
self.assertFalse(runCommand_mock.called)
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock,
execute_command_mock,
runCommand_mock,
status_update_callback):
csoMocks = [MagicMock()]
CustomServiceOrchestrator_mock.side_effect = csoMocks
csoMocks[0].status_commands_stdout = None
csoMocks[0].status_commands_stderr = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
runCommand_mock.return_value = {'configurations': {}}
actionQueue.execute_status_command(self.status_command_with_config)
report = actionQueue.result()
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertEqual(report['componentStatus'][0]["configurations"], {})
self.assertFalse(runCommand_mock.called)
def __init__(self, parent, nao):
super(ActionModel, self).__init__(parent)
self._list = ActionQueue(nao)
self._list.cleared.connect(self.on__list_cleared)
self._list.dequeued.connect(self.on__list_dequeued)
self._list.enqueued.connect(self.on__list_enqueued)
self._list.removed.connect(self.on__list_dequeued)
self._timer = ThreadTimer()
self._timer.start()
self._timer.addToThread(self._list)
self._timer.timeElapsed.connect(self._list.execute)
def test_process_command(self,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
wrong_command = {
'commandType': "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
print_exc_mock.reset_mock()
execute_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
def test_process_command(self,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
wrong_command = {
'commandType': "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
print_exc_mock.reset_mock()
execute_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
def test_build_result3(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller,
self.agentToggleLogger)
result_mock.return_value = {
'reports': [{
'status': 'COMPLETED',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': False
}],
'componentStatus': []
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
commandResult = {}
hb = heartbeat.build(commandResult, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {
'package': '',
'nodeStatus': {
'status': 'HEALTHY',
'cause': 'NONE'
},
'timestamp': 'timestamp',
'hostname': 'hostname',
'fqdn': 'fqdn',
'responseId': 10,
'reports': []
}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
def __init__(self, config, range=30):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
self.hostname = hostname.hostname()
server_secured_url = 'https://' + config.get('server', 'hostname') + ':' + config.get('server', 'secured_url_port')
self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname
self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname
self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname
self.netutil = NetUtil()
self.responseId = -1
self.repeatRegistration = False
self.cachedconnect = None
self.range = range
self.hasMappedComponents = True
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register(self.config)
self.unregister = Unregister(self.config)
self.heartbeat = Heartbeat(self.actionQueue, self.config)
def test_build(self):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue, config)
result = heartbeat.build({}, 100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
"hostname should not be empty")
self.assertEquals(result['responseId'], 100)
self.assertEquals('componentStatus' not in result, True,
"Heartbeat should contain componentStatus")
self.assertEquals(result['reports'] is not None, True,
"Heartbeat should contain reports")
self.assertEquals(result['timestamp'] >= 1353679373880L, True)
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 6) or (len(result) is 7), True)
self.assertEquals(not heartbeat.reports, True,
"Heartbeat should not contain task in progress")
def start(self): self.actionQueue = ActionQueue(self.config) self.actionQueue.start() self.heartbeat = Heartbeat(self.actionQueue)
class Controller(threading.Thread):
def __init__(self, config):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
#Disabled security until we have fix for AMBARI-157
#if(config.get('controller', 'user')!=None and config.get('controller', 'password')!=None):
# self.credential = { 'user' : config.get('controller', 'user'),
# 'password' : config.get('controller', 'password')
# }
self.url = config.get('controller', 'url') + '/agent/controller/heartbeat/' + socket.gethostname()
def start(self):
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.heartbeat = Heartbeat(self.actionQueue)
def __del__(self):
logger.info("Controller connection disconnected.")
def run(self):
id='-1'
if self.credential!=None:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(realm="Controller",
uri=self.url,
user=self.credential['user'],
passwd=self.credential['password'])
opener = urllib2.build_opener(auth_handler)
urllib2.install_opener(opener)
retry=False
firstTime=True
while True:
try:
if retry==False:
data = json.dumps(self.heartbeat.build(id))
logger.debug(data)
req = urllib2.Request(self.url, data, {'Content-Type': 'application/json'})
f = urllib2.urlopen(req)
response = f.read()
f.close()
data = json.loads(response)
id=int(data['responseId'])
self.actionQueue.put(data)
if retry==True or firstTime==True:
logger.info("Controller connection established")
firstTime=False
retry=False
except Exception, err:
retry=True
if "code" in err:
logger.error(err.code)
else:
logger.error("Unable to connect to: "+self.url,exc_info=True)
if self.actionQueue.isIdle():
time.sleep(30)
else:
time.sleep(1)
def test_build_long_result(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller,
self.agentToggleLogger)
result_mock.return_value = {
'reports': [{
'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitcode': 0,
'reportResult': True
}, {
'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 13,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0,
'reportResult': False
}],
'componentStatus': [
{
'status': 'HEALTHY',
'componentName': 'DATANODE',
'reportResult': True
},
{
'status': 'UNHEALTHY',
'componentName': 'NAMENODE',
'reportResult': True
},
{
'status': 'UNHEALTHY',
'componentName': 'HBASE_MASTER',
'reportResult': False
},
],
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
# State.STARTED results in agentState to be set to 4 (enum order)
hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {
'nodeStatus': {
'status': 'HEALTHY',
'cause': 'NONE'
},
'timestamp':
'timestamp',
'hostname':
'hostname',
'fqdn':
'fqdn',
'responseId':
10,
'reports': [{
'status': 'IN_PROGRESS',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'taskId': 3,
'exitcode': 777
}, {
'status': 'COMPLETED',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'taskId': 'taskId',
'exitcode': 0
}, {
'status': 'FAILED',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'taskId': 3,
'exitcode': 13
}, {
'status': 'COMPLETED',
'stdout': 'out',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'taskId': 3,
'exitcode': 0,
'roleCommand': u'INSTALL',
'clusterName': u'cc',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'stderr'
}],
'componentStatus': [{
'status': 'HEALTHY',
'componentName': 'DATANODE'
}, {
'status': 'UNHEALTHY',
'componentName': 'NAMENODE'
}]
}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock,
resolve_script_path_mock):
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
json_load_mock.return_value = ''
resolve_script_path_mock.return_value = "abc.py"
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
loglevel,
override_output_files,
environment_vars):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(PythonExecutor, "run_file") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
report = actionQueue.result()
if len(report['reports']) != 0:
break
expected = {'status': 'IN_PROGRESS',
'stderr': 'Read from {0}/errors-3.txt'.format(tempdir),
'stdout': 'Read from {0}/output-3.txt'.format(tempdir),
'structuredOut': '',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777}
self.assertEqual(report['reports'][0], expected)
# Continue command execution
unfreeze_flag.set()
# wait until ready
while report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
configname = os.path.join(tempdir, 'command-3.json')
expected = {'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {'global': {'tag': 'v1'}},
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
'exitcode': 0,
'allocatedPorts': {},
'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir}}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
# Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
self.assertEqual(status_update_callback_mock.call_count, 2)
os.remove(configname)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
'exitcode': 13}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
def start(self):
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register()
self.heartbeat = Heartbeat(self.actionQueue)
pass
def main(argv=None):
actionQueue = ActionQueue()
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build())
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock,
resolve_script_path_mock):
self.assertEqual.__self__.maxDiff = None
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
json_load_mock.return_value = ''
resolve_script_path_mock.return_value = "abc.py"
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
loglevel,
override_output_files,
environment_vars):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(PythonExecutor, "run_file") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
report = actionQueue.result()
if len(report['reports']) != 0:
break
expected = {'status': 'IN_PROGRESS',
'stderr': 'Read from {0}/errors-3.txt'.format(tempdir),
'stdout': 'Read from {0}/output-3.txt'.format(tempdir),
'structuredOut': '',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True}
if IS_WINDOWS:
expected = {'status': 'IN_PROGRESS',
'stderr': 'Read from {0}\\errors-3.txt'.format(tempdir),
'stdout': 'Read from {0}\\output-3.txt'.format(tempdir),
'structuredOut': '',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True}
self.assertEqual(report['reports'][0], expected)
# Continue command execution
unfreeze_flag.set()
# wait until ready
while report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
configname = os.path.join(tempdir, 'command-3.json')
expected = {'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {'global': {'tag': 'v1'}},
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
'exitcode': 0,
'allocatedPorts': {},
'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir},
'reportResult': True}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
# Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
self.assertEqual(status_update_callback_mock.call_count, 2)
os.remove(configname)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
'exitcode': 13,
'reportResult': True}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
class MyStrategy:
game = None
world = None
me = None
__weather_map = None
__terrain_map = None
vehicleById = {}
updateTickByVehicleId = {}
updatedVehicleXY = {}
allVehicles = []
initializer = None
actionQueue = None
strategy = None
_sos_mode = False
def initialize_strategy(self):
self.initializer = Initializer(self.world)
self.actionQueue = ActionQueue(self.world)
self.__weather_map = WeatherMap(self.world.weather_by_cell_x_y,
Constant.WEATHER_MAP_CELL_SIZE)
self.__terrain_map = TerrainMap(self.world.terrain_by_cell_x_y,
Constant.TERRAIN_MAP_CELL_SIZE)
def initialize_tick(self):
self.updatedVehicleXY = {}
for vehicle in self.world.new_vehicles:
self.vehicleById[vehicle.id] = vehicle
self.updateTickByVehicleId[vehicle.id] = self.world.tick_index
for vehicleUpdate in self.world.vehicle_updates:
vehicleId = vehicleUpdate.id
if vehicleUpdate.durability == 0:
self.vehicleById.pop(vehicleId)
self.updateTickByVehicleId.pop(vehicleId)
else:
x_old, y_old = self.vehicleById[vehicleId].x, self.vehicleById[
vehicleId].y
self.vehicleById[vehicleId].update(vehicleUpdate)
x_new, y_new = self.vehicleById[vehicleId].x, self.vehicleById[
vehicleId].y
if self.vehicleById[vehicleId].player_id == self.me.id and (
x_old != x_new or y_old != y_new):
self.updatedVehicleXY[vehicleId] = (x_new - x_old,
y_new - y_old)
self.updateTickByVehicleId[vehicleId] = self.world.tick_index
self.allVehicles = list(self.vehicleById.values())
def execute_action(self, move: Move, action: dict):
for field, value in action.items():
setattr(move, field, value)
return True
def execute_delayed_action(self, move: Move, world: World):
action = self.actionQueue.pop(world)
if action is None:
return False
return self.execute_action(move, action)
def move(self, me: Player, world: World, game: Game, move: Move):
self.game = game
self.world = world
self.me = me
if world.tick_index == 0:
self.initialize_strategy()
self.initialize_tick()
if self.strategy is None:
if self.initializer.current_step not in [
Initializer.STEP_STOP, Initializer.STEP_INTERRUPT_STOP
]:
self.initializer.prepare_step(self.me, self.world,
self.allVehicles)
self.initializer.perform_current_step(self.actionQueue,
self.updatedVehicleXY)
elif self.initializer.current_step == Initializer.STEP_INTERRUPT_STOP:
self.strategy = NaiveStrategy(self.actionQueue, self.world,
self.__weather_map,
self.__terrain_map)
else:
self.strategy = NaiveStrategy(self.actionQueue, self.world,
self.__weather_map,
self.__terrain_map)
if self.strategy:
#if False:
if not self._sos_mode:
if self.__save_our_souls(world):
return
else:
if self.__comeback_after_strike(world):
return
if me.remaining_action_cooldown_ticks > 0:
return
if self.execute_delayed_action(move, world):
return
self.determine_following_actions()
self.execute_delayed_action(move, world)
def determine_following_actions(self):
if self.strategy is None:
return
self.strategy.initialize_tick(self.game, self.world, self.me,
self.allVehicles,
self.updateTickByVehicleId)
self.strategy.determine_following_actions(self.updatedVehicleXY)
def __save_our_souls(self, world: World):
opponent = world.get_opponent_player()
next_nuclear_strike_tick = opponent.next_nuclear_strike_tick_index
if next_nuclear_strike_tick == -1:
return False
f = Formation(self.allVehicles, self.me, ownership=Ownership.ALLY)
x = opponent.next_nuclear_strike_x
y = opponent.next_nuclear_strike_y
next_nuclear_strike_vehicle_id = opponent.next_nuclear_strike_vehicle_id
if next_nuclear_strike_vehicle_id == -1:
return False
if next_nuclear_strike_vehicle_id not in self.vehicleById:
return False
is_aerial = self.vehicleById[next_nuclear_strike_vehicle_id].aerial
kill_factor = f.calc_nuclear_kill_factor(x, y)
if kill_factor['total_damage'] < 700 and kill_factor[
'survived'] > kill_factor['killed'] * 5 and not is_aerial:
return False
from_x, from_y = self.vehicleById[
next_nuclear_strike_vehicle_id].x, self.vehicleById[
next_nuclear_strike_vehicle_id].y
sos_vector = (y - from_y, x - from_x)
sos_vector_norm = 120 / math.hypot(sos_vector[0], sos_vector[1])
sos_vector = (sos_vector[0] * sos_vector_norm,
sos_vector[1] * sos_vector_norm)
topleft = f.find_topleft()
bottomright = f.find_bottomright()
if topleft[0] + sos_vector[0] < 0 or topleft[1] + sos_vector[1] < 0:
sos_vector = (-sos_vector[0], -sos_vector[1])
elif bottomright[0] + sos_vector[0] >= self.world.height or bottomright[
1] + sos_vector[1] >= self.world.width:
sos_vector = (-sos_vector[0], -sos_vector[1])
self.actionQueue.clear()
self.actionQueue.push(Action.move(sos_vector[1], sos_vector[0], 0.3))
self._sos_mode = True
return True
def __comeback_after_strike(self, world: World):
opponent = world.get_opponent_player()
next_nuclear_strike_tick = opponent.next_nuclear_strike_tick_index
if next_nuclear_strike_tick != -1:
return False
if not self.actionQueue.is_action_in_queue(ActionType.MOVE):
self.actionQueue.push(Action.move(0, 0))
else:
self.actionQueue.clear()
self._sos_mode = False
return True
def main(argv=None):
actionQueue = ActionQueue(AmbariConfig.config)
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3', 3))
class Controller(threading.Thread):
def __init__(self, config, range=30):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
self.hostname = hostname.hostname()
server_secured_url = 'https://' + config.get(
'server', 'hostname') + ':' + config.get('server',
'secured_url_port')
self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname
self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname
self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname
self.netutil = NetUtil()
self.responseId = -1
self.repeatRegistration = False
self.cachedconnect = None
self.range = range
self.hasMappedComponents = True
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register(self.config)
self.unregister = Unregister(self.config)
self.heartbeat = Heartbeat(self.actionQueue, self.config)
def __del__(self):
logger.info("Server connection disconnected.")
pass
def unregisterWithServer(self):
id = -1
ret = self.retriedRequest(self.unregisterUrl,
lambda: self.unregister.build(id))
if ret["response"] == "OK":
logger.info("Unregistered with the server")
print("Unregistered with the server")
else:
logMsg = "Error while unregistering with server: %s" % ret[
"errorMessage"]
logger.info(logMsg)
print(logMsg)
return ret
def registerWithServer(self):
id = -1
ret = self.retriedRequest(self.registerUrl,
lambda: self.register.build(id))
logger.info("Registered with the server")
print("Registered with the server")
return ret
def retriedRequest(self, requestUrl, payloadBuilder):
successfulRequest = False
id = -1
ret = {}
while not successfulRequest:
try:
data = json.dumps(payloadBuilder())
logger.info("Sending request to url " + requestUrl + " " +
pprint.pformat(data))
response = self.sendRequest(requestUrl, data)
ret = json.loads(response)
logger.info("Request successful " + pprint.pformat(ret))
self.responseId = int(ret['responseId'])
successfulRequest = True
if 'statusCommands' in ret.keys():
logger.info("Got status commands on registration " +
pprint.pformat(ret['statusCommands']))
self.addToQueue(ret['statusCommands'])
pass
else:
self.hasMappedComponents = False
pass
except ssl.SSLError:
self.repeatRegistration = False
return
except Exception, err:
# try a reconnect only after a certain amount of random time
delay = randint(0, self.range)
logger.info("Unable to connect to: " + requestUrl,
exc_info=True)
""" Sleeping for {0} seconds and then retrying again """.format(
delay)
time.sleep(delay)
pass
pass
return ret
class Controller(threading.Thread):
def __init__(self, config, range=30):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
self.hostname = hostname.hostname()
server_secured_url = 'https://' + config.get('server', 'hostname') + ':' + config.get('server', 'secured_url_port')
self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname
self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname
self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname
self.netutil = NetUtil()
self.responseId = -1
self.repeatRegistration = False
self.cachedconnect = None
self.range = range
self.hasMappedComponents = True
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register(self.config)
self.unregister = Unregister(self.config)
self.heartbeat = Heartbeat(self.actionQueue, self.config)
def __del__(self):
logger.info("Server connection disconnected.")
pass
def unregisterWithServer(self):
id = -1
ret = self.retriedRequest(self.unregisterUrl, lambda: self.unregister.build(id))
if ret["response"] == "OK":
logger.info("Unregistered with the server")
print("Unregistered with the server")
else:
logMsg = "Error while unregistering with server: %s" % ret["errorMessage"]
logger.info(logMsg)
print(logMsg)
return ret
def registerWithServer(self):
id = -1
ret = self.retriedRequest(self.registerUrl, lambda: self.register.build(id))
logger.info("Registered with the server")
print("Registered with the server")
return ret
def retriedRequest(self, requestUrl, payloadBuilder):
successfulRequest=False
id = -1
ret = {}
while not successfulRequest:
try:
data = json.dumps(payloadBuilder())
logger.info("Sending request to url " + requestUrl + " " + pprint.pformat(data))
response = self.sendRequest(requestUrl, data)
ret = json.loads(response)
logger.info("Request successful " + pprint.pformat(ret))
self.responseId= int(ret['responseId'])
successfulRequest = True
if 'statusCommands' in ret.keys():
logger.info("Got status commands on registration " + pprint.pformat(ret['statusCommands']) )
self.addToQueue(ret['statusCommands'])
pass
else:
self.hasMappedComponents = False
pass
except ssl.SSLError:
self.repeatRegistration=False
return
except Exception, err:
# try a reconnect only after a certain amount of random time
delay = randint(0, self.range)
logger.info("Unable to connect to: " + requestUrl, exc_info = True)
""" Sleeping for {0} seconds and then retrying again """.format(delay)
time.sleep(delay)
pass
pass
return ret
def main(argv=None):
actionQueue = ActionQueue(AgentConfig.getConfig())
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3', 3))
class ActionModel(QtCore.QAbstractTableModel):
def __init__(self, parent, nao):
super(ActionModel, self).__init__(parent)
self._list = ActionQueue(nao)
self._list.cleared.connect(self.on__list_cleared)
self._list.dequeued.connect(self.on__list_dequeued)
self._list.enqueued.connect(self.on__list_enqueued)
self._list.removed.connect(self.on__list_dequeued)
self._timer = ThreadTimer()
self._timer.start()
self._timer.addToThread(self._list)
self._timer.timeElapsed.connect(self._list.execute)
#END __init__()
def dispose(self):
self._timer.stop()
#END dispose()
def addActions(self, actions):
self._list.enqueue(actions)
#END addActions()
def clearActions(self):
self._list.clear()
#END clearActions()
def isRunning(self):
return self._list.isRunning()
#END isRunning()
def removeAction(self, index):
self._list.remove(index)
#END removeAction()
def setAutorun(self, value):
self._list.setAutorun(value)
#END setAutorun()
def setRunning(self, value):
self._list.setRunning(value)
#END setRunning()
# noinspection PyUnusedLocal
def columnCount(self, parent):
return 2
#END columnCount()
def data(self, index, role):
if role == QtCore.Qt.DisplayRole:
item = self._list.get(index.row())
if index.column() == 0:
return item.actionToString()
else:
return item.paramToString()
#END if
#END if
return QtCore.QVariant()
#END data()
def headerData(self, section, orientation, role):
if role == QtCore.Qt.DisplayRole:
if orientation == QtCore.Qt.Horizontal:
if section == 0:
return "Action"
else:
return "Parameter"
else:
return section
#END if
#END if
return QtCore.QVariant()
#END headerData()
# noinspection PyUnusedLocal
def rowCount(self, parent):
return self._list.length()
#END rowCount()
def on__list_cleared(self, length):
parent = QtCore.QModelIndex()
self.beginRemoveRows(parent, 0, length - 1)
self.endRemoveRows()
#END on__list_cleared()
def on__list_dequeued(self, index):
parent = QtCore.QModelIndex()
self.beginRemoveRows(parent, index, index)
self.endRemoveRows()
#END on__list_dequeued()
def on__list_enqueued(self, beginIndex, endIndex):
parent = QtCore.QModelIndex()
self.beginInsertRows(parent, beginIndex, endIndex)
self.endInsertRows()
#END on__list_enqueued()
#END ActionModel.py
