def initialize_strategy(self):
self.initializer = Initializer(self.world)
self.actionQueue = ActionQueue(self.world)
self.__weather_map = WeatherMap(self.world.weather_by_cell_x_y,
Constant.WEATHER_MAP_CELL_SIZE)
self.__terrain_map = TerrainMap(self.world.terrain_by_cell_x_y,
Constant.TERRAIN_MAP_CELL_SIZE)
def __init__(self, config, range=30):
threading.Thread.__init__(self)
logger.debug('Initializing Controller RPC thread.')
self.lock = threading.Lock()
self.safeMode = True
self.credential = None
self.config = config
self.hostname = hostname.hostname()
server_secured_url = 'https://' + config.get(
'server', 'hostname') + ':' + config.get('server',
'secured_url_port')
self.registerUrl = server_secured_url + '/agent/v1/register/' + self.hostname
self.unregisterUrl = server_secured_url + '/agent/v1/unregister/' + self.hostname
self.heartbeatUrl = server_secured_url + '/agent/v1/heartbeat/' + self.hostname
self.netutil = NetUtil()
self.responseId = -1
self.repeatRegistration = False
self.cachedconnect = None
self.range = range
self.hasMappedComponents = True
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register(self.config)
self.unregister = Unregister(self.config)
self.heartbeat = Heartbeat(self.actionQueue, self.config)
def test_process_command2(self, execute_status_command_mock,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType': ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType': "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
actionQueue.process_command(status_command)
self.assertFalse(execute_command_mock.called)
self.assertTrue(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
execute_status_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock,
get_mock, process_command_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
actionQueue.start()
time.sleep(3)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
execute_command_mock,
requestComponentStatus_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'}
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["status"], expected)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertTrue(requestComponentStatus_mock.called)
def test_build_result3(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller,
self.agentToggleLogger)
result_mock.return_value = {
'reports': [{
'status': 'COMPLETED',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': False
}],
'componentStatus': []
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
commandResult = {}
hb = heartbeat.build(commandResult, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {
'package': '',
'nodeStatus': {
'status': 'HEALTHY',
'cause': 'NONE'
},
'timestamp': 'timestamp',
'hostname': 'hostname',
'fqdn': 'fqdn',
'responseId': 10,
'reports': []
}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock,
execute_command_mock,
runCommand_mock,
status_update_callback):
csoMocks = [MagicMock()]
CustomServiceOrchestrator_mock.side_effect = csoMocks
csoMocks[0].status_commands_stdout = None
csoMocks[0].status_commands_stderr = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
runCommand_mock.return_value = {'configurations': {}}
actionQueue.execute_status_command(self.status_command_with_config)
report = actionQueue.result()
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertEqual(report['componentStatus'][0]["configurations"], {})
self.assertFalse(runCommand_mock.called)
def test_build(self):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue, config)
result = heartbeat.build({}, 100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
"hostname should not be empty")
self.assertEquals(result['responseId'], 100)
self.assertEquals('componentStatus' not in result, True,
"Heartbeat should contain componentStatus")
self.assertEquals(result['reports'] is not None, True,
"Heartbeat should contain reports")
self.assertEquals(result['timestamp'] >= 1353679373880L, True)
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 6) or (len(result) is 7), True)
self.assertEquals(not heartbeat.reports, True,
"Heartbeat should not contain task in progress")
def main(argv=None):
actionQueue = ActionQueue(AgentConfig.getConfig())
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3', 3))
def main(argv=None):
actionQueue = ActionQueue(AmbariConfig.config)
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3', 3))
def test_build_long_result(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller,
self.agentToggleLogger)
result_mock.return_value = {
'reports': [{
'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitcode': 0,
'reportResult': True
}, {
'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 13,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0,
'reportResult': False
}],
'componentStatus': [
{
'status': 'HEALTHY',
'componentName': 'DATANODE',
'reportResult': True
},
{
'status': 'UNHEALTHY',
'componentName': 'NAMENODE',
'reportResult': True
},
{
'status': 'UNHEALTHY',
'componentName': 'HBASE_MASTER',
'reportResult': False
},
],
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
# State.STARTED results in agentState to be set to 4 (enum order)
hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {
'nodeStatus': {
'status': 'HEALTHY',
'cause': 'NONE'
},
'timestamp':
'timestamp',
'hostname':
'hostname',
'fqdn':
'fqdn',
'responseId':
10,
'reports': [{
'status': 'IN_PROGRESS',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'taskId': 3,
'exitcode': 777
}, {
'status': 'COMPLETED',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'taskId': 'taskId',
'exitcode': 0
}, {
'status': 'FAILED',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'taskId': 3,
'exitcode': 13
}, {
'status': 'COMPLETED',
'stdout': 'out',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'taskId': 3,
'exitcode': 0,
'roleCommand': u'INSTALL',
'clusterName': u'cc',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'stderr'
}],
'componentStatus': [{
'status': 'HEALTHY',
'componentName': 'DATANODE'
}, {
'status': 'UNHEALTHY',
'componentName': 'NAMENODE'
}]
}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
def main(argv=None):
actionQueue = ActionQueue()
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build())
def test_execute_command(self, status_update_callback_mock, open_mock, json_load_mock,
resolve_script_path_mock):
self.assertEqual.__self__.maxDiff = None
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
json_load_mock.return_value = ''
resolve_script_path_mock.return_value = "abc.py"
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': ''
}
def side_effect(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
loglevel,
override_output_files,
environment_vars):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(PythonExecutor, "run_file") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
report = actionQueue.result()
if len(report['reports']) != 0:
break
expected = {'status': 'IN_PROGRESS',
'stderr': 'Read from {0}/errors-3.txt'.format(tempdir),
'stdout': 'Read from {0}/output-3.txt'.format(tempdir),
'structuredOut': '',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True}
if IS_WINDOWS:
expected = {'status': 'IN_PROGRESS',
'stderr': 'Read from {0}\\errors-3.txt'.format(tempdir),
'stdout': 'Read from {0}\\output-3.txt'.format(tempdir),
'structuredOut': '',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True}
self.assertEqual(report['reports'][0], expected)
# Continue command execution
unfreeze_flag.set()
# wait until ready
while report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
configname = os.path.join(tempdir, 'command-3.json')
expected = {'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {'global': {'tag': 'v1'}},
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
'exitcode': 0,
'allocatedPorts': {},
'folders': {'AGENT_LOG_ROOT': tempdir, 'AGENT_WORK_ROOT': tempdir},
'reportResult': True}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
# Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
self.assertEqual(status_update_callback_mock.call_count, 2)
os.remove(configname)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target=patched_aq_execute_command,
args=(self.datanode_install_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HBASE',
'role': u'HBASE_MASTER',
'actionId': '1-1',
'taskId': 3,
'structuredOut': '',
'exitcode': 13,
'reportResult': True}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
def start(self):
self.actionQueue = ActionQueue(self.config)
self.actionQueue.start()
self.register = Register()
self.heartbeat = Heartbeat(self.actionQueue)
pass
