def test_registration_build(self):
tmpdir = tempfile.gettempdir()
ver_dir = os.path.join(tmpdir, "infra")
app_ver = "1.0.0"
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, {}, app_ver, "tag", 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(data['label'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(data['agentVersion'], '1', "agentVersion should not be empty")
self.assertEquals(data['actualState'], State.INIT, "actualState should not be empty")
self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty")
self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty")
self.assertEquals(data['logFolders'], {}, "allocated log should be empty")
self.assertEquals(data['appVersion'], app_ver, "app version should match")
self.assertEquals(data['tags'], "tag", "tags should be tag")
self.assertEquals(len(data), 11)
self.assertEquals(posixpath.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
self.assertEquals(posixpath.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("app_log_dir"))
self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("log_dir"))
self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("app_task_dir"))
def test_build_result3(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller, self.agentToggleLogger)
result_mock.return_value = {
'reports': [{'status': 'COMPLETED',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult' : False}
],
'componentStatus': []
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
commandResult = {}
hb = heartbeat.build(commandResult, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {'nodeStatus':
{'status': 'HEALTHY',
'cause': 'NONE'},
'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
'responseId': 10, 'reports': []}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
def test_update_log_level(self, basicConfig_mock, setLevel_mock):
config = AgentConfig("", "")
_, tmpoutfile = tempfile.mkstemp()
# Testing with default setup (config file does not contain loglevel entry)
# Log level should not be changed
config.set('agent', 'log_level', None)
main.update_log_level(config, tmpoutfile)
self.assertFalse(setLevel_mock.called)
setLevel_mock.reset_mock()
# Testing debug mode
config.set('agent', 'log_level', 'DEBUG')
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.DEBUG)
setLevel_mock.reset_mock()
# Testing any other mode
config.set('agent', 'log_level', 'INFO')
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.INFO)
setLevel_mock.reset_mock()
config.set('agent', 'log_level', 'WRONG')
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.INFO)
def test_registration_build(self):
tmpdir = tempfile.gettempdir()
ver_dir = os.path.join(tmpdir, "infra")
config = AgentConfig(tmpdir, ver_dir)
config.set("agent", "prefix", tmpdir)
config.set("agent", "current_ping_port", "33777")
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, {}, "tag", 1)
# print ("Register: " + pprint.pformat(data))
self.assertEquals(data["label"] != "", True, "hostname should not be empty")
self.assertEquals(data["publicHostname"] != "", True, "publicHostname should not be empty")
self.assertEquals(data["responseId"], 1)
self.assertEquals(data["timestamp"] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(data["agentVersion"], "1", "agentVersion should not be empty")
self.assertEquals(data["actualState"], State.INIT, "actualState should not be empty")
self.assertEquals(data["expectedState"], State.INIT, "expectedState should not be empty")
self.assertEquals(data["allocatedPorts"], {}, "allocatedPorts should be empty")
self.assertEquals(data["logFolders"], {}, "allocated log should be empty")
self.assertEquals(data["tags"], "tag", "tags should be tag")
self.assertEquals(len(data), 10)
self.assertEquals(posixpath.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
self.assertEquals(posixpath.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("app_log_dir"))
self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("log_dir"))
self.assertEquals(posixpath.join(ver_dir, "."), config.getResolvedPath("app_task_dir"))
def test_update_log_level(self, basicConfig_mock, setLevel_mock):
config = AgentConfig("", "")
_, tmpoutfile = tempfile.mkstemp()
# Testing with default setup (config file does not contain loglevel entry)
# Log level should not be changed
config.set('agent', 'log_level', None)
main.update_log_level(config, tmpoutfile)
self.assertFalse(setLevel_mock.called)
setLevel_mock.reset_mock()
# Testing debug mode
config.set('agent', 'log_level', 'DEBUG')
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.DEBUG)
setLevel_mock.reset_mock()
# Testing any other mode
config.set('agent', 'log_level', 'INFO')
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.INFO)
setLevel_mock.reset_mock()
config.set('agent', 'log_level', 'WRONG')
main.update_log_level(config, tmpoutfile)
setLevel_mock.assert_called_with(logging.INFO)
def main():
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", help="verbose log output", default=False)
parser.add_option("-l", "--label", dest="label", help="label of the agent", default=None)
parser.add_option("--host", dest="host", help="AppMaster host", default=None)
parser.add_option("--port", dest="port", help="AppMaster port", default=None)
(options, args) = parser.parse_args()
if not "AGENT_WORK_ROOT" in os.environ:
parser.error("AGENT_WORK_ROOT environment variable must be set.")
options.root_folder = os.environ["AGENT_WORK_ROOT"]
if not "AGENT_LOG_ROOT" in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ["AGENT_LOG_ROOT"]
if not options.label:
parser.error("label is required.")
bind_signal_handlers()
# Check for configuration file.
agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label)
update_config_from_file(agentConfig)
# update configurations if needed
if options.host:
agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", options.host)
if options.port:
agentConfig.set(AgentConfig.SERVER_SECTION, "port", options.port)
logFile = os.path.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
perform_prestart_checks(agentConfig)
ensure_folder_layout(agentConfig)
setup_logging(options.verbose, logFile)
update_log_level(agentConfig, logFile)
write_pid()
logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
server_url = SERVER_STATUS_URL.format(
agentConfig.get(AgentConfig.SERVER_SECTION, "hostname"),
agentConfig.get(AgentConfig.SERVER_SECTION, "port"),
agentConfig.get(AgentConfig.SERVER_SECTION, "check_path"),
)
print("Connecting to the server at " + server_url + "...")
logger.info("Connecting to the server at: " + server_url)
# Wait until server is reachable
netutil = NetUtil()
netutil.try_to_connect(server_url, -1, logger)
# Launch Controller communication
controller = Controller(agentConfig)
controller.start()
controller.join()
stop_agent()
logger.info("... agent finished")
def test_watchdog_1(self, kill_process_with_children_mock):
"""
Tests whether watchdog works
"""
subproc_mock = self.Subprocess_mockup()
executor = PythonExecutor("/tmp", AgentConfig("", ""))
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
PYTHON_TIMEOUT_SECONDS = 0.1
kill_process_with_children_mock.side_effect = lambda pid: subproc_mock.terminate(
)
def launch_python_subprocess_method(command, tmpout, tmperr,
environment_vars):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess = launch_python_subprocess_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = None
thread = Thread(target=executor.run_file,
args=("fake_puppetFile", ["arg1", "arg2"
], tmpoutfile, tmperrfile,
PYTHON_TIMEOUT_SECONDS, tmpstrucout, "INFO"))
thread.start()
time.sleep(0.1)
subproc_mock.finished_event.wait()
self.assertEquals(subproc_mock.was_terminated, True,
"Subprocess should be terminated due to timeout")
def test_execution_results(self):
subproc_mock = self.Subprocess_mockup()
executor = PythonExecutor("/tmp", AgentConfig("", ""))
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstroutfile = tempfile.mkstemp()
PYTHON_TIMEOUT_SECONDS = 5
def launch_python_subprocess_method(command, tmpout, tmperr,
environment_vars):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess = launch_python_subprocess_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = 0
subproc_mock.should_finish_event.set()
result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile,
tmperrfile, PYTHON_TIMEOUT_SECONDS,
tmpstroutfile, "INFO")
self.assertEquals(
result, {
'exitcode': 0,
'stderr': 'Dummy err',
'stdout': 'Dummy output',
'structuredOut': {}
})
def test_debugSetupForRegister(self, sleepMock):
original_value = self.controller.config
self.controller.config = AgentConfig("", "")
self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.DEBUG_MODE_ENABLED, "true")
self.controller.processDebugCommandForRegister()
self.controller.processDebugCommandForHeartbeat()
assert not sleepMock.called, 'sleep should not have been called'
self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_RERISTER")
self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_HEARTBEET")
self.controller.processDebugCommandForRegister()
self.controller.processDebugCommandForHeartbeat()
assert not sleepMock.called, 'sleep should not have been called'
self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_REGISTER")
self.controller.processDebugCommandForRegister()
assert sleepMock.called, 'sleep should have been called'
self.controller.processDebugCommandForHeartbeat()
assert sleepMock.call_count == 1, 'sleep should have been called once'
self.controller.config.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, "DO_NOT_HEARTBEAT")
self.controller.processDebugCommandForHeartbeat()
assert sleepMock.call_count == 2, 'sleep should have been called twice'
self.controller.config = original_value
pass
def test_perform_prestart_checks(self, isdir_mock, isfile_mock, exit_mock,
remove_mock):
main.config = AgentConfig("", "")
# Trying case if there is another instance running
isfile_mock.return_value = True
isdir_mock.return_value = True
main.perform_prestart_checks(main.config)
self.assertFalse(exit_mock.called)
self.assertTrue(remove_mock.called)
isfile_mock.reset_mock()
isdir_mock.reset_mock()
exit_mock.reset_mock()
# Trying case if agent prefix dir does not exist
isfile_mock.return_value = False
isdir_mock.return_value = False
main.perform_prestart_checks(main.config)
self.assertTrue(exit_mock.called)
isfile_mock.reset_mock()
isdir_mock.reset_mock()
exit_mock.reset_mock()
# Trying normal case
isfile_mock.return_value = False
isdir_mock.return_value = True
main.perform_prestart_checks(main.config)
self.assertFalse(exit_mock.called)
def test_python_command(self):
executor = PythonExecutor("/tmp", AgentConfig("", ""))
command = executor.python_command("script", ["script_param1"])
self.assertEqual(4, len(command))
self.assertTrue("python" in command[0])
self.assertEquals("-S", command[1])
self.assertEquals("script", command[2])
self.assertEquals("script_param1", command[3])
def test_set_env_values(self, os_env_copy_mock, subprocess_mock,
open_mock):
actual_vars = {"someOther": "value1"}
executor = PythonExecutor("/tmp", AgentConfig("", ""))
environment_vars = [("PYTHONPATH", "a:b")]
os_env_copy_mock.return_value = actual_vars
executor.run_file("script.pynot", ["a", "b"], "", "", 10, "", "INFO",
True, environment_vars)
self.assertEquals(2, len(os_env_copy_mock.return_value))
def test_build_result3(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller,
self.agentToggleLogger)
result_mock.return_value = {
'reports': [{
'status': 'COMPLETED',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': False
}],
'componentStatus': []
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
commandResult = {}
hb = heartbeat.build(commandResult, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {
'package': '',
'nodeStatus': {
'status': 'HEALTHY',
'cause': 'NONE'
},
'timestamp': 'timestamp',
'hostname': 'hostname',
'fqdn': 'fqdn',
'responseId': 10,
'reports': []
}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
self.assertEquals(commandResult, {'commandStatus': 'COMPLETED'})
def test_is_successfull(self):
executor = PythonExecutor("/tmp", AgentConfig("", ""))
executor.python_process_has_been_killed = False
self.assertTrue(executor.isSuccessfull(0))
self.assertFalse(executor.isSuccessfull(1))
executor.python_process_has_been_killed = True
self.assertFalse(executor.isSuccessfull(0))
self.assertFalse(executor.isSuccessfull(1))
def test_python_command(self):
executor = PythonExecutor("/tmp", AgentConfig("", ""),
self.agentToggleLogger)
command = executor.python_command("script", ["script_param1"])
self.assertEqual(4, len(command))
self.assertTrue("python" in command[0].lower(),
"Looking for python in %s" % (command[0].lower()))
self.assertEquals("-S", command[1])
self.assertEquals("script", command[2])
self.assertEquals("script_param1", command[3])
def test_process_command2(self, execute_status_command_mock,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
execution_command = {
'commandType': ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType': ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType': "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
actionQueue.process_command(status_command)
self.assertFalse(execute_command_mock.called)
self.assertTrue(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
execute_status_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
def test_build(self):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue, config)
result = heartbeat.build({}, 100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
"hostname should not be empty")
self.assertEquals(result['responseId'], 100)
self.assertEquals('componentStatus' not in result, True,
"Heartbeat should contain componentStatus")
self.assertEquals(result['reports'] is not None, True,
"Heartbeat should contain reports")
self.assertEquals(result['timestamp'] >= 1353679373880L, True)
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 5) or (len(result) is 6), True)
self.assertEquals(not heartbeat.reports, True,
"Heartbeat should not contain task in progress")
def test_build(self):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue, config)
result = heartbeat.build({}, 100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
"hostname should not be empty")
self.assertEquals(result['responseId'], 100)
self.assertEquals('componentStatus' not in result, True,
"Heartbeat should contain componentStatus")
self.assertEquals(result['reports'] is not None, True,
"Heartbeat should contain reports")
self.assertEquals(result['timestamp'] >= 1353679373880L, True)
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 6) or (len(result) is 7), True)
self.assertEquals(not heartbeat.reports, True,
"Heartbeat should not contain task in progress")
def test_resolve_config(self, read_mock, exists_mock):
config = AgentConfig("", "")
# Trying case if conf file exists
exists_mock.return_value = True
main.update_config_from_file(config)
self.assertTrue(read_mock.called)
exists_mock.reset_mock()
read_mock.reset_mock()
# Trying case if conf file does not exist
exists_mock.return_value = False
main.update_config_from_file(config)
self.assertFalse(read_mock.called)
def test_registration_build(self):
tmpdir = tempfile.gettempdir()
ver_dir = os.path.join(tmpdir, "infra")
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
try:
os.mkdir(ver_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
pass
ver_file = os.path.join(ver_dir, "version")
with open(ver_file, "w") as text_file:
text_file.write("1.3.0")
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
self.assertEquals(data['actualState'], State.INIT, "actualState should not be empty")
self.assertEquals(data['expectedState'], State.INIT, "expectedState should not be empty")
self.assertEquals(data['allocatedPorts'], {}, "allocatedPorts should be empty")
self.assertEquals(len(data), 8)
self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("app_log_dir"))
self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("log_dir"))
self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("app_task_dir"))
os.remove(ver_file)
os.removedirs(ver_dir)
def test_failure_window(self, mock_time):
config = AgentConfig("", "")
original_config = config.get(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '2,1')
## The behavior of side_effect is different when you run tests in command line and when you do it through IDE
## So few extra items are there in the list
mock_time.side_effect = [200, 500, 500]
controller5 = Controller.Controller(config)
try:
self.assertTrue(controller5.shouldAutoRestart())
self.assertTrue(controller5.shouldAutoRestart())
finally:
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, original_config)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
execute_command_mock,
requestComponentStatus_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
requestComponentStatus_mock.return_value = {'exitcode': 'dummy report'}
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["status"], expected)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertTrue(requestComponentStatus_mock.called)
def test_watchdog_2(self):
# Test hangs on Windows TODO
if IS_WINDOWS:
return
"""
Tries to catch false positive watchdog invocations
"""
subproc_mock = self.Subprocess_mockup()
executor = PythonExecutor("/tmp", AgentConfig("", ""),
self.agentToggleLogger)
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
PYTHON_TIMEOUT_SECONDS = 5
environment_vars = [("PYTHONPATH", "a:b")]
def launch_python_subprocess_method(command, tmpout, tmperr,
environment_vars):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess = launch_python_subprocess_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = 0
thread = Thread(target=executor.run_file,
args=("fake_puppetFile", ["arg1", "arg2"
], tmpoutfile, tmperrfile,
PYTHON_TIMEOUT_SECONDS, tmpstrucout, "INFO"))
thread.start()
time.sleep(0.1)
subproc_mock.should_finish_event.set()
subproc_mock.finished_event.wait()
self.assertEquals(
subproc_mock.was_terminated, False,
"Subprocess should not be terminated before timeout")
self.assertEquals(
subproc_mock.returncode, 0,
"Subprocess should not be terminated before timeout")
def test_execute_status_command_expect_config(self, CustomServiceOrchestrator_mock,
execute_command_mock,
runCommand_mock,
status_update_callback):
csoMocks = [MagicMock()]
CustomServiceOrchestrator_mock.side_effect = csoMocks
csoMocks[0].status_commands_stdout = None
csoMocks[0].status_commands_stderr = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AgentConfig("", ""), dummy_controller, self.agentToggleLogger)
runCommand_mock.return_value = {'configurations': {}}
actionQueue.execute_status_command(self.status_command_with_config)
report = actionQueue.result()
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0]["componentName"], "ACCUMULO_MASTER")
self.assertEqual(report['componentStatus'][0]["serviceName"], "ACCUMULO")
self.assertEqual(report['componentStatus'][0]["clusterName"], "c1")
self.assertEqual(report['componentStatus'][0]["configurations"], {})
self.assertFalse(runCommand_mock.called)
def main():
parser = OptionParser()
parser.add_option("-v",
"--verbose",
dest="verbose",
help="verbose log output",
default=False)
parser.add_option("-l",
"--label",
dest="label",
help="label of the agent",
default=None)
parser.add_option("--host",
dest="host",
help="AppMaster host",
default=None)
parser.add_option("--port",
dest="port",
help="AppMaster port",
default=None)
(options, args) = parser.parse_args()
if not 'AGENT_WORK_ROOT' in os.environ:
parser.error("AGENT_WORK_ROOT environment variable must be set.")
options.root_folder = os.environ['AGENT_WORK_ROOT']
if not 'AGENT_LOG_ROOT' in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ['AGENT_LOG_ROOT']
if not options.label:
parser.error("label is required.")
bind_signal_handlers()
# Check for configuration file.
agentConfig = AgentConfig(options.root_folder, options.log_folder,
options.label)
update_config_from_file(agentConfig)
# update configurations if needed
if options.host:
agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", options.host)
if options.port:
agentConfig.set(AgentConfig.SERVER_SECTION, "port", options.port)
logFile = os.path.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR),
logFileName)
perform_prestart_checks(agentConfig)
ensure_folder_layout(agentConfig)
setup_logging(options.verbose, logFile)
update_log_level(agentConfig, logFile)
write_pid()
logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
server_url = SERVER_STATUS_URL.format(
agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'port'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'check_path'))
print("Connecting to the server at " + server_url + "...")
logger.info('Connecting to the server at: ' + server_url)
# Wait until server is reachable
netutil = NetUtil()
netutil.try_to_connect(server_url, -1, logger)
# Launch Controller communication
controller = Controller(agentConfig)
controller.start()
controller.join()
stop_agent()
logger.info("... agent finished")
def test_auto_start(self, dumpsMock, loadsMock, timeMock, waitMock, mock_createStatusCommand):
original_value = self.controller.config
self.controller.config = AgentConfig("", "")
out = StringIO.StringIO()
sys.stdout = out
heartbeat = MagicMock()
self.controller.heartbeat = heartbeat
dumpsMock.return_value = "data"
sendRequest = MagicMock(name="sendRequest")
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
response1 = {"responseId": "2", "restartAgent": False, "restartEnabled": True}
response2 = {"responseId": "2", "restartAgent": False, "restartEnabled": False}
loadsMock.side_effect = [response1, response2, response1]
def one_heartbeat(*args, **kwargs):
self.controller.DEBUG_STOP_HEARTBEATING = True
return "data"
sendRequest.side_effect = one_heartbeat
actionQueue = MagicMock()
actionQueue.isIdle.return_value = True
# one successful request, after stop
self.controller.actionQueue = actionQueue
self.controller.componentActualState = State.FAILED
self.controller.componentExpectedState = State.STARTED
self.assertTrue(self.controller.componentActualState, State.FAILED)
self.controller.actionQueue.customServiceOrchestrator.stored_command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'HBASE',
'roleCommand': u'START',
'commandId': '7-1',
'taskId': 7,
"componentName": "HBASE_MASTER",
'clusterName': u'cc',
'serviceName': u'HDFS'
}
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
self.assertTrue(self.controller.componentActualState, State.STARTING)
self.assertTrue(self.controller.componentExpectedState, State.STARTED)
self.assertEquals(self.controller.failureCount, 0)
self.assertFalse(mock_createStatusCommand.called)
addToQueue.assert_has_calls([call([{
'commandType': 'EXECUTION_COMMAND',
'clusterName': u'cc',
'serviceName': u'HDFS',
'role': u'HBASE',
'taskId': 8,
'roleCommand': u'START',
'componentName': 'HBASE_MASTER',
'commandId': '8-1',
'auto_generated': True}])])
self.controller.config = original_value
# restartEnabled = False
self.controller.componentActualState = State.FAILED
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
self.assertTrue(self.controller.componentActualState, State.FAILED)
self.assertTrue(self.controller.componentExpectedState, State.STARTED)
# restartEnabled = True
self.controller.componentActualState = State.INSTALLED
self.controller.componentExpectedState = State.INSTALLED
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
self.assertTrue(self.controller.componentActualState, State.INSTALLED)
self.assertTrue(self.controller.componentExpectedState, State.INSTALLED)
pass
def test_heartbeatWithServerTerminateAgent(self, dumpsMock, loadsMock, sleepMock, event_mock):
original_value = self.controller.config
self.controller.config = AgentConfig("", "")
out = StringIO.StringIO()
sys.stdout = out
hearbeat = MagicMock()
self.controller.heartbeat = hearbeat
dumpsMock.return_value = "data"
sendRequest = MagicMock(name="sendRequest")
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
def one_heartbeat(*args, **kwargs):
self.controller.DEBUG_STOP_HEARTBEATING = True
return "data"
sendRequest.side_effect = one_heartbeat
actionQueue = MagicMock()
actionQueue.isIdle.return_value = True
# one successful request, after stop
self.controller.actionQueue = actionQueue
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
calls = []
def retry(*args, **kwargs):
if len(calls) == 0:
calls.append(1)
response["responseId"] = "3"
raise Exception()
if len(calls) > 0:
self.controller.DEBUG_STOP_HEARTBEATING = True
return "data"
# exception, retry, successful and stop
sendRequest.side_effect = retry
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
original_stopApp = self.controller.stopApp
# terminateAgent command - test 1
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
response = {"responseId":"2", "terminateAgent": True}
loadsMock.return_value = response
stopApp = MagicMock(name="stopApp")
self.controller.stopApp = stopApp
self.controller.heartbeatWithServer()
stopApp.assert_called_once_with()
# reset for next test
self.controller.terminateAgent = False
# terminateAgent command - test 2
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
response = {"responseId":"2", "terminateAgent": True}
loadsMock.return_value = response
self.controller.stopApp = original_stopApp
stopCommand = {"roleCommand": "STOP"}
self.controller.stopCommand = stopCommand
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
self.controller.componentActualState = State.STARTED
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.terminateAgent)
self.assertTrue(self.controller.appGracefulStopQueued)
addToQueue.assert_has_calls([call([stopCommand])])
# reset for next test
self.controller.terminateAgent = False
self.controller.appGracefulStopQueued = False
# terminateAgent command - test 3
self.controller.responseId = 2
self.controller.DEBUG_STOP_HEARTBEATING = False
response = {"responseId":"3", "terminateAgent": True}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
self.controller.stopApp = original_stopApp
self.controller.componentActualState = State.STARTED
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.terminateAgent)
self.assertTrue(self.controller.appGracefulStopQueued)
addToQueue.assert_has_calls([call([stopCommand])])
self.controller.terminateAgent = False
sleepMock.assert_called_with(
self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
sys.stdout = sys.__stdout__
self.controller.sendRequest = Controller.Controller.sendRequest
self.controller.addToQueue = Controller.Controller.addToQueue
self.controller.config = original_value
pass
def test_heartbeatWithServer(self, dumpsMock, loadsMock, sleepMock, event_mock):
original_value = self.controller.config
self.controller.config = AgentConfig("", "")
out = StringIO.StringIO()
sys.stdout = out
hearbeat = MagicMock()
self.controller.heartbeat = hearbeat
dumpsMock.return_value = "data"
sendRequest = MagicMock(name="sendRequest")
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
def one_heartbeat(*args, **kwargs):
self.controller.DEBUG_STOP_HEARTBEATING = True
return "data"
sendRequest.side_effect = one_heartbeat
actionQueue = MagicMock()
actionQueue.isIdle.return_value = True
# one successful request, after stop
self.controller.actionQueue = actionQueue
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
calls = []
def retry(*args, **kwargs):
if len(calls) == 0:
calls.append(1)
response["responseId"] = "3"
raise Exception()
if len(calls) > 0:
self.controller.DEBUG_STOP_HEARTBEATING = True
return "data"
# exception, retry, successful and stop
sendRequest.side_effect = retry
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
# retry registration
response["registrationCommand"] = {"command": "register"}
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.repeatRegistration)
# components are not mapped
response["registrationCommand"] = {"command": "register"}
response["hasMappedComponents"] = False
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertFalse(self.controller.hasMappedComponents)
# components are mapped
response["hasMappedComponents"] = True
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.hasMappedComponents)
# components are mapped
del response["hasMappedComponents"]
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.hasMappedComponents)
# wrong responseId => restart
response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
restartAgent.assert_called_once_with()
# executionCommands
self.controller.responseId = 1
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
response["executionCommands"] = "executionCommands"
self.controller.statusCommand = ["statusCommand"]
updateStateBasedOnCommand = MagicMock(name="updateStateBasedOnCommand")
self.controller.updateStateBasedOnCommand = updateStateBasedOnCommand
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([call("executionCommands")])
updateStateBasedOnCommand.assert_has_calls([call("executionCommands")])
# just status command when state = STARTED
self.controller.responseId = 1
response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
self.controller.statusCommand = "statusCommand"
self.controller.componentActualState = State.STARTED
self.controller.componentExpectedState = State.STARTED
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([call(["statusCommand"])])
# just status command when state = FAILED
self.controller.responseId = 1
response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
self.controller.statusCommand = "statusCommand"
self.controller.componentActualState = State.FAILED
self.controller.componentExpectedState = State.STARTED
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([call(["statusCommand"])])
# no status command when state = STARTING
self.controller.responseId = 1
response = {"responseId":"2", "restartAgent": False}
loadsMock.return_value = response
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
self.controller.statusCommand = "statusCommand"
self.controller.componentActualState = State.STARTING
self.controller.componentExpectedState = State.STARTED
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([])
# statusCommands
response["statusCommands"] = "statusCommands"
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([call("statusCommands")])
# restartAgent command
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
response["restartAgent"] = True
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.heartbeatWithServer()
restartAgent.assert_called_once_with()
# actionQueue not idle
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
actionQueue.isIdle.return_value = False
response["restartAgent"] = False
self.controller.heartbeatWithServer()
sleepMock.assert_called_with(
self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
sys.stdout = sys.__stdout__
self.controller.sendRequest = Controller.Controller.sendRequest
self.controller.addToQueue = Controller.Controller.addToQueue
self.controller.config = original_value
pass
def test_registration_build(self):
tmpdir = tempfile.gettempdir()
ver_dir = os.path.join(tmpdir, "infra")
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, {}, "tag", 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(data['label'] != "", True,
"hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True,
"publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True,
"timestamp should not be empty")
self.assertEquals(data['agentVersion'], '1',
"agentVersion should not be empty")
self.assertEquals(data['actualState'], State.INIT,
"actualState should not be empty")
self.assertEquals(data['expectedState'], State.INIT,
"expectedState should not be empty")
self.assertEquals(data['allocatedPorts'], {},
"allocatedPorts should be empty")
self.assertEquals(data['logFolders'], {},
"allocated log should be empty")
self.assertEquals(data['tags'], "tag", "tags should be tag")
self.assertEquals(len(data), 10)
self.assertEquals(posixpath.join(tmpdir, "app/definition"),
config.getResolvedPath("app_pkg_dir"))
self.assertEquals(posixpath.join(tmpdir, "app/install"),
config.getResolvedPath("app_install_dir"))
self.assertEquals(posixpath.join(ver_dir, "."),
config.getResolvedPath("app_log_dir"))
self.assertEquals(posixpath.join(ver_dir, "."),
config.getResolvedPath("log_dir"))
self.assertEquals(posixpath.join(ver_dir, "."),
config.getResolvedPath("app_task_dir"))
def test_build_long_result(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777},
{'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitcode': 0},
{'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 13},
{'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {'global': {'tag': 'v1'}},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0}
],
'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE', 'reportResult' : True},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE', 'reportResult' : True},
{'status': 'UNHEALTHY', 'componentName': 'HBASE_MASTER', 'reportResult' : False},
],
}
heartbeat = Heartbeat(actionQueue, config)
hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {'nodeStatus':
{'status': 'HEALTHY',
'cause': 'NONE'},
'timestamp': 'timestamp', 'hostname': 'hostname', 'fqdn': 'fqdn',
'responseId': 10, 'reports': [
{'status': 'IN_PROGRESS', 'roleCommand': u'INSTALL',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt', 'clusterName': u'cc',
'taskId': 3, 'exitcode': 777},
{'status': 'COMPLETED', 'roleCommand': 'UPGRADE',
'serviceName': 'serviceName', 'role': 'role', 'actionId': 17,
'stderr': 'stderr', 'stdout': 'out', 'clusterName': 'clusterName',
'taskId': 'taskId', 'exitcode': 0},
{'status': 'FAILED', 'roleCommand': u'INSTALL', 'serviceName': u'HDFS',
'role': u'DATANODE', 'actionId': '1-1', 'stderr': 'stderr',
'stdout': 'out', 'clusterName': u'cc', 'taskId': 3, 'exitcode': 13},
{'status': 'COMPLETED', 'stdout': 'out',
'configurationTags': {'global': {'tag': 'v1'}}, 'taskId': 3,
'exitcode': 0, 'roleCommand': u'INSTALL', 'clusterName': u'cc',
'serviceName': u'HDFS', 'role': u'DATANODE', 'actionId': '1-1',
'stderr': 'stderr'}], 'componentStatus': [
{'status': 'HEALTHY', 'componentName': 'DATANODE'},
{'status': 'UNHEALTHY', 'componentName': 'NAMENODE'}]}
self.assertEquals(hb, expected)
def main():
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", help="verbose log output", default=False)
parser.add_option("-l", "--label", dest="label", help="label of the agent", default=None)
parser.add_option("--zk-quorum", dest=Constants.ZK_QUORUM, help="Zookeeper Quorum", default=None)
parser.add_option("--zk-reg-path", dest=Constants.ZK_REG_PATH, help="Zookeeper Registry Path", default=None)
parser.add_option("--debug", dest="debug", help="Agent debug hint", default="")
(options, args) = parser.parse_args()
if not Constants.AGENT_WORK_ROOT in os.environ and not 'PWD' in os.environ:
parser.error("AGENT_WORK_ROOT environment variable or PWD must be set.")
if Constants.AGENT_WORK_ROOT in os.environ:
options.root_folder = os.environ[Constants.AGENT_WORK_ROOT]
else:
# some launch environments do not end up setting all environment variables
options.root_folder = os.environ['PWD']
if not 'AGENT_LOG_ROOT' in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ['AGENT_LOG_ROOT']
all_log_folders = [x.strip() for x in options.log_folder.split(',')]
if len(all_log_folders) > 1:
options.log_folder = all_log_folders[0]
# If there are multiple log folder, separate by comma, pick one
if not options.label:
parser.error("label is required.");
if not IS_WINDOWS:
bind_signal_handlers()
# Check for configuration file.
agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label)
update_config_from_file(agentConfig)
# update configurations if needed
if options.zk_quorum:
agentConfig.set(AgentConfig.SERVER_SECTION, Constants.ZK_QUORUM, options.zk_quorum)
if options.zk_reg_path:
agentConfig.set(AgentConfig.SERVER_SECTION, Constants.ZK_REG_PATH, options.zk_reg_path)
if options.debug:
agentConfig.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, options.debug)
logFile = posixpath.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
setup_logging(options.verbose, logFile)
update_log_level(agentConfig, logFile)
secDir = posixpath.join(agentConfig.getResolvedPath(AgentConfig.RUN_DIR), "security")
logger.info("Security/Keys directory: " + secDir)
agentConfig.set(AgentConfig.SECURITY_SECTION, "keysdir", secDir)
perform_prestart_checks(agentConfig)
ensure_folder_layout(agentConfig)
# create security dir if necessary
ensure_path_exists(secDir)
write_pid()
logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
if len(all_log_folders) > 1:
logger.info("Selected log folder from available: " + ",".join(all_log_folders))
# Extract the AM hostname and secured port from ZK registry
zk_lookup_tries = 0
while zk_lookup_tries < Constants.MAX_AM_CONNECT_RETRIES:
registry = Registry(options.zk_quorum, options.zk_reg_path)
amHost, amUnsecuredPort, amSecuredPort = registry.readAMHostPort()
tryConnect = True
if not amHost or not amSecuredPort or not amUnsecuredPort:
logger.info("Unable to extract AM host details from ZK, retrying ...")
tryConnect = False
time.sleep(NetUtil.CONNECT_SERVER_RETRY_INTERVAL_SEC)
if tryConnect:
if amHost:
agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", amHost)
if amSecuredPort:
agentConfig.set(AgentConfig.SERVER_SECTION, "secured_port", amSecuredPort)
if amUnsecuredPort:
agentConfig.set(AgentConfig.SERVER_SECTION, "port", amUnsecuredPort)
server_url = SERVER_STATUS_URL.format(
agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'port'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'check_path'))
print("Connecting to the server at " + server_url + "...")
logger.info('Connecting to the server at: ' + server_url)
# Wait until server is reachable and continue to query ZK
netutil = NetUtil()
retries = netutil.try_to_connect(server_url, 3, logger)
if retries < 3:
break;
pass
pass
zk_lookup_tries += 1
pass
# Launch Controller communication
global controller
controller = Controller(agentConfig)
controller.start()
try:
while controller.is_alive():
controller.join(timeout=1.0)
except (KeyboardInterrupt, SystemExit):
logger.info("... agent interrupted")
pass
def test_registration_build(self):
tmpdir = tempfile.gettempdir()
ver_dir = os.path.join(tmpdir, "infra")
config = AgentConfig(tmpdir, ver_dir)
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
try:
os.mkdir(ver_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
pass
ver_file = os.path.join(ver_dir, "version")
with open(ver_file, "w") as text_file:
text_file.write("1.3.0")
register = Register(config)
data = register.build(State.INIT, State.INIT, {}, 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(data['hostname'] != "", True,
"hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True,
"publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True,
"timestamp should not be empty")
self.assertEquals(data['agentVersion'], '1.3.0',
"agentVersion should not be empty")
self.assertEquals(data['actualState'], State.INIT,
"actualState should not be empty")
self.assertEquals(data['expectedState'], State.INIT,
"expectedState should not be empty")
self.assertEquals(data['allocatedPorts'], {},
"allocatedPorts should be empty")
self.assertEquals(len(data), 8)
self.assertEquals(os.path.join(tmpdir, "app/definition"),
config.getResolvedPath("app_pkg_dir"))
self.assertEquals(os.path.join(tmpdir, "app/install"),
config.getResolvedPath("app_install_dir"))
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("app_log_dir"))
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("log_dir"))
self.assertEquals(os.path.join(ver_dir, "."),
config.getResolvedPath("app_task_dir"))
os.remove(ver_file)
os.removedirs(ver_dir)
def test_config1(self):
config = AgentConfig("", "")
(max, window) = config.getErrorWindow()
self.assertEqual(max, 5)
self.assertEqual(window, 5)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '33')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART,
'-4,-6')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART,
'wd,er')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART,
'2,20')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 2)
self.assertEqual(window, 20)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART,
' 2, 30')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
def test_build_long_result(self, result_mock):
config = AgentConfig("", "")
config.set('agent', 'prefix', 'tmp')
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller,
self.agentToggleLogger)
result_mock.return_value = {
'reports': [{
'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 777,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitcode': 0,
'reportResult': True
}, {
'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 13,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0,
'reportResult': True
}, {
'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitcode': 0,
'reportResult': False
}],
'componentStatus': [
{
'status': 'HEALTHY',
'componentName': 'DATANODE',
'reportResult': True
},
{
'status': 'UNHEALTHY',
'componentName': 'NAMENODE',
'reportResult': True
},
{
'status': 'UNHEALTHY',
'componentName': 'HBASE_MASTER',
'reportResult': False
},
],
}
heartbeat = Heartbeat(actionQueue, config, self.agentToggleLogger)
# State.STARTED results in agentState to be set to 4 (enum order)
hb = heartbeat.build({}, 10)
hb['hostname'] = 'hostname'
hb['timestamp'] = 'timestamp'
hb['fqdn'] = 'fqdn'
expected = {
'nodeStatus': {
'status': 'HEALTHY',
'cause': 'NONE'
},
'timestamp':
'timestamp',
'hostname':
'hostname',
'fqdn':
'fqdn',
'responseId':
10,
'reports': [{
'status': 'IN_PROGRESS',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'taskId': 3,
'exitcode': 777
}, {
'status': 'COMPLETED',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'taskId': 'taskId',
'exitcode': 0
}, {
'status': 'FAILED',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'taskId': 3,
'exitcode': 13
}, {
'status': 'COMPLETED',
'stdout': 'out',
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'taskId': 3,
'exitcode': 0,
'roleCommand': u'INSTALL',
'clusterName': u'cc',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'stderr': 'stderr'
}],
'componentStatus': [{
'status': 'HEALTHY',
'componentName': 'DATANODE'
}, {
'status': 'UNHEALTHY',
'componentName': 'NAMENODE'
}]
}
self.assertEqual.__self__.maxDiff = None
self.assertEquals(hb, expected)
def test_config1(self):
config = AgentConfig("", "")
(max, window) = config.getErrorWindow()
self.assertEqual(max, 5)
self.assertEqual(window, 5)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '33')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '-4,-6')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, 'wd,er')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, '2,20')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 2)
self.assertEqual(window, 20)
config.set(AgentConfig.COMMAND_SECTION, AgentConfig.AUTO_RESTART, ' 2, 30')
(max, window) = config.getErrorWindow()
self.assertEqual(max, 0)
self.assertEqual(window, 0)
def main():
parser = OptionParser()
parser.add_option("-v", "--verbose", dest="verbose", help="verbose log output", default=False)
parser.add_option("-l", "--label", dest="label", help="label of the agent", default=None)
parser.add_option("--host", dest="host", help="AppMaster host", default=None)
parser.add_option("--port", dest="port", help="AppMaster port", default=None)
parser.add_option("--secured_port", dest="secured_port", help="AppMaster 2 Way port", default=None)
parser.add_option("--debug", dest="debug", help="Agent debug hint", default="")
(options, args) = parser.parse_args()
if not 'AGENT_WORK_ROOT' in os.environ:
parser.error("AGENT_WORK_ROOT environment variable must be set.");
options.root_folder = os.environ['AGENT_WORK_ROOT']
if not 'AGENT_LOG_ROOT' in os.environ:
parser.error("AGENT_LOG_ROOT environment variable must be set.");
options.log_folder = os.environ['AGENT_LOG_ROOT']
if not options.label:
parser.error("label is required.");
bind_signal_handlers()
# Check for configuration file.
agentConfig = AgentConfig(options.root_folder, options.log_folder, options.label)
update_config_from_file(agentConfig)
# update configurations if needed
if options.host:
agentConfig.set(AgentConfig.SERVER_SECTION, "hostname", options.host)
if options.port:
agentConfig.set(AgentConfig.SERVER_SECTION, "port", options.port)
if options.secured_port:
agentConfig.set(AgentConfig.SERVER_SECTION, "secured_port", options.secured_port)
if options.debug:
agentConfig.set(AgentConfig.AGENT_SECTION, AgentConfig.APP_DBG_CMD, options.debug)
# set the security directory to a subdirectory of the run dir
secDir = os.path.join(agentConfig.getResolvedPath(AgentConfig.RUN_DIR), "security")
logger.info("Security/Keys directory: " + secDir)
agentConfig.set(AgentConfig.SECURITY_SECTION, "keysdir", secDir)
logFile = os.path.join(agentConfig.getResolvedPath(AgentConfig.LOG_DIR), logFileName)
perform_prestart_checks(agentConfig)
ensure_folder_layout(agentConfig)
# create security dir if necessary
ensure_path_exists(secDir)
setup_logging(options.verbose, logFile)
update_log_level(agentConfig, logFile)
write_pid()
logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
server_url = SERVER_STATUS_URL.format(
agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'port'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'check_path'))
print("Connecting to the server at " + server_url + "...")
logger.info('Connecting to the server at: ' + server_url)
# Wait until server is reachable
netutil = NetUtil()
netutil.try_to_connect(server_url, -1, logger)
# Launch Controller communication
controller = Controller(agentConfig)
controller.start()
try:
while controller.is_alive():
controller.join(timeout=1.0)
except (KeyboardInterrupt, SystemExit):
logger.info("... agent interrupted")
pass
