def test_RetryAction(self):
action={'id' : 'tttt'}
config = AmbariConfig().getConfig()
actionQueue = ActionQueue(config)
path = actionQueue.getInstallFilename(action['id'])
configFile = {
"data" : "test",
"owner" : os.getuid(),
"group" : os.getgid() ,
"permission" : 0700,
"path" : path,
"umask" : 022
}
#note that the command in the action is just a listing of the path created
#we just want to ensure that 'ls' can run on the data file (in the actual world
#this 'ls' would be a puppet or a chef command that would work on a data
#file
badAction = {
'id' : 'tttt',
'kind' : 'INSTALL_AND_CONFIG_ACTION',
'workDirComponent' : 'abc-hdfs',
'file' : configFile,
'clusterDefinitionRevision' : 12,
'command' : ['/bin/ls',"/foo/bar/badPath1234"]
}
path=getFilePath(action,path)
goodAction = {
'id' : 'tttt',
'kind' : 'INSTALL_AND_CONFIG_ACTION',
'workDirComponent' : 'abc-hdfs',
'file' : configFile,
'clusterDefinitionRevision' : 12,
'command' : ['/bin/ls',path]
}
actionQueue.start()
response = {'actions' : [badAction,goodAction]}
actionQueue.maxRetries = 2
actionQueue.sleepInterval = 1
result = actionQueue.put(response)
results = actionQueue.result()
sleptCount = 1
while (len(results) < 2 and sleptCount < 15):
time.sleep(1)
sleptCount += 1
results = actionQueue.result()
actionQueue.stop()
actionQueue.join()
self.assertEqual(len(results), 2, 'Number of results is not 2.')
result = results[0]
maxretries = config.get('command', 'maxretries')
self.assertEqual(int(result['retryActionCount']),
int(maxretries),
"Number of retries is %d and not %d" %
(int(result['retryActionCount']), int(str(maxretries))))
result = results[1]
self.assertEqual(int(result['retryActionCount']),
1,
"Number of retries is %d and not %d" %
(int(result['retryActionCount']), 1))
def test_read_write_component(self):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
tags1 = { "global": "version1", "core-site": "version2" }
handler = ActualConfigHandler(config, {})
handler.write_actual(tags1)
handler.write_actual_component('FOO', tags1)
output1 = handler.read_actual_component('FOO')
output2 = handler.read_actual_component('GOO')
self.assertEquals(tags1, output1)
self.assertEquals(None, output2)
tags2 = { "global": "version1", "core-site": "version2" }
handler.write_actual(tags2)
output3 = handler.read_actual()
output4 = handler.read_actual_component('FOO')
self.assertEquals(tags2, output3)
self.assertEquals(tags1, output4)
os.remove(os.path.join(tmpdir, "FOO_" + ActualConfigHandler.CONFIG_NAME))
os.remove(os.path.join(tmpdir, ActualConfigHandler.CONFIG_NAME))
def test_build(self, read_actual_component_mock):
for component in LiveStatus.COMPONENTS:
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', "ambari_agent" + os.sep + "dummy_files")
livestatus = LiveStatus('', component['serviceName'], component['componentName'], {}, config, {})
livestatus.versionsHandler.versionsFilePath = "ambari_agent" + os.sep + "dummy_files" + os.sep + "dummy_current_stack"
result = livestatus.build()
print "LiveStatus of {0}: {1}".format(component['serviceName'], str(result))
self.assertEquals(len(result) > 0, True, 'Livestatus should not be empty')
if component['componentName'] == 'GANGLIA_SERVER':
self.assertEquals(result['stackVersion'],'{"stackName":"HDP","stackVersion":"1.2.2"}',
'Livestatus should contain component stack version')
# Test build status for CLIENT component (in LiveStatus.CLIENT_COMPONENTS)
read_actual_component_mock.return_value = "some tags"
livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config, {})
result = livestatus.build()
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result.has_key('configurationTags'))
# Test build status with forsed_component_status
## Alive
livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config, {})
result = livestatus.build(forsed_component_status = LiveStatus.LIVE_STATUS)
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result['status'], LiveStatus.LIVE_STATUS)
## Dead
livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', { }, config, {})
result = livestatus.build(forsed_component_status = LiveStatus.DEAD_STATUS)
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result['status'], LiveStatus.DEAD_STATUS)
livestatus = LiveStatus('c1', 'TEZ', 'TEZ_CLIENT', { }, config, {})
result = livestatus.build(forsed_component_status = LiveStatus.LIVE_STATUS)
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result['status'], LiveStatus.LIVE_STATUS)
def test_registration_build(self, get_os_version_mock, get_os_type_mock, run_os_cmd_mock):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
run_os_cmd_mock.return_value = (3, "", "")
ver_file = os.path.join(tmpdir, "version")
with open(ver_file, "w") as text_file:
text_file.write("1.3.0")
register = Register(config)
data = register.build(1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(len(data['hardwareProfile']) > 0, True, "hardwareProfile should contain content")
self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True, "publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(len(data['agentEnv']) > 0, True, "agentEnv should not be empty")
self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
print data['agentEnv']['umask']
self.assertEquals(not data['agentEnv']['umask']== "", True, "agents umask should not be empty")
self.assertEquals(data['currentPingPort'] == 33777, True, "current ping port should be 33777")
self.assertEquals(len(data), 8)
os.remove(ver_file)
def test_status_command_without_globals_section(self, stopped_method,
read_stack_version_method):
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', TestStackVersionsFileHandler.dummyVersionsFile)
queue = ActionQueue(config)
statusCommand = {
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "",
"componentName" : "DATANODE",
'configurations':{}
}
queue.stopped = stopped_method
stopped_method.side_effect = [False, False, True, True, True]
read_stack_version_method.return_value="1.3.0"
queue.IDLE_SLEEP_TIME = 0.001
queue.put(statusCommand)
queue.run()
returned_result = queue.resultQueue.get()
returned_result[1]['status'] = 'INSTALLED' # Patch live value
self.assertEquals(returned_result, ('STATUS_COMMAND',
{'clusterName': '',
'componentName': 'DATANODE',
'msg': '',
'serviceName': 'HDFS',
'stackVersion': '1.3.0',
'status': 'INSTALLED'}))
def test_add_reg_listener_to_controller(self, FileCache_mock):
FileCache_mock.return_value = None
dummy_controller = MagicMock()
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
CustomServiceOrchestrator(config, dummy_controller)
self.assertTrue(dummy_controller.registration_listeners.append.called)
def setup(options): config = AmbariConfig() configFile = config.getConfigFile() updateConfigServerHostname(configFile, options.host_name) AmbariAgentService.set_ctrl_c_handler(ctrlHandler) AmbariAgentService.Install()
def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock,
decompress_cluster_host_info_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations':{'global' : {}},
'configurationTags':{'global' : { 'tag': 'v1' }},
'clusterHostInfo':{'namenode_host' : ['1'],
'slave_hosts' : ['0', '1'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']},
'hostLevelParams':{}
}
decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670', '8670']}
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
self.assertTrue(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType']='STATUS_COMMAND'
decompress_cluster_host_info_mock.reset_mock()
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
self.assertFalse(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertEquals(command['agentConfigParams']['agent']['parallel_execution'], 0)
self.assertTrue(unlink_mock.called)
def run_simulation():
Controller.logger = MagicMock()
sendRequest_method = MagicMock()
tmpfile = tempfile.gettempdir()
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', tmpfile)
scriptsDir = os.path.join(os.getcwd(), os.pardir,os.pardir,
os.pardir, 'main', 'upgrade_stack')
config.set('stack', 'upgradeScriptsDir', scriptsDir)
ver_file = os.path.join(tmpfile, "version")
with open(ver_file, "w") as text_file:
text_file.write(agent_version)
controller = Controller.Controller(config)
controller.sendRequest = sendRequest_method
controller.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC = 0.1
controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
controller.range = 1
for responce in responces:
queue.put(responce)
def send_stub(url, data):
logger.info("Controller sends data to %s :" % url)
logger.info(pprint.pformat(data))
if not queue.empty():
responce = queue.get()
else:
responce = responces[-1]
logger.info("There is no predefined responce available, sleeping for 30 sec")
time.sleep(30)
responce = json.loads(responce)
responseId.inc()
responce["responseId"] = responseId.val()
responce = json.dumps(responce)
logger.info("Returning data to Controller:" + responce)
return responce
sendRequest_method.side_effect = send_stub
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(filename)s:%(lineno)d - \
%(message)s")
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
logger.info("Starting")
controller.start()
controller.actionQueue.IDLE_SLEEP_TIME = 0.1
controller.run()
def test_update_log_level(self, basicConfig_mock, setLevel_mock):
config = AmbariConfig().getConfig()
# Testing with default setup (config file does not contain loglevel entry)
# Log level should not be changed
config.set('agent', 'loglevel', None)
main.update_log_level(config)
self.assertFalse(setLevel_mock.called)
setLevel_mock.reset_mock()
# Testing debug mode
config.set('agent', 'loglevel', 'DEBUG')
main.update_log_level(config)
setLevel_mock.assert_called_with(logging.DEBUG)
setLevel_mock.reset_mock()
# Testing any other mode
config.set('agent', 'loglevel', 'INFO')
main.update_log_level(config)
setLevel_mock.assert_called_with(logging.INFO)
setLevel_mock.reset_mock()
config.set('agent', 'loglevel', 'WRONG')
main.update_log_level(config)
setLevel_mock.assert_called_with(logging.INFO)
def test_process_command(self, execute_status_command_mock,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
config = AmbariConfig()
config.set('agent', 'tolerate_download_failures', "true")
actionQueue = ActionQueue(config, dummy_controller)
execution_command = {
'commandType' : ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType' : ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType' : "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
actionQueue.process_command(status_command)
self.assertFalse(execute_command_mock.called)
self.assertTrue(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
execute_status_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
def test_read_write(self):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
tags = { "global": "version1", "core-site": "version2" }
handler = ActualConfigHandler(config, tags)
handler.write_actual(tags)
output = handler.read_actual()
self.assertEquals(tags, output)
os.remove(os.path.join(tmpdir, ActualConfigHandler.CONFIG_NAME))
def test_build(self):
for component in LiveStatus.COMPONENTS:
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', "dummy_files")
livestatus = LiveStatus('', component['serviceName'], component['componentName'], {}, config)
livestatus.versionsHandler.versionsFilePath = os.path.join("dummy_files","dummy_current_stack")
result = livestatus.build()
print "LiveStatus of {0}: {1}".format(component['serviceName'], str(result))
self.assertEquals(len(result) > 0, True, 'Livestatus should not be empty')
if component['componentName'] == 'GANGLIA_SERVER':
self.assertEquals(result['stackVersion'],'{"stackName":"HDP","stackVersion":"1.2.2"}',
'Livestatus should contain component stack version')
def test_read_agent_version(self, get_os_version_mock, get_os_type_mock):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
ver_file = os.path.join(tmpdir, "version")
reference_version = "1.3.0"
with open(ver_file, "w") as text_file:
text_file.write(reference_version)
version = self.controller.read_agent_version(config)
os.remove(ver_file)
self.assertEqual(reference_version, version)
def test_command_in_progress(self):
config = AmbariConfig().getConfig()
tmpfile = tempfile.gettempdir()
config.set('agent', 'prefix', tmpfile)
actionQueue = ActionQueue(config)
actionQueue.IDLE_SLEEP_TIME = 0.01
executor_started_event = threading.Event()
end_executor_event = threading.Event()
actionQueue.puppetExecutor = FakeExecutor(executor_started_event, end_executor_event)
before_start_result = actionQueue.result()
command = {
'commandId': 17,
'role' : "role",
'taskId' : "taskId",
'clusterName' : "clusterName",
'serviceName' : "serviceName",
'status' : 'IN_PROGRESS',
'hostname' : "localhost.localdomain",
'hostLevelParams': "hostLevelParams",
'clusterHostInfo': "clusterHostInfo",
'roleCommand': "roleCommand",
'configurations': "configurations",
'commandType': "EXECUTION_COMMAND",
'configurations':{'global' : {}}
}
actionQueue.put(command)
actionQueue.start()
executor_started_event.wait()
#print ("ii: " + pprint.pformat(actionQueue.commandInProgress))
in_progress_result = actionQueue.result()
end_executor_event.set()
actionQueue.stop()
actionQueue.join()
after_start_result = actionQueue.result()
self.assertEquals(len(before_start_result['componentStatus']), 0)
self.assertEquals(len(before_start_result['reports']), 0)
self.assertEquals(len(in_progress_result['componentStatus']), 0)
self.assertEquals(len(in_progress_result['reports']), 1)
self.assertEquals(in_progress_result['reports'][0]['status'], "IN_PROGRESS")
self.assertEquals(in_progress_result['reports'][0]['stdout'], "Dummy output")
self.assertEquals(in_progress_result['reports'][0]['exitCode'], 777)
self.assertEquals(in_progress_result['reports'][0]['stderr'], 'Dummy err')
self.assertEquals(len(after_start_result['componentStatus']), 0)
self.assertEquals(len(after_start_result['reports']), 1)
self.assertEquals(after_start_result['reports'][0]['status'], "COMPLETED")
self.assertEquals(after_start_result['reports'][0]['stdout'], "returned stdout")
self.assertEquals(after_start_result['reports'][0]['exitCode'], 0)
self.assertEquals(after_start_result['reports'][0]['stderr'], 'returned stderr')
def run_simulation():
Controller.logger = MagicMock()
sendRequest_method = MagicMock()
tmpfile = tempfile.gettempdir()
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', tmpfile)
ver_file = os.path.join(tmpfile, "version")
with open(ver_file, "w") as text_file:
text_file.write(agent_version)
controller = Controller.Controller(config)
controller.sendRequest = sendRequest_method
controller.netutil.HEARTBEAT_IDLE_INTERVAL_DEFAULT_MAX_SEC = 0.1
controller.netutil.HEARTBEAT_NOT_IDDLE_INTERVAL_SEC = 0.1
controller.range = 1
for responce in responces:
queue.put(responce)
def send_stub(url, data):
logger.info("Controller sends data to %s :" % url)
logger.info(pprint.pformat(data))
if not queue.empty():
responce = queue.get()
else:
responce = responces[-1]
logger.info("There is no predefined responce available, sleeping for 30 sec")
time.sleep(30)
responce = json.loads(responce)
responseId.inc()
responce["responseId"] = responseId.val()
responce = json.dumps(responce)
logger.info("Returning data to Controller:" + responce)
return responce
sendRequest_method.side_effect = send_stub
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s %(filename)s:%(lineno)d - \
%(message)s")
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)
logger.addHandler(stream_handler)
logger.info("Starting")
controller.start()
controller.actionQueue.IDLE_SLEEP_TIME = 0.1
controller.run()
def test_dump_command_to_json_with_retry(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations':{'global' : {}},
'configurationTags':{'global' : { 'tag': 'v1' }},
'clusterHostInfo':{'namenode_host' : ['1'],
'slave_hosts' : ['0', '1'],
'all_racks' : [u'/default-rack:0'],
'ambari_server_host' : 'a.b.c',
'ambari_server_port' : '123',
'ambari_server_use_ssl' : 'false',
'all_ipv4_ips' : [u'192.168.12.101:0'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']},
'hostLevelParams':{}
}
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Test dumping STATUS_COMMAND
json_file = orchestrator.dump_command_to_json(command, True)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertEquals(command['agentConfigParams']['agent']['parallel_execution'], 0)
self.assertTrue(unlink_mock.called)
def test_update_open_files_ulimit(self):
# get the current soft and hard limits
(soft_limit, hard_limit) = resource.getrlimit(resource.RLIMIT_NOFILE)
# update will be successful only if the new value is >= soft limit
if hard_limit != resource.RLIM_INFINITY:
open_files_ulimit = soft_limit + (hard_limit - soft_limit) / 2
else:
open_files_ulimit = soft_limit
config = AmbariConfig()
config.set_ulimit_open_files(open_files_ulimit)
main.update_open_files_ulimit(config)
(soft_limit, hard_limit) = resource.getrlimit(resource.RLIMIT_NOFILE)
self.assertEquals(hard_limit, open_files_ulimit)
def test_read_empty(self):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
handler = ActualConfigHandler(config, {})
conf_file = open(os.path.join(tmpdir, ActualConfigHandler.CONFIG_NAME), 'w')
conf_file.write("")
conf_file.close()
output = handler.read_actual()
self.assertEquals(None, output)
os.remove(os.path.join(tmpdir, ActualConfigHandler.CONFIG_NAME))
def test_update_log_level(self, basicConfig_mock, setLevel_mock):
config = AmbariConfig().getConfig()
# Testing with default setup (config file does not contain loglevel entry)
# Log level should not be changed
main.update_log_level(config)
self.assertFalse(setLevel_mock.called)
setLevel_mock.reset_mock()
# Testing debug mode
config.set('agent', 'loglevel', 'DEBUG')
main.update_log_level(config)
setLevel_mock.assert_called_with(logging.DEBUG)
setLevel_mock.reset_mock()
# Testing any other mode
config.set('agent', 'loglevel', 'INFO')
main.update_log_level(config)
setLevel_mock.assert_called_with(logging.INFO)
setLevel_mock.reset_mock()
config.set('agent', 'loglevel', 'WRONG')
main.update_log_level(config)
setLevel_mock.assert_called_with(logging.INFO)
def setUp(self):
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
# generate sample config
tmpdir = tempfile.gettempdir()
exec_tmp_dir = os.path.join(tmpdir, 'tmp')
self.config = AmbariConfig()
self.config.config = ConfigParser.RawConfigParser()
self.config.add_section('agent')
self.config.set('agent', 'prefix', tmpdir)
self.config.set('agent', 'cache_dir', "/cachedir")
self.config.add_section('python')
self.config.set('python', 'custom_actions_dir', tmpdir)
def test_execute_python_executor(self, read_stack_version_mock,
resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir,
actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock(
)
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {
'condensed_result':
copy.copy(process_condensed_result),
'handle':
copy.copy(handle),
'command_status':
actionQueue.commandStatuses.get_command_status(
handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(
actionQueue.on_background_command_complete_callback, None,
command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'],
ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(
self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0]['stdout'], 'process_out')
def main(argv=None):
from ambari_agent.ActionQueue import ActionQueue
from ambari_agent.AmbariConfig import AmbariConfig
from ambari_agent.Controller import Controller
cfg = AmbariConfig()
if os.path.exists(AmbariConfig.getConfigFile()):
cfg.read(AmbariConfig.getConfigFile())
else:
raise Exception("No config found, use default")
ctl = Controller(cfg)
actionQueue = ActionQueue(cfg, ctl)
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3',3))
def test_configtags(self):
config = AmbariConfig().getConfig()
tmpfile = tempfile.gettempdir()
config.set('agent', 'prefix', tmpfile)
actionQueue = ActionQueue(config)
actionQueue.IDLE_SLEEP_TIME = 0.01
executor_started_event = threading.Event()
end_executor_event = threading.Event()
actionQueue.puppetExecutor = FakeExecutor(executor_started_event, end_executor_event)
command = {
'commandId': 17,
'role' : "role",
'taskId' : "taskId",
'clusterName' : "clusterName",
'serviceName' : "serviceName",
'status' : 'IN_PROGRESS',
'hostname' : "localhost.localdomain",
'hostLevelParams': "hostLevelParams",
'clusterHostInfo': "clusterHostInfo",
'roleCommand': "roleCommand",
'configurations': "configurations",
'commandType': "EXECUTION_COMMAND",
'configurations':{'global' : {}},
'configurationTags':{'global' : { 'tag': 'v1' }}
}
actionQueue.put(command)
actionQueue.start()
executor_started_event.wait()
end_executor_event.set()
actionQueue.stop()
actionQueue.join()
after_start_result = actionQueue.result()
configname = os.path.join(tmpfile, 'config.json')
self.assertEquals(len(after_start_result['componentStatus']), 0)
self.assertEquals(len(after_start_result['reports']), 1)
self.assertEquals(after_start_result['reports'][0]['status'], "COMPLETED")
self.assertEquals(after_start_result['reports'][0]['stdout'], "returned stdout")
self.assertEquals(after_start_result['reports'][0]['exitCode'], 0)
self.assertEquals(after_start_result['reports'][0]['stderr'], 'returned stderr')
self.assertEquals(len(after_start_result['reports'][0]['configurationTags']), 1)
self.assertEquals(True, os.path.isfile(configname))
os.remove(configname)
def test_watchdog_1(self, kill_process_with_children_mock):
"""
Tests whether watchdog works
"""
subproc_mock = self.subprocess32_mockup()
executor = PythonExecutor("/tmp", AmbariConfig())
_, tmpoutfile = tempfile.mkstemp()
_, tmperrfile = tempfile.mkstemp()
_, tmpstrucout = tempfile.mkstemp()
PYTHON_TIMEOUT_SECONDS = 0.1
kill_process_with_children_mock.side_effect = lambda pid: subproc_mock.terminate(
)
def launch_python_subprocess32_method(command, tmpout, tmperr):
subproc_mock.tmpout = tmpout
subproc_mock.tmperr = tmperr
return subproc_mock
executor.launch_python_subprocess32 = launch_python_subprocess32_method
runShellKillPgrp_method = MagicMock()
runShellKillPgrp_method.side_effect = lambda python: python.terminate()
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = None
callback_method = MagicMock()
thread = Thread(target=executor.run_file,
args=("fake_puppetFile", ["arg1", "arg2"], tmpoutfile,
tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout,
callback_method, '1'))
thread.start()
time.sleep(0.1)
subproc_mock.finished_event.wait()
self.assertEquals(subproc_mock.was_terminated, True,
"subprocess32 should be terminated due to timeout")
self.assertTrue(callback_method.called)
def test_command_execution_depending_on_command_format(
self, CustomServiceOrchestrator_mock, status_update_callback_mock,
custom_ex_runCommand_mock, puppet_runCommand_mock, open_mock,
determine_command_format_version_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
ret = {
'stdout': '',
'stderr': '',
'exitcode': 1,
}
puppet_runCommand_mock.return_value = ret
determine_command_format_version_mock.return_value = \
ActionQueue.COMMAND_FORMAT_V1
actionQueue.execute_command(self.datanode_install_command)
self.assertTrue(puppet_runCommand_mock.called)
self.assertFalse(custom_ex_runCommand_mock.called)
puppet_runCommand_mock.reset_mock()
custom_ex_runCommand_mock.return_value = ret
determine_command_format_version_mock.return_value = \
ActionQueue.COMMAND_FORMAT_V2
actionQueue.execute_command(self.datanode_install_command)
self.assertFalse(puppet_runCommand_mock.called)
self.assertTrue(custom_ex_runCommand_mock.called)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentStatus_mock,
read_stack_version_mock,
determine_command_format_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
build_mock.return_value = "dummy report"
# Check execution ov V1 status command
determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V1
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertFalse(requestComponentStatus_mock.called)
# Check execution ov V2 status command
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0}
determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V2
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
def test_execute_status_command_with_alerts(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentStatus_mock, read_stack_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'err',
'structuredOut': {'alerts': [ {'name': 'flume_alert'} ] }
}
build_mock.return_value = {'somestatusresult': 'aresult'}
actionQueue.execute_status_command(self.status_command_for_alerts)
report = actionQueue.result()
self.assertTrue(requestComponentStatus_mock.called)
self.assertEqual(len(report['componentStatus']), 1)
self.assertTrue(report['componentStatus'][0].has_key('alerts'))
def test_perform_prestart_checks(self, isdir_mock, isfile_mock, exit_mock):
main.config = AmbariConfig().getConfig()
# Trying case if there is another instance running
isfile_mock.return_value = True
isdir_mock.return_value = True
main.perform_prestart_checks()
self.assertTrue(exit_mock.called)
isfile_mock.reset_mock()
isdir_mock.reset_mock()
exit_mock.reset_mock()
# Trying case if agent prefix dir does not exist
isfile_mock.return_value = False
isdir_mock.return_value = False
main.perform_prestart_checks()
self.assertTrue(exit_mock.called)
isfile_mock.reset_mock()
isdir_mock.reset_mock()
exit_mock.reset_mock()
# Trying normal case
isfile_mock.return_value = False
isdir_mock.return_value = True
main.perform_prestart_checks()
self.assertFalse(exit_mock.called)
def main(argv=None):
from ambari_agent.ActionQueue import ActionQueue
from ambari_agent.AmbariConfig import AmbariConfig
from ambari_agent.Controller import Controller
cfg = AmbariConfig()
config_file_path = AmbariConfig.getConfigFile(home_dir="")
if os.path.exists(config_file_path):
cfg.read(config_file_path)
else:
raise Exception("No config found, use default")
ctl = Controller(cfg)
actionQueue = ActionQueue(cfg, ctl)
heartbeat = Heartbeat(actionQueue)
print json.dumps(heartbeat.build('3', 3))
def test_hostname(self):
hostname.cached_hostname = None
hostname.cached_public_hostname = None
config = AmbariConfig()
self.assertEquals(hostname.hostname(config), socket.getfqdn().lower(),
"hostname should equal the socket-based hostname")
pass
def init(self):
"""
Initialize properties
"""
self.config = AmbariConfig.get_resolved_config()
self.is_registered = False
self.metadata_cache = ClusterMetadataCache(
self.config.cluster_cache_dir)
self.topology_cache = ClusterTopologyCache(
self.config.cluster_cache_dir, self.config)
self.host_level_params_cache = ClusterHostLevelParamsCache(
self.config.cluster_cache_dir)
self.configurations_cache = ClusterConfigurationCache(
self.config.cluster_cache_dir)
self.alert_definitions_cache = ClusterAlertDefinitionsCache(
self.config.cluster_cache_dir)
self.configuration_builder = ConfigurationBuilder(self)
self.stale_alerts_monitor = StaleAlertsMonitor(self)
self.file_cache = FileCache(self.config)
self.customServiceOrchestrator = CustomServiceOrchestrator(self)
self.recovery_manager = RecoveryManager(self.config.recovery_cache_dir)
self.commandStatuses = CommandStatusDict(self)
self.action_queue = ActionQueue(self)
self.alert_scheduler_handler = AlertSchedulerHandler(self)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock, requestComponentSecurityState_mock,
requestComponentStatus_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
build_mock.return_value = {'dummy report': '' }
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0 }
requestComponentSecurityState_mock.reset_mock()
requestComponentSecurityState_mock.return_value = 'UNKNOWN'
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = {'dummy report': '',
'securityState' : 'UNKNOWN'}
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
def test_ActionQueueStartStop(self):
actionQueue = ActionQueue(AmbariConfig().getConfig())
actionQueue.start()
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
def test_execute_retryable_command_fail_and_succeed(self, CustomServiceOrchestrator_mock,
sleep_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execution_result_fail_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
execution_result_succ_dict = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'COMPLETED'
}
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = [execution_result_fail_dict, execution_result_succ_dict]
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_any_call(2)
def test_execute_background_command(self, CustomServiceOrchestrator_mock,
runCommand_mock,
read_stack_version_mock):
CustomServiceOrchestrator_mock.return_value = None
CustomServiceOrchestrator.runCommand.return_value = {
'exitcode': 0,
'stdout': 'out-11',
'stderr': 'err-13'
}
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execute_command = copy.deepcopy(self.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
#assert that python execturor start
self.assertTrue(runCommand_mock.called)
runningCommand = actionQueue.commandStatuses.current_state.get(
execute_command['taskId'])
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand[1]['status'],
ActionQueue.IN_PROGRESS_STATUS)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
def test_execute_retryable_command(self, CustomServiceOrchestrator_mock,
sleep_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(3, runCommand_mock.call_count)
self.assertEqual(2, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(2), call(3)], False)
runCommand_mock.assert_has_calls([
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=True, retry=False),
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=False, retry=True),
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=False, retry=True)])
def test_python_command(self):
executor = PythonExecutor("/tmp", AmbariConfig())
command = executor.python_command("script", ["script_param1"])
self.assertEqual(3, len(command))
self.assertTrue("python" in command[0].lower())
self.assertEquals("script", command[1])
self.assertEquals("script_param1", command[2])
def test_process_command(self, execute_command_mock, log_exc_mock):
dummy_controller = MagicMock()
config = AmbariConfig()
config.set('agent', 'tolerate_download_failures', "true")
initializer_module = InitializerModule()
initializer_module.init()
actionQueue = ActionQueue(initializer_module)
execution_command = {
'commandType' : ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType' : ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType' : "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
log_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
log_exc_mock.reset_mock()
execute_command_mock.reset_mock()
log_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(log_exc_mock.called)
log_exc_mock.reset_mock()
actionQueue.process_command(execution_command)
self.assertTrue(log_exc_mock.called)
def test_ambari_config_get(self):
config = AmbariConfig()
#default
self.assertEqual(config.get("security", "keysdir"),
"/tmp/ambari-agent")
#non-default
config.set("security", "keysdir", "/tmp/non-default-path")
self.assertEqual(config.get("security", "keysdir"),
"/tmp/non-default-path")
#whitespace handling
config.set("security", "keysdir", " /tmp/non-stripped")
self.assertEqual(config.get("security", "keysdir"),
"/tmp/non-stripped")
def test_write_empty_client_components(self, write_file_mock):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
tags0 = {"global": "version0", "core-site": "version0"}
tags1 = {"global": "version1", "core-site": "version2"}
tags2 = {"global": "version33", "core-site": "version33"}
clientsToUpdateConfigs1 = []
configTags = {'HDFS_CLIENT': tags0, 'HBASE_CLIENT': tags1}
handler = ActualConfigHandler(config, configTags)
self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
handler.write_client_components('HDFS', tags2, clientsToUpdateConfigs1)
self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
self.assertFalse(write_file_mock.called)
def test_kill_stale_process(self, timeSleepMock, os_killPgMock):
temp_path = AmbariConfig().getConfig().get(
"stack", "installprefix") + '/9999.pid'
file = open(temp_path, 'w')
file.close()
shell.killstaleprocesses()
self.assertFalse(os.path.exists(temp_path))
def test_write_empty_client_components(self, write_file_mock):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
tags0 = {"global": "version0", "core-site": "version0"}
tags1 = {"global": "version1", "core-site": "version2"}
tags2 = {"global": "version33", "core-site": "version33"}
clientsToUpdateConfigs1 = []
configTags = {'HDFS_CLIENT': tags0, 'HBASE_CLIENT': tags1}
handler = ActualConfigHandler(config, configTags)
self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
handler.write_client_components('HDFS', tags2, clientsToUpdateConfigs1)
self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
self.assertFalse(write_file_mock.called)
def __test_execute_python_executor(self, resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
initializer_module = InitializerModule()
initializer_module.init()
initializer_module.config = cfg
initializer_module.metadata_cache.cache_update({CLUSTER_ID:{'clusterLevelParams':{}}}, 'abc')
initializer_module.configurations_cache.cache_update({CLUSTER_ID:{}}, 'abc')
initializer_module.host_level_params_cache.cache_update({CLUSTER_ID:{}}, 'abc')
CustomServiceOrchestrator.runCommand = default_run_command
actionQueue = ActionQueue(initializer_module)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock()
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {'condensed_result' : copy.copy(process_condensed_result),
'handle' : copy.copy(handle),
'command_status' : actionQueue.commandStatuses.get_command_status(handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(actionQueue.on_background_command_complete_callback,
None, command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty();
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'], ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(reports), 1)
self.assertEqual(reports[0]['stdout'], 'process_out')
def test_ambari_config_get(self):
config = AmbariConfig()
#default
self.assertEqual(config.get("security", "keysdir"), "/tmp/ambari-agent")
#non-default
config.set("security", "keysdir", "/tmp/non-default-path")
self.assertEqual(config.get("security", "keysdir"), "/tmp/non-default-path")
#whitespace handling
config.set("security", "keysdir", " /tmp/non-stripped")
self.assertEqual(config.get("security", "keysdir"), "/tmp/non-stripped")
def test_registration_build(self, get_os_version_mock, get_os_type_mock,
run_os_cmd_mock):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
run_os_cmd_mock.return_value = (3, "", "")
register = Register(config)
reference_version = '2.1.0'
data = register.build(reference_version, 1)
#print ("Register: " + pprint.pformat(data))
self.assertEquals(
len(data['hardwareProfile']) > 0, True,
"hardwareProfile should contain content")
self.assertEquals(data['hostname'] != "", True,
"hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True,
"publicHostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True,
"timestamp should not be empty")
self.assertEquals(
len(data['agentEnv']) > 0, True, "agentEnv should not be empty")
self.assertEquals(data['agentVersion'], reference_version,
"agentVersion should not be empty")
print data['agentEnv']['umask']
self.assertEquals(not data['agentEnv']['umask'] == "", True,
"agents umask should not be empty")
self.assertEquals(data['currentPingPort'] == 33777, True,
"current ping port should be 33777")
self.assertEquals(data['prefix'], config.get('agent', 'prefix'),
'The prefix path does not match')
self.assertEquals(len(data), 9)
def test_build(self, read_actual_component_mock, isdir_mock):
isdir_mock.return_value = False
for component in LiveStatus.COMPONENTS:
config = AmbariConfig().getConfig()
config.set('agent', 'prefix',
"ambari_agent" + os.sep + "dummy_files")
livestatus = LiveStatus('', component['serviceName'],
component['componentName'], {}, config, {})
livestatus.versionsHandler.versionsFilePath = "ambari_agent" + os.sep + "dummy_files" + os.sep + "dummy_current_stack"
result = livestatus.build()
print "LiveStatus of {0}: {1}".format(component['serviceName'],
str(result))
self.assertEquals(
len(result) > 0, True, 'Livestatus should not be empty')
if component['componentName'] == 'GANGLIA_SERVER':
self.assertEquals(
result['stackVersion'],
'{"stackName":"HDP","stackVersion":"1.2.2"}',
'Livestatus should contain component stack version')
# Test build status for CLIENT component (in LiveStatus.CLIENT_COMPONENTS)
read_actual_component_mock.return_value = "some tags"
livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', {}, config, {})
result = livestatus.build()
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result.has_key('configurationTags'))
# Test build status with forced_component_status
## Alive
livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', {}, config, {})
result = livestatus.build(
forced_component_status=LiveStatus.LIVE_STATUS)
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result['status'], LiveStatus.LIVE_STATUS)
## Dead
livestatus = LiveStatus('c1', 'HDFS', 'HDFS_CLIENT', {}, config, {})
result = livestatus.build(
forced_component_status=LiveStatus.DEAD_STATUS)
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result['status'], LiveStatus.DEAD_STATUS)
livestatus = LiveStatus('c1', 'TEZ', 'TEZ_CLIENT', {}, config, {})
result = livestatus.build(
forced_component_status=LiveStatus.LIVE_STATUS)
self.assertTrue(len(result) > 0, 'Livestatus should not be empty')
self.assertTrue(result['status'], LiveStatus.LIVE_STATUS)
def test_registration_build(self, get_os_version_mock, get_os_family_mock,
get_os_type_mock, run_os_cmd_mock, Popen_mock):
config = AmbariConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
config.set('agent', 'current_ping_port', '33777')
get_os_family_mock.return_value = "suse"
get_os_type_mock.return_value = "suse"
get_os_version_mock.return_value = "11"
run_os_cmd_mock.return_value = (3, "", "")
from ambari_agent.Register import Register
register = Register(config)
data = register.build()
self.assertEquals(
len(data['hardwareProfile']) > 0, True,
"hardwareProfile should contain content")
self.assertEquals(data['hostname'] != "", True,
"hostname should not be empty")
self.assertEquals(data['publicHostname'] != "", True,
"publicHostname should not be empty")
self.assertEquals(data['id'], -1)
self.assertEquals(data['timestamp'] > 1353678475465L, True,
"timestamp should not be empty")
self.assertEquals(
len(data['agentEnv']) > 0, True, "agentEnv should not be empty")
self.assertEquals(not data['agentEnv']['umask'] == "", True,
"agents umask should not be empty")
self.assertEquals(data['prefix'], config.get('agent', 'prefix'),
'The prefix path does not match')
self.assertEquals(len(data), 10)
def setUp(self):
# disable stdout
out = StringIO.StringIO()
sys.stdout = out
# Create config
self.config = AmbariConfig().getConfig()
# Instantiate CachedHTTPSConnection (skip connect() call)
with patch.object(security.VerifiedHTTPSConnection, "connect"):
self.cachedHTTPSConnection = security.CachedHTTPSConnection(self.config)
def resolve_ambari_config():
config_path = os.path.abspath(AmbariConfig.getConfigFile())
try:
if os.path.exists(config_path):
agent_config.read(config_path)
else:
raise Exception("No config found at %s" % str(config_path))
except Exception, err:
Logger.warn(err)
def test_unregistration_build(self):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
ver_file = os.path.join(tmpdir, "version")
with open(ver_file, "w") as text_file:
text_file.write("1.3.0")
register = Unregister(config)
data = register.build(1)
self.assertEquals(data['hostname'] != "", True, "hostname should not be empty")
self.assertEquals(data['responseId'], 1)
self.assertEquals(data['timestamp'] > 1353678475465L, True, "timestamp should not be empty")
self.assertEquals(data['agentVersion'], '1.3.0', "agentVersion should not be empty")
self.assertEquals(len(data), 4)
os.remove(ver_file)
def test_build(self):
for component in LiveStatus.COMPONENTS:
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', "dummy_files")
livestatus = LiveStatus('', component['serviceName'],
component['componentName'], {}, config)
livestatus.versionsHandler.versionsFilePath = os.path.join(
"dummy_files", "dummy_current_stack")
result = livestatus.build()
print "LiveStatus of {0}: {1}".format(component['serviceName'],
str(result))
self.assertEquals(
len(result) > 0, True, 'Livestatus should not be empty')
if component['componentName'] == 'GANGLIA_SERVER':
self.assertEquals(
result['stackVersion'],
'{"stackName":"HDP","stackVersion":"1.2.2"}',
'Livestatus should contain component stack version')
def test_read_actual_component_inmemory(self, read_file_mock, write_file_mock):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
tags1 = { "global": "version1", "core-site": "version2" }
read_file_mock.return_value = tags1
handler = ActualConfigHandler(config, {})
handler.write_actual_component('NAMENODE', tags1)
self.assertTrue(write_file_mock.called)
self.assertEquals(tags1, handler.read_actual_component('NAMENODE'))
self.assertFalse(read_file_mock.called)
self.assertEquals(tags1, handler.read_actual_component('DATANODE'))
self.assertTrue(read_file_mock.called)
self.assertEquals(1, read_file_mock.call_count)
self.assertEquals(tags1, handler.read_actual_component('DATANODE'))
self.assertEquals(1, read_file_mock.call_count)
def test_server_hostname_override(self):
hostname.cached_server_hostname = None
fd = tempfile.mkstemp(text=True)
tmpname = fd[1]
os.close(fd[0])
os.chmod(tmpname, os.stat(tmpname).st_mode | stat.S_IXUSR)
tmpfile = file(tmpname, "w+")
config = AmbariConfig()
try:
tmpfile.write("#!/bin/sh\n\necho 'test.example.com'")
tmpfile.close()
config.set('server', 'hostname_script', tmpname)
self.assertEquals(hostname.server_hostname(config), 'test.example.com', "expected hostname 'test.example.com'")
finally:
os.remove(tmpname)
config.remove_option('server', 'hostname_script')
pass
def test_build_predefined(self, getStatus_mock, read_actual_component_mock):
read_actual_component_mock.return_value = "actual_component"
"""
Tests that if live status us defined (using default parameter),
then no StatusCheck is executed
"""
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', "ambari_agent" + os.sep + "dummy_files")
livestatus = LiveStatus('', 'SOME_UNKNOWN_SERVICE',
'SOME_UNKNOWN_COMPONENT', {}, config, {})
livestatus.versionsHandler.versionsFilePath = "ambari_agent" + \
os.sep + "dummy_files" + os.sep + "dummy_current_stack"
result = livestatus.build(forsed_component_status = "STARTED")
result_str = pprint.pformat(result)
self.assertEqual(result_str,
"{'clusterName': '',\n "
"'componentName': 'SOME_UNKNOWN_COMPONENT',\n "
"'configurationTags': 'actual_component',\n "
"'msg': '',\n 'serviceName': 'SOME_UNKNOWN_SERVICE',\n "
"'stackVersion': '',\n 'status': 'STARTED'}")
self.assertFalse(getStatus_mock.called)
def test_server_hostnames_multiple_override(self):
hostname.cached_server_hostnames = []
fd = tempfile.mkstemp(text=True)
tmpname = fd[1]
os.close(fd[0])
os.chmod(tmpname, os.stat(tmpname).st_mode | stat.S_IXUSR)
tmpfile = file(tmpname, "w+")
config = AmbariConfig()
try:
tmpfile.write("#!/bin/sh\n\necho 'host1.example.com, host2.example.com, host3.example.com'")
tmpfile.close()
config.set('server', 'hostname_script', tmpname)
expected_hostnames = ['host1.example.com', 'host2.example.com', 'host3.example.com']
server_hostnames = hostname.server_hostnames(config)
self.assertEquals(server_hostnames, expected_hostnames, "expected hostnames {0}; got {1}".format(expected_hostnames, server_hostnames))
finally:
os.remove(tmpname)
config.remove_option('server', 'hostname_script')
pass
def test_write_actual_component_and_client_components(self):
config = AmbariConfig().getConfig()
tmpdir = tempfile.gettempdir()
config.set('agent', 'prefix', tmpdir)
tags1 = { "global": "version1", "core-site": "version2" }
tags2 = { "global": "version33", "core-site": "version33" }
handler = ActualConfigHandler(config, {})
handler.write_actual_component('HDFS_CLIENT', tags1)
handler.write_actual_component('HBASE_CLIENT', tags1)
self.assertEquals(tags1, handler.read_actual_component('HDFS_CLIENT'))
self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
handler.write_actual_component('DATANODE', tags2)
self.assertEquals(tags2, handler.read_actual_component('DATANODE'))
self.assertEquals(tags1, handler.read_actual_component('HDFS_CLIENT'))
handler.write_client_components('HDFS', tags2)
self.assertEquals(tags2, handler.read_actual_component('HDFS_CLIENT'))
self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
os.remove(os.path.join(tmpdir, "DATANODE_" + ActualConfigHandler.CONFIG_NAME))
os.remove(os.path.join(tmpdir, "HBASE_CLIENT_" + ActualConfigHandler.CONFIG_NAME))
os.remove(os.path.join(tmpdir, "HDFS_CLIENT_" + ActualConfigHandler.CONFIG_NAME))
def test_server_hostname(self):
hostname.cached_server_hostname = None
config = AmbariConfig()
default_server_hostname = config.get('server', 'hostname')
config.set('server', 'hostname', 'ambari-host')
self.assertEquals('ambari-host', hostname.server_hostname(config),
"hostname should equal the socket-based hostname")
config.set('server', 'hostname', default_server_hostname)
pass
def test_server_hostnames(self):
hostname.cached_server_hostnames = []
config = AmbariConfig()
default_server_hostname = config.get('server', 'hostname')
config.set('server', 'hostname', 'ambari-host')
server_hostnames = hostname.server_hostnames(config)
self.assertEquals(['ambari-host'], server_hostnames,
"expected host name ['ambari-host']; got {0}".format(server_hostnames))
config.set('server', 'hostname', default_server_hostname)
pass
