def test_heartbeat_with_status_multiple(self, read_stack_version_method):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
actionQueue.IDLE_SLEEP_TIME = 0.01
read_stack_version_method.return_value="1.3.0"
heartbeat = Heartbeat(actionQueue)
actionQueue.start()
max_number_of_status_entries = 0
for i in range(1,5):
statusCommand = {
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "",
"componentName" : "DATANODE",
'configurations':{'global' : {}}
}
actionQueue.put(statusCommand)
time.sleep(0.1)
result = heartbeat.build(101)
number_of_status_entries = len(result['componentStatus'])
# print "Heartbeat with status: " + str(result) + " XXX " + str(number_of_status_entries)
if max_number_of_status_entries < number_of_status_entries:
max_number_of_status_entries = number_of_status_entries
actionQueue.stop()
actionQueue.join()
NUMBER_OF_COMPONENTS = 1
self.assertEquals(max_number_of_status_entries == NUMBER_OF_COMPONENTS, True)
def test_execute_status_command_with_alerts(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentStatus_mock, read_stack_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'err',
'structuredOut': {'alerts': [ {'name': 'flume_alert'} ] }
}
build_mock.return_value = {'somestatusresult': 'aresult'}
actionQueue.execute_status_command(self.status_command_for_alerts)
report = actionQueue.result()
self.assertTrue(requestComponentStatus_mock.called)
self.assertEqual(len(report['componentStatus']), 1)
self.assertTrue(report['componentStatus'][0].has_key('alerts'))
def test_heartbeat_with_task_in_progress(self):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
actionQueue.commandInProgress= {
'role' : "role",
'actionId' : "actionId",
'taskId' : "taskId",
'stdout' : "stdout",
'clusterName' : "clusterName",
'stderr' : 'none',
'exitCode' : 777,
'serviceName' : "serviceName",
'status' : 'IN_PROGRESS',
'configurations':{'global' : {}},
'roleCommand' : 'START'
}
heartbeat = Heartbeat(actionQueue)
result = heartbeat.build(100)
#print "Heartbeat: " + str(result)
self.assertEquals(len(result['reports']), 1)
self.assertEquals(result['reports'][0]['role'], "role")
self.assertEquals(result['reports'][0]['actionId'], "actionId")
self.assertEquals(result['reports'][0]['taskId'], "taskId")
self.assertEquals(result['reports'][0]['stdout'], "...")
self.assertEquals(result['reports'][0]['clusterName'], "clusterName")
self.assertEquals(result['reports'][0]['stderr'], "...")
self.assertEquals(result['reports'][0]['exitCode'], 777)
self.assertEquals(result['reports'][0]['serviceName'], "serviceName")
self.assertEquals(result['reports'][0]['status'], "IN_PROGRESS")
self.assertEquals(result['reports'][0]['roleCommand'], "START")
pass
def test_heartbeat_no_host_check_cmd_in_queue(self, register_mock,
Popen_mock):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
statusCommand = {
"serviceName": 'HDFS',
"commandType": "STATUS_COMMAND",
"clusterName": "c1",
"componentName": "DATANODE",
"role": "DATANODE",
'configurations': {
'global': {}
}
}
actionQueue.put_status([statusCommand])
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertFalse(args[2])
self.assertFalse(args[1])
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock, requestComponentSecurityState_mock,
requestComponentStatus_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
build_mock.return_value = {'dummy report': '' }
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0 }
requestComponentSecurityState_mock.reset_mock()
requestComponentSecurityState_mock.return_value = 'UNKNOWN'
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = {'dummy report': '',
'securityState' : 'UNKNOWN'}
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
def test_execute_retryable_command_fail_and_succeed(self, CustomServiceOrchestrator_mock,
sleep_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execution_result_fail_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
execution_result_succ_dict = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'COMPLETED'
}
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = [execution_result_fail_dict, execution_result_succ_dict]
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_any_call(2)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock, requestComponentSecurityState_mock,
requestComponentStatus_mock, read_stack_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
build_mock.return_value = {'dummy report': '' }
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0 }
requestComponentSecurityState_mock.reset_mock()
requestComponentSecurityState_mock.return_value = 'UNKNOWN'
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = {'dummy report': '',
'securityState' : 'UNKNOWN'}
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
def test_execute_retryable_command(self, CustomServiceOrchestrator_mock,
sleep_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(3, runCommand_mock.call_count)
self.assertEqual(2, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(2), call(3)], False)
runCommand_mock.assert_has_calls([
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=True, retry=False),
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=False, retry=True),
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=False, retry=True)])
def test_execute_retryable_command_with_time_lapse(self, CustomServiceOrchestrator_mock,
read_stack_version_mock, sleep_mock, time_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
time_mock.side_effect = [4, 8, 10, 14, 18, 22]
def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(2)], False)
runCommand_mock.assert_has_calls([
call(command, '/tmp/ambari-agent/output-19.txt', '/tmp/ambari-agent/errors-19.txt', override_output_files=True, retry=False),
call(command, '/tmp/ambari-agent/output-19.txt', '/tmp/ambari-agent/errors-19.txt', override_output_files=False, retry=True)])
def test_installAndConfigAction(self):
action={'id' : 'tttt'}
actionQueue = ActionQueue(AmbariConfig().getConfig())
path = actionQueue.getInstallFilename(action['id'])
configFile = {
"data" : "test",
"owner" : os.getuid(),
"group" : os.getgid() ,
"permission" : 0700,
"path" : path,
"umask" : 022
}
#note that the command in the action is just a listing of the path created
#we just want to ensure that 'ls' can run on the data file (in the actual world
#this 'ls' would be a puppet or a chef command that would work on a data
#file
path=getFilePath(action,path)
action = {
'id' : 'tttt',
'kind' : 'INSTALL_AND_CONFIG_ACTION',
'workDirComponent' : 'abc-hdfs',
'file' : configFile,
'clusterDefinitionRevision' : 12,
'command' : ['/bin/ls',path]
}
result = { }
actionQueue = ActionQueue(AmbariConfig().getConfig())
result = actionQueue.installAndConfigAction(action)
cmdResult = result['commandResult']
self.assertEqual(cmdResult['exitCode'], 0, "installAndConfigAction test failed. Returned %d " % cmdResult['exitCode'])
self.assertEqual(cmdResult['output'], path + "\n", "installAndConfigAction test failed Returned %s " % cmdResult['output'])
def test_execute_retryable_command_fail_and_succeed(self, CustomServiceOrchestrator_mock,
read_stack_version_mock, sleep_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execution_result_fail_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
execution_result_succ_dict = {
'exitcode': 0,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'COMPLETED'
}
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = [execution_result_fail_dict, execution_result_succ_dict]
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_any_call(2)
def test_heartbeat_with_task_in_progress(self):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
actionQueue.commandInProgress = {
'role': "role",
'actionId': "actionId",
'taskId': "taskId",
'stdout': "stdout",
'clusterName': "clusterName",
'stderr': 'none',
'exitCode': 777,
'serviceName': "serviceName",
'status': 'IN_PROGRESS',
'configurations': {
'global': {}
},
'roleCommand': 'START'
}
heartbeat = Heartbeat(actionQueue)
result = heartbeat.build(100)
#print "Heartbeat: " + str(result)
self.assertEquals(len(result['reports']), 1)
self.assertEquals(result['reports'][0]['role'], "role")
self.assertEquals(result['reports'][0]['actionId'], "actionId")
self.assertEquals(result['reports'][0]['taskId'], "taskId")
self.assertEquals(result['reports'][0]['stdout'], "...")
self.assertEquals(result['reports'][0]['clusterName'], "clusterName")
self.assertEquals(result['reports'][0]['stderr'], "...")
self.assertEquals(result['reports'][0]['exitCode'], 777)
self.assertEquals(result['reports'][0]['serviceName'], "serviceName")
self.assertEquals(result['reports'][0]['status'], "IN_PROGRESS")
self.assertEquals(result['reports'][0]['roleCommand'], "START")
pass
def test_execute_python_executor(self, read_stack_version_mock,
resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir,
actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock(
)
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {
'condensed_result':
copy.copy(process_condensed_result),
'handle':
copy.copy(handle),
'command_status':
actionQueue.commandStatuses.get_command_status(
handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(
actionQueue.on_background_command_complete_callback, None,
command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'],
ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(
self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0]['stdout'], 'process_out')
def test_ActionQueueStartStop(self):
actionQueue = ActionQueue(AmbariConfig().getConfig())
actionQueue.IDLE_SLEEP_TIME = 0.01
actionQueue.start()
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
def test_ActionQueueStartStop(self): actionQueue = ActionQueue(AmbariConfig().getConfig()) actionQueue.IDLE_SLEEP_TIME = 0.01 actionQueue.start() actionQueue.stop() actionQueue.join() self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
def test_parallel_exec_no_retry(self, CustomServiceOrchestrator_mock,
process_command_mock, gpeo_mock, threading_mock):
CustomServiceOrchestrator_mock.return_value = None
initializer_module = InitializerModule()
initializer_module.init()
dummy_controller = MagicMock(initializer_module)
config = MagicMock()
gpeo_mock.return_value = 1
config.get_parallel_exec_option = gpeo_mock
initializer_module = InitializerModule()
initializer_module.init()
actionQueue = ActionQueue(initializer_module)
actionQueue.put([self.datanode_install_no_retry_command, self.snamenode_install_command])
self.assertEqual(2, actionQueue.commandQueue.qsize())
actionQueue.start()
time.sleep(1)
initializer_module.stop_event.set()
actionQueue.join()
self.assertEqual(actionQueue.is_alive(), False, 'Action queue is not stopped.')
self.assertEqual(2, process_command_mock.call_count)
self.assertEqual(0, threading_mock.call_count)
process_command_mock.assert_any_calls([call(self.datanode_install_command), call(self.hbase_install_command)])
def test_execute_background_command(self, CustomServiceOrchestrator_mock,
runCommand_mock,
read_stack_version_mock):
CustomServiceOrchestrator_mock.return_value = None
CustomServiceOrchestrator.runCommand.return_value = {
'exitcode': 0,
'stdout': 'out-11',
'stderr': 'err-13'
}
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execute_command = copy.deepcopy(self.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
actionQueue.processStatusCommandQueueSafeEmpty()
#assert that python execturor start
self.assertTrue(runCommand_mock.called)
runningCommand = actionQueue.commandStatuses.current_state.get(
execute_command['taskId'])
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand[1]['status'],
ActionQueue.IN_PROGRESS_STATUS)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 1)
def test_execute_status_command(self, build_mock, execute_command_mock,
read_stack_version_mock,
status_update_callback):
actionQueue = ActionQueue(AmbariConfig().getConfig(),
'dummy_controller')
build_mock.return_value = "dummy report"
# Try normal execution
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
def test_ActionQueueStartStop(self, get_mock, process_command_mock):
actionQueue = ActionQueue(AmbariConfig().getConfig(),
'dummy_controller')
actionQueue.start()
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True,
'Action queue is not stopped.')
self.assertTrue(process_command_mock.call_count > 1)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentStatus_mock,
read_stack_version_mock,
determine_command_format_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
build_mock.return_value = "dummy report"
# Check execution ov V1 status command
determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V1
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertFalse(requestComponentStatus_mock.called)
# Check execution ov V2 status command
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0}
determine_command_format_version_mock.return_value = ActionQueue.COMMAND_FORMAT_V2
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
def test_status_command_without_globals_section(self, stopped_method,
read_stack_version_method):
config = AmbariConfig().getConfig()
config.set('agent', 'prefix',
TestStackVersionsFileHandler.dummyVersionsFile)
queue = ActionQueue(config)
statusCommand = {
"serviceName": 'HDFS',
"commandType": "STATUS_COMMAND",
"clusterName": "",
"componentName": "DATANODE",
'configurations': {}
}
queue.stopped = stopped_method
stopped_method.side_effect = [False, False, True, True, True]
read_stack_version_method.return_value = "1.3.0"
queue.IDLE_SLEEP_TIME = 0.001
queue.put(statusCommand)
queue.run()
returned_result = queue.resultQueue.get()
returned_result[1]['status'] = 'INSTALLED' # Patch live value
self.assertEquals(returned_result, ('STATUS_COMMAND', {
'clusterName': '',
'componentName': 'DATANODE',
'msg': '',
'serviceName': 'HDFS',
'stackVersion': '1.3.0',
'status': 'INSTALLED'
}))
def test_upgradeCommand_dispatching(self, stopped_method,
executeCommand_method):
queue = ActionQueue(config=MagicMock())
command = {
'commandId': 17,
'role': "role",
'taskId': "taskId",
'clusterName': "clusterName",
'serviceName': "serviceName",
'roleCommand': 'UPGRADE',
'hostname': "localhost.localdomain",
'hostLevelParams': "hostLevelParams",
'clusterHostInfo': "clusterHostInfo",
'configurations': "configurations",
'commandType': "EXECUTION_COMMAND",
'configurations': {
'global': {}
},
'roleParams': {},
'commandParams': {
'source_stack_version': 'HDP-1.2.1',
'target_stack_version': 'HDP-1.3.0'
}
}
result = [{'exitcode': 0, 'stdout': 'abc', 'stderr': 'def'}]
executeCommand_method.return_value = result
stopped_method.side_effect = [False, False, True, True, True]
queue.stopped = stopped_method
queue.IDLE_SLEEP_TIME = 0.001
queue.put(command)
queue.run()
self.assertTrue(executeCommand_method.called)
self.assertEquals(queue.resultQueue.qsize(), 1)
returned_result = queue.resultQueue.get()
self.assertTrue(returned_result[1] is result[0])
def test_ActionQueueStartStop(self, CustomServiceOrchestrator_mock,
get_mock, process_command_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.start()
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
self.assertTrue(process_command_mock.call_count > 1)
def test_run_unrecognized_command(self, logger_method, stopped_method):
config = AmbariConfig().getConfig()
actionQueue = ActionQueue(config)
command = {
"serviceName" : 'HDFS',
"commandType" : "SOME_UNRECOGNIZED_COMMAND",
"clusterName" : "",
"componentName" : "DATANODE",
'configurations':{}
}
actionQueue.commandQueue.put(command)
actionQueue.stopped = stopped_method
stopped_method.side_effect = [False, False, True, True, True]
actionQueue.IDLE_SLEEP_TIME = 0.001
actionQueue.run()
self.assertTrue(logger_method.call_args[0][0].startswith('Unrecognized command'))
def init(self):
"""
Initialize properties
"""
self.config = AmbariConfig.get_resolved_config()
self.is_registered = False
self.metadata_cache = ClusterMetadataCache(
self.config.cluster_cache_dir)
self.topology_cache = ClusterTopologyCache(
self.config.cluster_cache_dir, self.config)
self.host_level_params_cache = ClusterHostLevelParamsCache(
self.config.cluster_cache_dir)
self.configurations_cache = ClusterConfigurationCache(
self.config.cluster_cache_dir)
self.alert_definitions_cache = ClusterAlertDefinitionsCache(
self.config.cluster_cache_dir)
self.configuration_builder = ConfigurationBuilder(self)
self.stale_alerts_monitor = StaleAlertsMonitor(self)
self.file_cache = FileCache(self.config)
self.customServiceOrchestrator = CustomServiceOrchestrator(self)
self.recovery_manager = RecoveryManager(self.config.recovery_cache_dir)
self.commandStatuses = CommandStatusDict(self)
self.action_queue = ActionQueue(self)
self.alert_scheduler_handler = AlertSchedulerHandler(self)
def test_build(self):
testsPath = os.path.dirname(os.path.realpath(__file__))
dictPath = testsPath + os.sep + '..' + os.sep + '..' + os.sep + 'main' + os.sep + 'python' + os.sep + 'ambari_agent' + os.sep + 'servicesToPidNames.dict'
AmbariConfig.config.set('services', 'serviceToPidMapFile', dictPath)
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
heartbeat = Heartbeat(actionQueue)
result = heartbeat.build(100)
def test_no_mapping(self, register_mock, result_mock):
result_mock.return_value = {
'reports': [{
'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 777
}],
'componentStatus': [{
'status': 'HEALTHY',
'componentName': 'NAMENODE'
}]
}
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig(),
'dummy_controller')
heartbeat = Heartbeat(actionQueue)
hb = heartbeat.build(id=10, state_interval=1, componentsMapped=True)
self.assertEqual(register_mock.call_args_list[0][0][1], True)
register_mock.reset_mock()
hb = heartbeat.build(id=0, state_interval=1, componentsMapped=True)
self.assertEqual(register_mock.call_args_list[0][0][1], False)
def test_no_mapping(self, register_mock, result_mock, Popen_mock):
result_mock.return_value = {
'reports': [{'status': 'IN_PROGRESS',
'stderr': 'Read from /tmp/errors-3.txt',
'stdout': 'Read from /tmp/output-3.txt',
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 777}],
'componentStatus': [{'status': 'HEALTHY', 'componentName': 'NAMENODE'}]
}
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue)
hb = heartbeat.build(id = 10, state_interval=1, componentsMapped=True)
self.assertEqual(register_mock.call_args_list[0][0][1], True)
register_mock.reset_mock()
hb = heartbeat.build(id = 0, state_interval=1, componentsMapped=True)
self.assertEqual(register_mock.call_args_list[0][0][1], False)
def test_build(self):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
heartbeat = Heartbeat(actionQueue)
result = heartbeat.build(100)
print "Heartbeat: " + str(result)
self.assertEquals(result['hostname'] != '', True,
"hostname should not be empty")
self.assertEquals(result['responseId'], 100)
self.assertEquals(result['componentStatus'] is not None, True,
"Heartbeat should contain componentStatus")
self.assertEquals(result['reports'] is not None, True,
"Heartbeat should contain reports")
self.assertEquals(result['timestamp'] >= 1353679373880L, True)
self.assertEquals(len(result['nodeStatus']), 2)
self.assertEquals(result['nodeStatus']['cause'], "NONE")
self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
# result may or may NOT have an agentEnv structure in it
self.assertEquals((len(result) is 6) or (len(result) is 7), True)
self.assertEquals(not heartbeat.reports, True,
"Heartbeat should not contain task in progress")
def test_process_command(self, execute_command_mock, log_exc_mock):
dummy_controller = MagicMock()
config = AmbariConfig()
config.set('agent', 'tolerate_download_failures', "true")
initializer_module = InitializerModule()
initializer_module.init()
actionQueue = ActionQueue(initializer_module)
execution_command = {
'commandType' : ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType' : ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType' : "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
log_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
log_exc_mock.reset_mock()
execute_command_mock.reset_mock()
log_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(log_exc_mock.called)
log_exc_mock.reset_mock()
actionQueue.process_command(execution_command)
self.assertTrue(log_exc_mock.called)
def test_store_configuration_tags_no_clients(self,
status_update_callback_mock,
command_status_dict_mock,
cso_runCommand_mock,
write_client_components_mock):
custom_service_orchestrator_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'exitcode': 0
}
cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.execute_command(
self.datanode_restart_command_no_clients_update)
report = actionQueue.result()
expected = {
'status': 'COMPLETED',
'configurationTags': {
'global': {
'tag': 'v123'
}
},
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'CUSTOM_COMMAND',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 9,
'customCommand': 'RESTART',
'exitCode': 0
}
# Agent caches configurationTags if custom_command RESTART completed
self.assertEqual(len(report['reports']), 1)
self.assertEqual(expected, report['reports'][0])
self.assertFalse(write_client_components_mock.called)
def test_determine_command_format_version(self):
v1_command = {'commandParams': {'schema_version': '1.0'}}
v2_command = {'commandParams': {'schema_version': '2.0'}}
current_command = {
# Absent 'commandParams' section
}
actionQueue = ActionQueue(AmbariConfig().getConfig(),
'dummy_controller')
self.assertEqual(
actionQueue.determine_command_format_version(v1_command),
ActionQueue.COMMAND_FORMAT_V1)
self.assertEqual(
actionQueue.determine_command_format_version(v2_command),
ActionQueue.COMMAND_FORMAT_V2)
self.assertEqual(
actionQueue.determine_command_format_version(current_command),
ActionQueue.COMMAND_FORMAT_V1)
def __test_execute_python_executor(self, resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
initializer_module = InitializerModule()
initializer_module.init()
initializer_module.config = cfg
initializer_module.metadata_cache.cache_update({CLUSTER_ID:{'clusterLevelParams':{}}}, 'abc')
initializer_module.configurations_cache.cache_update({CLUSTER_ID:{}}, 'abc')
initializer_module.host_level_params_cache.cache_update({CLUSTER_ID:{}}, 'abc')
CustomServiceOrchestrator.runCommand = default_run_command
actionQueue = ActionQueue(initializer_module)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock()
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {'condensed_result' : copy.copy(process_condensed_result),
'handle' : copy.copy(handle),
'command_status' : actionQueue.commandStatuses.get_command_status(handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(actionQueue.on_background_command_complete_callback,
None, command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty();
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'], ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(reports), 1)
self.assertEqual(reports[0]['stdout'], 'process_out')
def test_parallel_exec(self, CustomServiceOrchestrator_mock,
process_command_mock, gpeo_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
config = MagicMock()
gpeo_mock.return_value = 1
config.get_parallel_exec_option = gpeo_mock
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.put([self.datanode_install_command, self.hbase_install_command])
self.assertEqual(2, actionQueue.commandQueue.qsize())
actionQueue.start()
time.sleep(1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
self.assertEqual(2, process_command_mock.call_count)
process_command_mock.assert_any_calls([call(self.datanode_install_command), call(self.hbase_install_command)])
def test_heartbeat_host_check_no_cmd(self, register_mock):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertFalse(args[1])
self.assertFalse(args[2])
def test_status_command_without_globals_section(self, stopped_method,
read_stack_version_method):
config = AmbariConfig().getConfig()
config.set('agent', 'prefix', TestStackVersionsFileHandler.dummyVersionsFile)
queue = ActionQueue(config)
statusCommand = {
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "",
"componentName" : "DATANODE",
'configurations':{}
}
queue.stopped = stopped_method
stopped_method.side_effect = [False, False, True, True, True]
read_stack_version_method.return_value="1.3.0"
queue.IDLE_SLEEP_TIME = 0.001
queue.put(statusCommand)
queue.run()
returned_result = queue.resultQueue.get()
returned_result[1]['status'] = 'INSTALLED' # Patch live value
self.assertEquals(returned_result, ('STATUS_COMMAND',
{'clusterName': '',
'componentName': 'DATANODE',
'msg': '',
'serviceName': 'HDFS',
'stackVersion': '1.3.0',
'status': 'INSTALLED'}))
def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,
kill_process_with_children_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
# shell.kill_process_with_children = MagicMock()
dummy_controller = MagicMock()
cfg = AmbariConfig().getConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f (a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
TestActionQueue.patch_output_file(orchestrator.python_executor)
orchestrator.python_executor.prepare_process_result = MagicMock()
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited']= ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19,'')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
def test_execute_status_command(self, CustomServiceOrchestrator_mock,
build_mock, execute_command_mock,
requestComponentStatus_mock, read_stack_version_mock,
status_update_callback):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
build_mock.return_value = "dummy report"
requestComponentStatus_mock.reset_mock()
requestComponentStatus_mock.return_value = {'exitcode': 0}
actionQueue.execute_status_command(self.status_command)
report = actionQueue.result()
expected = 'dummy report'
self.assertEqual(len(report['componentStatus']), 1)
self.assertEqual(report['componentStatus'][0], expected)
self.assertTrue(requestComponentStatus_mock.called)
def test_execute_python_executor(self, read_stack_version_mock, resolve_script_path_mock,
get_py_executor_mock):
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir, actionQueue.customServiceOrchestrator.config)
patch_output_file(pyex)
get_py_executor_mock.return_value = pyex
actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock()
result = {}
lock = threading.RLock()
complete_done = threading.Condition(lock)
def command_complete_w(process_condensed_result, handle):
with lock:
result['command_complete'] = {'condensed_result' : copy.copy(process_condensed_result),
'handle' : copy.copy(handle),
'command_status' : actionQueue.commandStatuses.get_command_status(handle.command['taskId'])
}
complete_done.notifyAll()
actionQueue.on_background_command_complete_callback = wraped(actionQueue.on_background_command_complete_callback,
None, command_complete_w)
actionQueue.put([self.background_command])
actionQueue.processBackgroundQueueSafeEmpty();
actionQueue.processStatusCommandQueueSafeEmpty();
with lock:
complete_done.wait(0.1)
finished_status = result['command_complete']['command_status']
self.assertEqual(finished_status['status'], ActionQueue.COMPLETED_STATUS)
self.assertEqual(finished_status['stdout'], 'process_out')
self.assertEqual(finished_status['stderr'], 'process_err')
self.assertEqual(finished_status['exitCode'], 0)
runningCommand = actionQueue.commandStatuses.current_state.get(self.background_command['taskId'])
self.assertTrue(runningCommand is not None)
report = actionQueue.result()
self.assertEqual(len(report['reports']),1)
self.assertEqual(report['reports'][0]['stdout'],'process_out')
def test_determine_command_format_version(self,
CustomServiceOrchestrator_mock):
CustomServiceOrchestrator_mock.return_value = None
v1_command = {'commandParams': {'schema_version': '1.0'}}
v2_command = {'commandParams': {'schema_version': '2.0'}}
current_command = {
# Absent 'commandParams' section
}
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
self.assertEqual(
actionQueue.determine_command_format_version(v1_command),
ActionQueue.COMMAND_FORMAT_V1)
self.assertEqual(
actionQueue.determine_command_format_version(v2_command),
ActionQueue.COMMAND_FORMAT_V2)
self.assertEqual(
actionQueue.determine_command_format_version(current_command),
ActionQueue.COMMAND_FORMAT_V1)
def init_threads(self):
"""
Initialize thread objects
"""
self.component_status_executor = ComponentStatusExecutor(self)
self.action_queue = ActionQueue(self)
self.alert_scheduler_handler = AlertSchedulerHandler(self)
self.command_status_reporter = CommandStatusReporter(self)
self.host_status_reporter = HostStatusReporter(self)
self.alert_status_reporter = AlertStatusReporter(self)
self.heartbeat_thread = HeartbeatThread.HeartbeatThread(self)
def test_heartbeat_no_host_check_cmd_in_queue(self, register_mock):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig(),
'dummy_controller')
statusCommand = {
"serviceName": 'HDFS',
"commandType": "STATUS_COMMAND",
"clusterName": "c1",
"componentName": "DATANODE",
'configurations': {
'global': {}
}
}
actionQueue.put([statusCommand])
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertTrue(args[2])
self.assertFalse(args[1])
def test_status_commands_does_not_stack_up(self, register_mock,
Popen_mock):
config = AmbariConfig.AmbariConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
dummy_controller.statusCommandsExecutor = SingleProcessStatusCommandsExecutor(
config, actionQueue)
statusCommands = [{
"serviceName": 'HDFS',
"commandType": "STATUS_COMMAND",
"clusterName": "c1",
"componentName": "DATANODE",
"role": "DATANODE",
'configurations': {
'cluster-env': {}
}
}, {
"serviceName": 'HDFS',
"commandType": "STATUS_COMMAND",
"clusterName": "c1",
"componentName": "NAMENODE",
"role": "NAMENODE",
'configurations': {
'cluster-env': {}
}
}]
# add commands ten times
for i in range(10):
actionQueue.put_status(statusCommands)
# status commands should not stack up. Size should be 2 not 20.
self.assertEquals(
len(dummy_controller.statusCommandsExecutor.statusCommandQueue.
queue), 2)
def test_heartbeat_no_host_check_cmd_in_progress(self, register_mock):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
actionQueue.commandInProgress= {
'role' : "role",
'actionId' : "actionId",
'taskId' : "taskId",
'stdout' : "stdout",
'clusterName' : "clusterName",
'stderr' : 'none',
'exitCode' : 777,
'serviceName' : "serviceName",
'status' : 'IN_PROGRESS',
'configurations':{'global' : {}},
'roleCommand' : 'START'
}
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertTrue(args[2])
self.assertFalse(args[1])
def test_heartbeat_with_status(self, read_stack_version_method):
actionQueue = ActionQueue(AmbariConfig.AmbariConfig().getConfig())
read_stack_version_method.return_value="1.3.0"
heartbeat = Heartbeat(actionQueue)
statusCommand = {
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "",
"componentName" : "DATANODE",
'configurations':{'global' : {}}
}
actionQueue.put(statusCommand)
actionQueue.start()
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
result = heartbeat.build(101)
self.assertEquals(len(result['componentStatus']) > 0, True, 'Heartbeat should contain status of HDFS components')
def test_heartbeat_no_host_check_cmd_in_queue(self, register_mock):
config = AmbariConfig.AmbariConfig().getConfig()
config.set('agent', 'prefix', 'tmp')
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
statusCommand = {
"serviceName" : 'HDFS',
"commandType" : "STATUS_COMMAND",
"clusterName" : "c1",
"componentName" : "DATANODE",
'configurations':{'global' : {}}
}
actionQueue.put([statusCommand])
heartbeat = Heartbeat(actionQueue)
heartbeat.build(12, 6)
self.assertTrue(register_mock.called)
args, kwargs = register_mock.call_args_list[0]
self.assertTrue(args[2])
self.assertFalse(args[1])
def test_do_not_log_execution_commands(self, status_update_callback_mock,
command_status_dict_mock,
cso_runCommand_mock, mock_log_command_output):
custom_service_orchestrator_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'exitcode': 0
}
cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
config.set('logging', 'log_command_executes', 1)
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.execute_command(self.datanode_restart_command_no_logging)
report = actionQueue.result()
expected = {'status': 'COMPLETED',
'configurationTags': {'global': {'tag': 'v123'}},
'stderr': 'stderr',
'stdout': 'out\n\nCommand completed successfully!\n',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'CUSTOM_COMMAND',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 9,
'customCommand': 'RESTART',
'exitCode': 0}
# Agent caches configurationTags if custom_command RESTART completed
mock_log_command_output.assert_not_called(
[call("out\n\nCommand completed successfully!\n", "9"), call("stderr", "9")], any_order=True)
self.assertEqual(len(report['reports']), 1)
self.assertEqual(expected, report['reports'][0])
def test_startAndStopAction(self):
command = {'script' : 'import os,sys,time\ni = 0\nwhile (i < 1000):\n print "testhello"\n sys.stdout.flush()\n time.sleep(1)\n i+=1',
'param' : ''}
action={'id' : 'ttt',
'kind' : 'START_ACTION',
'clusterId' : 'foobar',
'clusterDefinitionRevision' : 1,
'component' : 'foocomponent',
'role' : 'foorole',
'command' : command,
'user' : getpass.getuser()
}
actionQueue = ActionQueue(AmbariConfig().getConfig())
result = actionQueue.startAction(action)
cmdResult = result['commandResult']
self.assertEqual(cmdResult['exitCode'], 0, "starting a process failed")
shell = actionQueue.getshellinstance()
key = shell.getServerKey(action['clusterId'],action['clusterDefinitionRevision'],
action['component'],action['role'])
keyPresent = True
if not key in serverTracker:
keyPresent = False
self.assertEqual(keyPresent, True, "Key not present")
plauncher = serverTracker[key]
self.assertTrue(plauncher.getpid() > 0, "Pid less than 0!")
time.sleep(5)
shell.stopProcess(key)
keyPresent = False
if key in serverTracker:
keyPresent = True
self.assertEqual(keyPresent, False, "Key present")
processexists = True
try:
os.kill(serverTracker[key].getpid(),0)
except:
processexists = False
self.assertEqual(processexists, False, "Process still exists!")
self.assertTrue("testhello" in plauncher.out, "Output doesn't match!")
def test_execute_retryable_command_with_time_lapse(self, CustomServiceOrchestrator_mock,
sleep_mock, time_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
times_arr = [8, 10, 14, 18, 22, 26, 30, 34]
if self.logger.isEnabledFor(logging.INFO):
times_arr.insert(0, 4)
time_mock.side_effect = times_arr
def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(1)], False)
runCommand_mock.assert_has_calls([
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=True, retry=False),
call(command, os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'output-19.txt',
os.sep + 'tmp' + os.sep + 'ambari-agent' + os.sep + 'errors-19.txt', override_output_files=False, retry=True)])
def test_store_configuration_tags_no_clients(self, status_update_callback_mock,
command_status_dict_mock,
cso_runCommand_mock, write_client_components_mock):
custom_service_orchestrator_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut' : '',
'exitcode' : 0
}
cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.execute_command(self.datanode_restart_command_no_clients_update)
report = actionQueue.result()
expected = {'status': 'COMPLETED',
'configurationTags': {'global': {'tag': 'v123'}},
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'CUSTOM_COMMAND',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 9,
'customCommand': 'RESTART',
'exitCode': 0}
# Agent caches configurationTags if custom_command RESTART completed
self.assertEqual(len(report['reports']), 1)
self.assertEqual(expected, report['reports'][0])
self.assertFalse(write_client_components_mock.called)
def test_execute_background_command(self, CustomServiceOrchestrator_mock,
runCommand_mock, read_stack_version_mock
):
CustomServiceOrchestrator_mock.return_value = None
CustomServiceOrchestrator.runCommand.return_value = {'exitcode' : 0,
'stdout': 'out-11',
'stderr' : 'err-13'}
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
execute_command = copy.deepcopy(self.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty();
actionQueue.processStatusCommandQueueSafeEmpty();
#assert that python execturor start
self.assertTrue(runCommand_mock.called)
runningCommand = actionQueue.commandStatuses.current_state.get(execute_command['taskId'])
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand[1]['status'], ActionQueue.IN_PROGRESS_STATUS)
report = actionQueue.result()
self.assertEqual(len(report['reports']),1)
def test_upgradeCommand_dispatching(self, stopped_method, executeCommand_method):
queue = ActionQueue(config = MagicMock())
command = {
'commandId': 17,
'role' : "role",
'taskId' : "taskId",
'clusterName' : "clusterName",
'serviceName' : "serviceName",
'roleCommand' : 'UPGRADE',
'hostname' : "localhost.localdomain",
'hostLevelParams': "hostLevelParams",
'clusterHostInfo': "clusterHostInfo",
'configurations': "configurations",
'commandType': "EXECUTION_COMMAND",
'configurations':{'global' : {}},
'roleParams': {},
'commandParams' : {
'source_stack_version' : 'HDP-1.2.1',
'target_stack_version' : 'HDP-1.3.0'
}
}
result = [{
'exitcode' : 0,
'stdout' : 'abc',
'stderr' : 'def'
}]
executeCommand_method.return_value = result
stopped_method.side_effect = [False, False, True, True, True]
queue.stopped = stopped_method
queue.IDLE_SLEEP_TIME = 0.001
queue.put(command)
queue.run()
self.assertTrue(executeCommand_method.called)
self.assertEquals(queue.resultQueue.qsize(), 1)
returned_result = queue.resultQueue.get()
self.assertTrue(returned_result[1] is result[0])
def test_command_in_progress(self):
config = AmbariConfig().getConfig()
tmpfile = tempfile.gettempdir()
config.set('agent', 'prefix', tmpfile)
actionQueue = ActionQueue(config)
actionQueue.IDLE_SLEEP_TIME = 0.01
executor_started_event = threading.Event()
end_executor_event = threading.Event()
actionQueue.puppetExecutor = FakeExecutor(executor_started_event, end_executor_event)
before_start_result = actionQueue.result()
command = {
'commandId': 17,
'role' : "role",
'taskId' : "taskId",
'clusterName' : "clusterName",
'serviceName' : "serviceName",
'status' : 'IN_PROGRESS',
'hostname' : "localhost.localdomain",
'hostLevelParams': "hostLevelParams",
'clusterHostInfo': "clusterHostInfo",
'roleCommand': "roleCommand",
'configurations': "configurations",
'commandType': "EXECUTION_COMMAND",
'configurations':{'global' : {}}
}
actionQueue.put(command)
actionQueue.start()
executor_started_event.wait()
#print ("ii: " + pprint.pformat(actionQueue.commandInProgress))
in_progress_result = actionQueue.result()
end_executor_event.set()
actionQueue.stop()
actionQueue.join()
after_start_result = actionQueue.result()
self.assertEquals(len(before_start_result['componentStatus']), 0)
self.assertEquals(len(before_start_result['reports']), 0)
self.assertEquals(len(in_progress_result['componentStatus']), 0)
self.assertEquals(len(in_progress_result['reports']), 1)
self.assertEquals(in_progress_result['reports'][0]['status'], "IN_PROGRESS")
self.assertEquals(in_progress_result['reports'][0]['stdout'], "Dummy output")
self.assertEquals(in_progress_result['reports'][0]['exitCode'], 777)
self.assertEquals(in_progress_result['reports'][0]['stderr'], 'Dummy err')
self.assertEquals(len(after_start_result['componentStatus']), 0)
self.assertEquals(len(after_start_result['reports']), 1)
self.assertEquals(after_start_result['reports'][0]['status'], "COMPLETED")
self.assertEquals(after_start_result['reports'][0]['stdout'], "returned stdout")
self.assertEquals(after_start_result['reports'][0]['exitCode'], 0)
self.assertEquals(after_start_result['reports'][0]['stderr'], 'returned stderr')
def test_execute_command(self, runCommand_mock, status_update_callback_mock, open_mock):
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut' : ''
}
def side_effect(command, tmpoutfile, tmperrfile):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
# with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
### Test install/start/stop command ###
## Test successful execution with configuration tags
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
# We call method in a separate thread
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_install_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
report = actionQueue.result()
if len(report['reports']) != 0:
break
expected = {'status': 'IN_PROGRESS',
'stderr': 'Read from {0}/errors-3.txt'.format(tempdir),
'stdout': 'Read from {0}/output-3.txt'.format(tempdir),
'structuredOut' : 'Read from {0}/structured-out-3.json'.format(tempdir),
'clusterName': u'cc',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 777}
self.assertEqual(report['reports'][0], expected)
# Continue command execution
unfreeze_flag.set()
# wait until ready
while report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
configname = os.path.join(tempdir, 'config.json')
expected = {'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'configurationTags': {'global': {'tag': 'v1'}},
'exitCode': 0}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
self.assertTrue(os.path.isfile(configname))
# Check that we had 2 status update calls ( IN_PROGRESS and COMPLETE)
self.assertEqual(status_update_callback_mock.call_count, 2)
os.remove(configname)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_install_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {'status': 'FAILED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': u'cc',
'structuredOut': '""',
'roleCommand': u'INSTALL',
'serviceName': u'HDFS',
'role': u'DATANODE',
'actionId': '1-1',
'taskId': 3,
'exitCode': 13}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
### Test upgrade command ###
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_upgrade_command, ))
execution_thread.start()
unfreeze_flag.set()
# wait until ready
report = actionQueue.result()
while len(report['reports']) == 0 or \
report['reports'][0]['status'] == 'IN_PROGRESS':
time.sleep(0.1)
report = actionQueue.result()
# check report
expected = {'status': 'COMPLETED',
'stderr': 'stderr',
'stdout': 'out',
'clusterName': 'clusterName',
'structuredOut': '""',
'roleCommand': 'UPGRADE',
'serviceName': 'serviceName',
'role': 'role',
'actionId': 17,
'taskId': 'taskId',
'exitCode': 0}
self.assertEqual(len(report['reports']), 1)
self.assertEqual(report['reports'][0], expected)
# now should not have reports (read complete/failed reports are deleted)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
def test_process_command(self, execute_status_command_mock,
execute_command_mock, print_exc_mock):
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
execution_command = {
'commandType' : ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType' : ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType' : "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
actionQueue.process_command(status_command)
self.assertFalse(execute_command_mock.called)
self.assertTrue(execute_status_command_mock.called)
self.assertFalse(print_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
print_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
print_exc_mock.reset_mock()
execute_status_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(print_exc_mock.called)
def test_process_command(self, execute_status_command_mock,
execute_command_mock, log_exc_mock):
dummy_controller = MagicMock()
config = AmbariConfig()
config.set('agent', 'tolerate_download_failures', "true")
actionQueue = ActionQueue(config, dummy_controller)
execution_command = {
'commandType' : ActionQueue.EXECUTION_COMMAND,
}
status_command = {
'commandType' : ActionQueue.STATUS_COMMAND,
}
wrong_command = {
'commandType' : "SOME_WRONG_COMMAND",
}
# Try wrong command
actionQueue.process_command(wrong_command)
self.assertFalse(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
log_exc_mock.reset_mock()
# Try normal execution
actionQueue.process_command(execution_command)
self.assertTrue(execute_command_mock.called)
self.assertFalse(execute_status_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
log_exc_mock.reset_mock()
actionQueue.process_command(status_command)
self.assertFalse(execute_command_mock.called)
self.assertTrue(execute_status_command_mock.called)
self.assertFalse(log_exc_mock.called)
execute_command_mock.reset_mock()
execute_status_command_mock.reset_mock()
log_exc_mock.reset_mock()
# Try exception to check proper logging
def side_effect(self):
raise Exception("TerribleException")
execute_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(log_exc_mock.called)
log_exc_mock.reset_mock()
execute_status_command_mock.side_effect = side_effect
actionQueue.process_command(execution_command)
self.assertTrue(log_exc_mock.called)
def test_auto_execute_command(self, status_update_callback_mock, open_mock):
# Make file read calls visible
def open_side_effect(file, mode):
if mode == 'r':
file_mock = MagicMock()
file_mock.read.return_value = "Read from " + str(file)
return file_mock
else:
return self.original_open(file, mode)
open_mock.side_effect = open_side_effect
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
config.set('agent', 'tolerate_download_failures', "true")
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
dummy_controller.recovery_manager.update_config(5, 5, 1, 11, True, False, "", "")
actionQueue = ActionQueue(config, dummy_controller)
unfreeze_flag = threading.Event()
python_execution_result_dict = {
'stdout': 'out',
'stderr': 'stderr',
'structuredOut' : ''
}
def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
unfreeze_flag.wait()
return python_execution_result_dict
def patched_aq_execute_command(command):
# We have to perform patching for separate thread in the same thread
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.process_command(command)
python_execution_result_dict['status'] = 'COMPLETE'
python_execution_result_dict['exitcode'] = 0
self.assertFalse(actionQueue.tasks_in_progress_or_pending())
# We call method in a separate thread
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_auto_start_command, ))
execution_thread.start()
# check in progress report
# wait until ready
while True:
time.sleep(0.1)
if actionQueue.tasks_in_progress_or_pending():
break
# Continue command execution
unfreeze_flag.set()
# wait until ready
while actionQueue.tasks_in_progress_or_pending():
time.sleep(0.1)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
## Test failed execution
python_execution_result_dict['status'] = 'FAILED'
python_execution_result_dict['exitcode'] = 13
# We call method in a separate thread
execution_thread = Thread(target = patched_aq_execute_command ,
args = (self.datanode_auto_start_command, ))
execution_thread.start()
unfreeze_flag.set()
# check in progress report
# wait until ready
report = actionQueue.result()
while actionQueue.tasks_in_progress_or_pending():
time.sleep(0.1)
report = actionQueue.result()
self.assertEqual(len(report['reports']), 0)
def test_reset_queue(self, CustomServiceOrchestrator_mock,
get_mock, process_command_mock, gpeo_mock):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
dummy_controller.recovery_manager = RecoveryManager(tempfile.mktemp())
config = MagicMock()
gpeo_mock.return_value = 0
config.get_parallel_exec_option = gpeo_mock
actionQueue = ActionQueue(config, dummy_controller)
actionQueue.start()
actionQueue.put([self.datanode_install_command, self.hbase_install_command])
self.assertEqual(2, actionQueue.commandQueue.qsize())
self.assertTrue(actionQueue.tasks_in_progress_or_pending())
actionQueue.reset()
self.assertTrue(actionQueue.commandQueue.empty())
self.assertFalse(actionQueue.tasks_in_progress_or_pending())
time.sleep(0.1)
actionQueue.stop()
actionQueue.join()
self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
