def test_dump_command_to_json_with_retry(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'clusterHostInfo': {
'namenode_host': ['1'],
'slave_hosts': ['0', '1'],
'all_racks': [u'/default-rack:0'],
'ambari_server_host': 'a.b.c',
'all_ipv4_ips': [u'192.168.12.101:0'],
'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']
},
'hostLevelParams': {}
}
config = AmbariConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Test dumping STATUS_COMMAND
json_file = orchestrator.dump_command_to_json(command, True)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertEquals(
command['agentConfigParams']['agent']['parallel_execution'], 0)
self.assertTrue(unlink_mock.called)
def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock,
decompress_cluster_host_info_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations':{'global' : {}},
'configurationTags':{'global' : { 'tag': 'v1' }},
'clusterHostInfo':{'namenode_host' : ['1'],
'slave_hosts' : ['0', '1'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']},
'hostLevelParams':{}
}
decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670', '8670']}
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
self.assertTrue(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType']='STATUS_COMMAND'
decompress_cluster_host_info_mock.reset_mock()
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
if get_platform() != PLATFORM_WINDOWS:
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
self.assertFalse(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
def test_dump_command_to_json(self, unlink_mock, isfile_mock,
hostname_mock):
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'componentName': 'NAMENODE',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v1'
}
},
'clusterHostInfo': {
'namenode_host': ['1'],
'slave_hosts': ['0', '1'],
'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']
}
}
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType'] = 'STATUS_COMMAND'
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
def test_dump_command_to_json(self, FileCache_mock, unlink_mock,
isfile_mock, hostname_mock,
decompress_cluster_host_info_mock):
FileCache_mock.return_value = None
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations':{'global' : {}},
'configurationTags':{'global' : { 'tag': 'v1' }},
'clusterHostInfo':{'namenode_host' : ['1'],
'slave_hosts' : ['0', '1'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']}
}
decompress_cluster_host_info_mock.return_value = {'namenode_host' : ['h2.hortonworks.com'],
'slave_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_hosts' : ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670', '8670']}
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
self.assertTrue(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType']='STATUS_COMMAND'
decompress_cluster_host_info_mock.reset_mock()
json_file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
self.assertFalse(decompress_cluster_host_info_mock.called)
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
def test_dump_command_to_json(self, unlink_mock,
isfile_mock, hostname_mock):
hostname_mock.return_value = "test.hst"
command = {
'commandType': 'EXECUTION_COMMAND',
'componentName': 'NAMENODE',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {'global': {}},
'configurationTags': {'global': {'tag': 'v1'}},
'clusterHostInfo': {'namenode_host': ['1'],
'slave_hosts': ['0', '1'],
'all_hosts': ['h1.hortonworks.com', 'h2.hortonworks.com'],
'all_ping_ports': ['8670:0,1']}
}
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
isfile_mock.return_value = True
# Test dumping EXECUTION_COMMAND
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("command-3.json"))
os.unlink(json_file)
# Test dumping STATUS_COMMAND
command['commandType'] = 'STATUS_COMMAND'
json_file = orchestrator.dump_command_to_json(command, {})
self.assertTrue(os.path.exists(json_file))
self.assertTrue(os.path.getsize(json_file) > 0)
self.assertEqual(oct(os.stat(json_file).st_mode & 0777), '0600')
self.assertTrue(json_file.endswith("status_command.json"))
os.unlink(json_file)
# Testing side effect of dump_command_to_json
self.assertEquals(command['public_hostname'], "test.hst")
self.assertTrue(unlink_mock.called)
def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
FileCache_mock,
dump_command_to_json_mock):
FileCache_mock.return_value = None
get_custom_actions_base_dir_mock.return_value = "some path"
_, script = tempfile.mkstemp()
command = {
'role' : 'any',
'commandParams': {
'script_type': 'PYTHON',
'script': 'some_custom_action.py',
'command_timeout': '600',
'jdk_location' : 'some_location'
},
'taskId' : '13',
'roleCommand': 'ACTIONEXECUTE',
'commandType': 'BACKGROUND_EXECUTION_COMMAND',
'__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
MagicMock(), MagicMock())
}
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
import TestActionQueue
TestActionQueue.patch_output_file(orchestrator.python_executor)
orchestrator.python_executor.condenseOutput = MagicMock()
orchestrator.dump_command_to_json = MagicMock()
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 777)
def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,
kill_process_with_children_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
# shell.kill_process_with_children = MagicMock()
dummy_controller = MagicMock()
cfg = AmbariConfig().getConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f (a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
TestActionQueue.patch_output_file(orchestrator.python_executor)
orchestrator.python_executor.prepare_process_result = MagicMock()
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited']= ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None)
execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19,'')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
FileCache_mock,
dump_command_to_json_mock,
get_py_executor_mock):
FileCache_mock.return_value = None
get_custom_actions_base_dir_mock.return_value = "some path"
_, script = tempfile.mkstemp()
command = {
'role' : 'any',
'commandParams': {
'script_type': 'PYTHON',
'script': 'some_custom_action.py',
'command_timeout': '600',
'jdk_location' : 'some_location'
},
'taskId' : '13',
'roleCommand': 'ACTIONEXECUTE',
'commandType': 'BACKGROUND_EXECUTION_COMMAND',
'__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
MagicMock(), MagicMock())
}
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
import TestActionQueue
pyex = PythonExecutor(orchestrator.tmp_dir, orchestrator.config)
TestActionQueue.patch_output_file(pyex)
pyex.condenseOutput = MagicMock()
get_py_executor_mock.return_value = pyex
orchestrator.dump_command_to_json = MagicMock()
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 777)
def test_dump_command_to_json(self):
command = {
'commandType': 'EXECUTION_COMMAND',
'role': u'DATANODE',
'roleCommand': u'INSTALL',
'commandId': '1-1',
'taskId': 3,
'clusterName': u'cc',
'serviceName': u'HDFS',
'configurations': {
'global': {}
},
'configurationTags': {
'global': {
'tag': 'v1'
}
}
}
config = AmbariConfig().getConfig()
tempdir = tempfile.gettempdir()
config.set('agent', 'prefix', tempdir)
orchestrator = CustomServiceOrchestrator(config)
file = orchestrator.dump_command_to_json(command)
self.assertTrue(os.path.exists(file))
self.assertTrue(os.path.getsize(file) > 0)
self.assertEqual(oct(os.stat(file).st_mode & 0777), '0600')
os.unlink(file)
def test_cancel_backgound_command(self, read_stack_version_mock,
resolve_hook_script_path_mock,
resolve_script_path_mock, FileCache_mock,
kill_process_with_children_mock,
get_py_executor_mock):
FileCache_mock.return_value = None
FileCache_mock.cache_dir = MagicMock()
resolve_hook_script_path_mock.return_value = None
# shell.kill_process_with_children = MagicMock()
dummy_controller = MagicMock()
cfg = AmbariConfig()
cfg.set('agent', 'tolerate_download_failures', 'true')
cfg.set('agent', 'prefix', '.')
cfg.set('agent', 'cache_dir', 'background_tasks')
actionQueue = ActionQueue(cfg, dummy_controller)
dummy_controller.actionQueue = actionQueue
orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
orchestrator.file_cache = MagicMock()
def f(a, b):
return ""
orchestrator.file_cache.get_service_base_dir = f
actionQueue.customServiceOrchestrator = orchestrator
import TestActionQueue
import copy
pyex = PythonExecutor(actionQueue.customServiceOrchestrator.tmp_dir,
actionQueue.customServiceOrchestrator.config)
TestActionQueue.patch_output_file(pyex)
pyex.prepare_process_result = MagicMock()
get_py_executor_mock.return_value = pyex
orchestrator.dump_command_to_json = MagicMock()
lock = threading.RLock()
complete_done = threading.Condition(lock)
complete_was_called = {}
def command_complete_w(process_condenced_result, handle):
with lock:
complete_was_called['visited'] = ''
complete_done.wait(3)
actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(
actionQueue.on_background_command_complete_callback,
command_complete_w, None)
execute_command = copy.deepcopy(
TestActionQueue.TestActionQueue.background_command)
actionQueue.put([execute_command])
actionQueue.processBackgroundQueueSafeEmpty()
time.sleep(.1)
orchestrator.cancel_command(19, '')
self.assertTrue(kill_process_with_children_mock.called)
kill_process_with_children_mock.assert_called_with(33)
with lock:
complete_done.notifyAll()
with lock:
self.assertTrue(complete_was_called.has_key('visited'))
time.sleep(.1)
runningCommand = actionQueue.commandStatuses.get_command_status(19)
self.assertTrue(runningCommand is not None)
self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
