def testLastAgentEnv(self, time_mock, checkReverseLookup_mock,
checkFirewall_mock, getTransparentHugePage_mock,
getUMask_mock, checkLiveServices_mock, javaProcs_mock,
put_structured_out_mock, get_tmp_dir_mock,
get_config_mock, systemmock):
jsonFilePath = os.path.join(
TestCheckHost.current_dir + "/../../resources/custom_actions",
"check_last_agent_env.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
get_config_mock.return_value = ConfigDictionary(jsonPayload)
get_tmp_dir_mock.return_value = "/tmp"
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(time_mock.called)
self.assertTrue(checkReverseLookup_mock.called)
self.assertTrue(checkFirewall_mock.called)
self.assertTrue(getTransparentHugePage_mock.called)
self.assertTrue(getUMask_mock.called)
self.assertTrue(checkLiveServices_mock.called)
self.assertTrue(javaProcs_mock.called)
self.assertTrue(put_structured_out_mock.called)
# ensure the correct keys are in the result map
last_agent_env_check_result = put_structured_out_mock.call_args[0][0]
self.assertTrue('last_agent_env_check' in last_agent_env_check_result)
self.assertTrue('hostHealth' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('firewallRunning' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('firewallName' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('reverseLookup' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('alternatives' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue(
'umask' in last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('stackFoldersAndFiles' in
last_agent_env_check_result['last_agent_env_check'])
self.assertTrue('existingUsers' in
last_agent_env_check_result['last_agent_env_check'])
# try it now with errors
javaProcs_mock.side_effect = Exception("test exception")
checkHost.actionexecute(None)
#ensure the correct response is returned
put_structured_out_mock.assert_called_with({
'last_agent_env_check': {
'message': 'test exception',
'exit_code': 1
}
})
pass
def test_skippable_hosts(self, family_mock, get_config_mock, call_mock,
exists_mock):
"""
Tests that hosts are skippable if they don't have stack components installed
:return:
"""
# Mock the config objects
json_file_path = os.path.join(
self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
json_payload["configurations"]["cluster-env"][
"stack_tools"] = self.get_stack_tools()
json_payload["configurations"]["cluster-env"][
"stack_features"] = self.get_stack_features()
json_payload[
"upgradeSummary"] = TestStackSelectSetAll._get_upgrade_summary_no_downgrade(
)["upgradeSummary"]
config_dict = ConfigDictionary(json_payload)
family_mock.return_value = False
get_config_mock.return_value = config_dict
exists_mock.return_value = True
def hdp_select_call(command, **kwargs):
# return no versions
if "versions" in command:
return (0, "")
return (0, command)
call_mock.side_effect = hdp_select_call
# Ensure that the json file was actually read.
stack_name = default("/clusterLevelParams/stack_name", None)
stack_version = default("/clusterLevelParams/stack_version", None)
service_package_folder = default('/roleParams/service_package_folder',
None)
self.assertEqual(stack_name, "HDP")
self.assertEqual(stack_version, '2.2')
self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
# Begin the test
ru_execute = UpgradeSetAll()
ru_execute.actionexecute(None)
call_mock.assert_called_with(
('ambari-python-wrap', u'/usr/bin/hdp-select', 'versions'),
sudo=True)
self.assertEqual(call_mock.call_count, 1)
def test_execution_custom_action(self, cache_mock, get_config_mock,
get_tmp_dir_mock, get_config_file_mock,
os_path_exists_mock, call_mock):
# Mock the config objects
json_file_path = os.path.join(
self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
del json_payload['roleParams']['service_package_folder']
del json_payload['roleParams']['hooks_folder']
config_dict = ConfigDictionary(json_payload)
cache_mock.return_value = "/var/lib/ambari-agent/cache/custom_actions"
get_config_mock.return_value = config_dict
get_tmp_dir_mock.return_value = "/tmp"
ambari_agent_ini_file_path = os.path.join(
self.get_src_folder(),
"../../ambari-agent/conf/unix/ambari-agent.ini")
self.assertTrue(os.path.isfile(ambari_agent_ini_file_path))
get_config_file_mock.return_value = ambari_agent_ini_file_path
# Mock os calls
os_path_exists_mock.return_value = True
call_mock.side_effect = fake_call # echo the command
# Ensure that the json file was actually read.
stack_name = default("/hostLevelParams/stack_name", None)
stack_version = default("/hostLevelParams/stack_version", None)
service_package_folder = default('/roleParams/service_package_folder',
None)
self.assertEqual(stack_name, "HDP")
self.assertEqual(stack_version, 2.2)
self.assertEqual(service_package_folder, None)
# Begin the test
ru_execute = ExecuteUpgradeTasks()
ru_execute.actionexecute(None)
call_mock.assert_called_with(
"/usr/bin/ambari-python-wrap /var/lib/ambari-agent/cache/custom_actions"
+ os.sep + "scripts/namenode.py prepare_rolling_upgrade /tmp",
logoutput=True,
quiet=True)
pass
def testInvalidCheck(self, structured_out_mock, mock_config):
jsonFilePath = os.path.join("../resources/custom_actions", "invalid_check.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
mock_config.return_value = ConfigDictionary(jsonPayload)
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(structured_out_mock.called)
structured_out_mock.assert_called_with({})
def test_execution_23(self, link_mock, family_mock, get_config_mock,
call_mock, exists_mock):
# Mock the config objects
json_file_path = os.path.join(
self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
json_payload['hostLevelParams']['stack_name'] = "HDP"
json_payload['hostLevelParams']['stack_version'] = "2.3"
json_payload['commandParams']['version'] = "2.3.0.0-1234"
json_payload["configurations"]["cluster-env"][
"stack_tools"] = self.get_stack_tools()
json_payload["configurations"]["cluster-env"][
"stack_features"] = self.get_stack_features()
json_payload["configurations"]["cluster-env"][
"stack_packages"] = self.get_stack_packages()
config_dict = ConfigDictionary(json_payload)
family_mock.return_value = True
get_config_mock.return_value = config_dict
call_mock.side_effect = fake_call # echo the command
exists_mock.return_value = True
# Ensure that the json file was actually read.
stack_name = default("/hostLevelParams/stack_name", None)
stack_version = default("/hostLevelParams/stack_version", None)
service_package_folder = default('/roleParams/service_package_folder',
None)
self.assertEqual(stack_name, "HDP")
self.assertEqual(stack_version, '2.3')
self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
# Begin the test
ru_execute = UpgradeSetAll()
ru_execute.actionexecute(None)
self.assertTrue(link_mock.called)
call_mock.assert_called_with(
('ambari-python-wrap', '/usr/bin/hdp-select', 'set', 'all',
'2.3.0.0-1234'),
sudo=True)
def test_execution(self, cache_mock, get_config_mock, get_tmp_dir_mock,
get_config_file_mock, os_path_exists_mock, call_mock):
# Mock the config objects
json_file_path = os.path.join(
self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
config_dict = ConfigDictionary(json_payload)
cache_mock.return_value = self.CACHE_MOCK_DIR
get_config_mock.return_value = config_dict
get_tmp_dir_mock.return_value = "/tmp"
ambari_agent_ini_file_path = self.AGENT_INI_FILE_PATH
self.assertTrue(os.path.isfile(ambari_agent_ini_file_path))
get_config_file_mock.return_value = ambari_agent_ini_file_path
# Mock os calls
os_path_exists_mock.return_value = True
call_mock.side_effect = fake_call # echo the command
# Ensure that the json file was actually read.
stack_name = default("/clusterLevelParams/stack_name", None)
stack_version = default("/clusterLevelParams/stack_version", None)
service_package_folder = default('/roleParams/service_package_folder',
None)
self.assertEqual(stack_name, "HDP")
self.assertEqual(stack_version, '2.2')
self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
# Begin the test
ru_execute = ExecuteUpgradeTasks()
ru_execute.actionexecute(None)
call_mock.assert_called_with(
"source /var/lib/ambari-agent/ambari-env.sh ; /usr/bin/ambari-python-wrap /var/lib/ambari-agent/cache/common-services/HDFS/2.1.0.2.0/package"
+ os.sep + "scripts/namenode.py prepare_rolling_upgrade /tmp",
logoutput=True,
quiet=True)
pass
def test_execution_with_downgrade_allowed(self, family_mock,
get_config_mock, call_mock,
exists_mock):
# Mock the config objects
json_file_path = os.path.join(
self.get_custom_actions_dir(),
"ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
json_payload["configurations"]["cluster-env"][
"stack_tools"] = self.get_stack_tools()
json_payload["configurations"]["cluster-env"][
"stack_features"] = self.get_stack_features()
json_payload[
"upgradeSummary"] = TestStackSelectSetAll._get_upgrade_summary_downgrade_allowed(
)["upgradeSummary"]
config_dict = ConfigDictionary(json_payload)
family_mock.return_value = True
get_config_mock.return_value = config_dict
call_mock.side_effect = fake_call # echo the command
exists_mock.return_value = True
# Ensure that the json file was actually read.
stack_name = default("/clusterLevelParams/stack_name", None)
stack_version = default("/clusterLevelParams/stack_version", None)
service_package_folder = default('/roleParams/service_package_folder',
None)
self.assertEqual(stack_name, "HDP")
self.assertEqual(stack_version, '2.2')
self.assertEqual(service_package_folder,
"common-services/HDFS/2.1.0.2.0/package")
# Begin the test
ru_execute = UpgradeSetAll()
ru_execute.actionexecute(None)
call_mock.assert_not_called()
def testHostResolution(self, structured_out_mock, get_tmp_dir_mock, mock_config, mock_socket):
mock_socket.return_value = "192.168.1.1"
jsonFilePath = os.path.join(TestCheckHost.current_dir+"/../../resources/custom_actions", "check_host_ip_addresses.json")
with open(jsonFilePath, "r") as jsonFile:
jsonPayload = json.load(jsonFile)
mock_config.return_value = ConfigDictionary(jsonPayload)
get_tmp_dir_mock.return_value = "/tmp"
checkHost = CheckHost()
checkHost.actionexecute(None)
# ensure the correct function was called
self.assertTrue(structured_out_mock.called)
structured_out_mock.assert_called_with({'host_resolution_check':
{'failures': [],
'message': 'All hosts resolved to an IP address.',
'failed_count': 0,
'success_count': 5,
'exit_code': 0,
'hosts_with_failures': []}})
# try it now with errors
mock_socket.side_effect = socket.error
checkHost.actionexecute(None)
structured_out_mock.assert_called_with({'host_resolution_check':
{'failures': [
{'cause': (), 'host': u'c6401.ambari.apache.org', 'type': 'FORWARD_LOOKUP'},
{'cause': (), 'host': u'c6402.ambari.apache.org', 'type': 'FORWARD_LOOKUP'},
{'cause': (), 'host': u'c6403.ambari.apache.org', 'type': 'FORWARD_LOOKUP'},
{'cause': (), 'host': u'foobar', 'type': 'FORWARD_LOOKUP'},
{'cause': (), 'host': u'!!!', 'type': 'FORWARD_LOOKUP'}],
'message': 'There were 5 host(s) that could not resolve to an IP address.',
'failed_count': 5, 'success_count': 0, 'exit_code': 0, 'hosts_with_failures': [u'c6401.ambari.apache.org',
u'c6402.ambari.apache.org',
u'c6403.ambari.apache.org',
u'foobar', u'!!!']}})
pass
def test_downgrade_unlink_configs(self, family_mock, get_config_mock, call_mock,
isdir_mock, islink_mock):
"""
Tests downgrading from 2.3 to 2.2 to ensure that conf symlinks are removed and the backup
directories restored.
"""
isdir_mock.return_value = True
# required for the test to run since the Execute calls need this
from resource_management.core.environment import Environment
env = Environment(test_mode=True)
with env:
# Mock the config objects
json_file_path = os.path.join(self.get_custom_actions_dir(), "ru_execute_tasks_namenode_prepare.json")
self.assertTrue(os.path.isfile(json_file_path))
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
# alter JSON for a downgrade from 2.3 to 2.2
json_payload['commandParams']['version'] = "2.2.0.0-1234"
json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-1234"
json_payload['commandParams']['original_stack'] = "HDP-2.2"
json_payload['commandParams']['target_stack'] = "HDP-2.3"
json_payload['commandParams']['upgrade_direction'] = "downgrade"
json_payload['hostLevelParams']['stack_version'] = "2.2"
config_dict = ConfigDictionary(json_payload)
family_mock.return_value = True
get_config_mock.return_value = config_dict
call_mock.side_effect = fake_call # echo the command
# test the function
ru_execute = UpgradeSetAll()
ru_execute.unlink_all_configs(None)
# verify that os.path.islink was called for each conf
self.assertTrue(islink_mock.called)
for key, value in conf_select.get_package_dirs().iteritems():
for directory_mapping in value:
original_config_directory = directory_mapping['conf_dir']
is_link_called = False
for call in islink_mock.call_args_list:
call_tuple = call[0]
if original_config_directory in call_tuple:
is_link_called = True
if not is_link_called:
self.fail("os.path.islink({0}) was never called".format(original_config_directory))
# alter JSON for a downgrade from 2.3 to 2.3
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
json_payload['commandParams']['version'] = "2.3.0.0-1234"
json_payload['commandParams']['downgrade_from_version'] = "2.3.0.0-5678"
json_payload['commandParams']['original_stack'] = "HDP-2.3"
json_payload['commandParams']['target_stack'] = "HDP-2.3"
json_payload['commandParams']['upgrade_direction'] = "downgrade"
json_payload['hostLevelParams']['stack_version'] = "2.3"
# reset config
config_dict = ConfigDictionary(json_payload)
family_mock.return_value = True
get_config_mock.return_value = config_dict
# reset mock
islink_mock.reset_mock()
# test the function
ru_execute = UpgradeSetAll()
ru_execute.unlink_all_configs(None)
# ensure it wasn't called this time
self.assertFalse(islink_mock.called)
with open(json_file_path, "r") as json_file:
json_payload = json.load(json_file)
# alter JSON for a downgrade from 2.2 to 2.2
json_payload['commandParams']['version'] = "2.2.0.0-1234"
json_payload['commandParams']['downgrade_from_version'] = "2.2.0.0-5678"
json_payload['commandParams']['original_stack'] = "HDP-2.2"
json_payload['commandParams']['target_stack'] = "HDP-2.2"
json_payload['commandParams']['upgrade_direction'] = "downgrade"
json_payload['hostLevelParams']['stack_version'] = "2.2"
# reset config
config_dict = ConfigDictionary(json_payload)
family_mock.return_value = True
get_config_mock.return_value = config_dict
# reset mock
islink_mock.reset_mock()
# test the function
ru_execute = UpgradeSetAll()
ru_execute.unlink_all_configs(None)
# ensure it wasn't called this time
self.assertFalse(islink_mock.called)
