def test_delete_mr(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["delete-mr"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
get_yn_mock.return_value = True
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6401"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
call(False, "-u", "admin:admin", "-X", "DELETE",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_no_hbase(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock,
get_config_mock, get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
get_config_resp_mock.return_value = "hbase-site", None
get_config_mock.return_value = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"mapred.jobtracker.taskScheduler": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
UpgradeHelper_HDP2.main()
self.assertEqual(6, len(curl_mock.call_args_list))
def test_save_mr_mapping(self, option_parser_mock, curl_mock, json_loads_mock, write_mapping_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["save-mr-mapping"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "']
json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
self.assertTrue(write_mapping_mock.called)
write_call = write_mapping_mock.call_args
args, kargs = write_call
self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
self.assertTrue('TASKTRACKER' in args[0].keys())
self.assertTrue('TASKTRACKER' in args[0].keys())
pass
def test_update_single_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock,
read_mapping_mock, update_config_mock,
rename_all_prop_mock, get_config_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs", "hdfs-site"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = ['', '', '', '', '', '', '']
read_mapping_mock.return_value = {"JOBTRACKER": ["c6401"]}
update_config_mock.side_effect = [None]
get_config_mock.return_value = {}
prop_to_move = {
"dfs.namenode.checkpoint.edits.dir": "a1",
"dfs.namenode.checkpoint.dir": "a2",
"dfs.namenode.checkpoint.period": "a3"
}
rename_all_prop_mock.side_effect = [prop_to_move, {}, {}]
UpgradeHelper_HDP2.main()
self.assertTrue(update_config_mock.call_count == 1)
args, kargs = update_config_mock.call_args_list[0]
self.assertEqual("hdfs-site", args[1])
for key in prop_to_move.keys():
self.assertEqual(prop_to_move[key], args[3][key])
pass
def test_no_hbase(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, read_mapping_mock, get_config_mock,
get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
get_config_resp_mock.return_value = "hbase-site", None
get_config_mock.return_value = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"mapred.jobtracker.taskScheduler": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
UpgradeHelper_HDP2.main()
self.assertEqual(6, len(curl_mock.call_args_list))
def test_update_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock,
get_config_mock, get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
get_config_resp_mock.return_value = "hbase-site", {}
site_properties = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"hbase.rpc.engine": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
get_config_mock.side_effect = [
site_properties.copy(), site_properties.copy(), site_properties.copy(),
site_properties.copy(), site_properties.copy(), site_properties.copy()]
saved_global = UpgradeHelper_HDP2.GLOBAL
saved_hdfs = UpgradeHelper_HDP2.HDFS_SITE
saved_core = UpgradeHelper_HDP2.CORE_SITE
saved_habse = UpgradeHelper_HDP2.HBASE_SITE
saved_mapred = UpgradeHelper_HDP2.MAPRED_SITE
try:
UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.main()
finally:
UpgradeHelper_HDP2.GLOBAL = saved_global
UpgradeHelper_HDP2.HDFS_SITE = saved_hdfs
UpgradeHelper_HDP2.CORE_SITE = saved_core
UpgradeHelper_HDP2.MAPRED_SITE = saved_mapred
self.assertEqual(7, len(curl_mock.call_args_list))
self.validate_update_config_call(curl_mock.call_args_list[0], "capacity-scheduler")
self.validate_update_config_call(curl_mock.call_args_list[1], "yarn-site")
self.validate_update_config_call(curl_mock.call_args_list[3], "mapred-site")
self.validate_update_config_call(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[1], "yarn-site")
self.validate_config_replacememt(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[3], "mapred-site")
self.validate_config_replacememt(curl_mock.call_args_list[4], "hdfs-site")
self.validate_config_replacememt(curl_mock.call_args_list[5], "core-site")
self.validate_config_replacememt(curl_mock.call_args_list[6], "hbase-site")
pass
def test_install_yarn_mr2(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["install-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = '"href" : "'
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install MapReduce2"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install YARN"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/YARN")
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=False)
pass
def test_update_single_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock,
update_config_mock, rename_all_prop_mock, get_config_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs", "hdfs-site"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = ['', '', '', '', '', '', '']
read_mapping_mock.return_value = {"JOBTRACKER": ["c6401"]}
update_config_mock.side_effect = [None]
get_config_mock.return_value = {}
prop_to_move = {"dfs.namenode.checkpoint.edits.dir": "a1",
"dfs.namenode.checkpoint.dir": "a2",
"dfs.namenode.checkpoint.period": "a3"}
rename_all_prop_mock.side_effect = [
prop_to_move,
{}, {}]
UpgradeHelper_HDP2.main()
self.assertTrue(update_config_mock.call_count == 1)
args, kargs = update_config_mock.call_args_list[0]
self.assertEqual("hdfs-site", args[1])
for key in prop_to_move.keys():
self.assertEqual(prop_to_move[key], args[3][key])
pass
def test_save_configs(self, option_parser_mock, path_exists_mock,
shutil_copy_mock, os_remove_mock,
get_config_resp_mock, write_config_mock,
file_handler_mock, logging_mock, set_formatter_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
path_exists_mock.return_value = True
shutil_copy_mock = MagicMock()
options = self.get_mock_options()
args = ["backup-configs"]
opm.parse_args.return_value = (options, args)
def get_config_resp_side_effect(ops, type, error_if_na):
if type == "global":
return ("version1", "")
elif type == "core-site":
return ("version1", "")
elif type == "hdfs-site":
return ("version1", "")
elif type == "mapred-site":
return ("version2", "")
elif type == "hbase-site":
return ("version2", "")
else:
return (None, None)
get_config_resp_mock.side_effect = get_config_resp_side_effect
UpgradeHelper_HDP2.main()
shutil_copy_mock.assert_called_once()
os_remove_mock.assert_called_once_with("logfile")
write_expected = [
call("", "global", "version1"),
call("", "core-site", "version1"),
call("", "hdfs-site", "version1"),
call("", "mapred-site", "version2"),
call("", "hbase-site", "version2")
]
write_config_mock.assert_has_calls(write_expected, any_order=True)
get_config_expected = [
call(options, "global", True),
call(options, "core-site", True),
call(options, "hdfs-site", True),
call(options, "mapred-site", True),
call(options, "hbase-site", False),
call(options, "oozie-site", False),
call(options, "webhcat-site", False),
call(options, "hive-site", False)
]
get_config_resp_mock.assert_has_calls(get_config_expected,
any_order=True)
pass
def test_add_yarn_mr_without_ATS(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock, get_stack_mock, has_comp_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
has_comp_mock.return_value = False
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"HISTORYSERVER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
call(False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari', '-X', 'POST',
'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"),
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
# assert no ATS was added
self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")
self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")
pass
def test_install_yarn_mr2(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["install-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = '"href" : "'
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install YARN"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install MapReduce2"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=False)
pass
def test_save_mr_mapping(self, option_parser_mock, curl_mock,
json_loads_mock, write_mapping_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["save-mr-mapping"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = [
'"href" : "', '"href" : "', '"href" : "', '"href" : "'
]
json_loads_mock.return_value = {
"host_components": [{
"HostRoles": {
"host_name": "host1"
}
}]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(
False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"
),
call(
False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"
),
call(
False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER"
)
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
self.assertTrue(write_mapping_mock.called)
write_call = write_mapping_mock.call_args
args, kargs = write_call
self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
self.assertTrue('TASKTRACKER' in args[0].keys())
self.assertTrue('TASKTRACKER' in args[0].keys())
pass
def test_add_yarn_mr(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_delete_mr(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, read_mapping_mock, get_yn_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["delete-mr"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
get_yn_mock.return_value = True
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6401"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "DELETE",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_save_configs(self, option_parser_mock, path_exists_mock, shutil_copy_mock, os_remove_mock,
get_config_resp_mock, write_config_mock, file_handler_mock, logging_mock, set_formatter_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
path_exists_mock.return_value = True
shutil_copy_mock = MagicMock()
options = self.get_mock_options()
args = ["backup-configs"]
opm.parse_args.return_value = (options, args)
def get_config_resp_side_effect(ops, type, error_if_na):
if type == "global":
return ("version1", "")
elif type == "core-site":
return ("version1", "")
elif type == "hdfs-site":
return ("version1", "")
elif type == "mapred-site":
return ("version2", "")
elif type == "hbase-site":
return ("version2", "")
else:
return (None, None)
get_config_resp_mock.side_effect = get_config_resp_side_effect
UpgradeHelper_HDP2.main()
shutil_copy_mock.assert_called_once()
os_remove_mock.assert_called_once_with("logfile")
write_expected = [call("", "global", "version1"), call("", "core-site", "version1"),
call("", "hdfs-site", "version1"), call("", "mapred-site", "version2"),
call("", "hbase-site", "version2")]
write_config_mock.assert_has_calls(write_expected, any_order=True)
get_config_expected = [call(options, "global", True), call(options, "core-site", True),
call(options, "hdfs-site", True), call(options, "mapred-site", True),
call(options, "hbase-site", False), call(options, "oozie-site", False),
call(options, "webhcat-site", False), call(options, "hive-site", False)]
get_config_resp_mock.assert_has_calls(get_config_expected, any_order=True)
pass
def test_add_yarn_mr_without_ATS(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock,
read_mapping_mock, get_stack_mock,
has_comp_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
has_comp_mock.return_value = False
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"HISTORYSERVER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"
),
call(
False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari',
'-X', 'POST',
'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"
),
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
# assert no ATS was added
self.assert_(
not call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER"
) in curl_mock.call_args_list,
"ATS should not be added if it's not present!")
self.assert_(
not call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER"
) in curl_mock.call_args_list,
"ATS should not be added if it's not present!")
pass
def test_update_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock,
read_mapping_mock, get_config_mock,
get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
get_config_resp_mock.return_value = "hbase-site", {}
site_properties = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"hbase.rpc.engine": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
get_config_mock.side_effect = [
site_properties.copy(),
site_properties.copy(),
site_properties.copy(),
site_properties.copy(),
site_properties.copy(),
site_properties.copy()
]
saved_global = UpgradeHelper_HDP2.GLOBAL
saved_hdfs = UpgradeHelper_HDP2.HDFS_SITE
saved_core = UpgradeHelper_HDP2.CORE_SITE
saved_habse = UpgradeHelper_HDP2.HBASE_SITE
saved_hive = UpgradeHelper_HDP2.HIVE_SITE
saved_mapred = UpgradeHelper_HDP2.MAPRED_SITE
try:
UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.main()
finally:
UpgradeHelper_HDP2.GLOBAL = saved_global
UpgradeHelper_HDP2.HDFS_SITE = saved_hdfs
UpgradeHelper_HDP2.CORE_SITE = saved_core
UpgradeHelper_HDP2.MAPRED_SITE = saved_mapred
self.assertEqual(8, len(curl_mock.call_args_list))
self.validate_update_config_call(curl_mock.call_args_list[0],
"capacity-scheduler")
self.validate_update_config_call(curl_mock.call_args_list[1],
"yarn-site")
self.validate_update_config_call(curl_mock.call_args_list[3],
"mapred-site")
self.validate_update_config_call(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[1],
"yarn-site")
self.validate_config_replacememt(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[3],
"mapred-site")
self.validate_config_replacememt(curl_mock.call_args_list[4],
"hdfs-site")
self.validate_config_replacememt(curl_mock.call_args_list[5],
"core-site")
self.validate_config_replacememt(curl_mock.call_args_list[6],
"hbase-site")
pass
def test_add_yarn_mr(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"
)
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
