def test_update_single_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock,
update_config_mock, rename_all_prop_mock, get_config_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs", "hdfs-site"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = ['', '', '', '', '', '', '']
read_mapping_mock.return_value = {"JOBTRACKER": ["c6401"]}
update_config_mock.side_effect = [None]
get_config_mock.return_value = {}
prop_to_move = {"dfs.namenode.checkpoint.edits.dir": "a1",
"dfs.namenode.checkpoint.dir": "a2",
"dfs.namenode.checkpoint.period": "a3"}
rename_all_prop_mock.side_effect = [
prop_to_move,
{}, {}]
UpgradeHelper_HDP2.main()
self.assertTrue(update_config_mock.call_count == 1)
args, kargs = update_config_mock.call_args_list[0]
self.assertEqual("hdfs-site", args[1])
for key in prop_to_move.keys():
self.assertEqual(prop_to_move[key], args[3][key])
pass
def test_delete_mr(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["delete-mr"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
get_yn_mock.return_value = True
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6401"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
call(False, "-u", "admin:admin", "-X", "DELETE",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_update_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock,
get_config_mock, get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
get_config_resp_mock.return_value = "hbase-site", {}
site_properties = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"hbase.rpc.engine": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
get_config_mock.side_effect = [
site_properties.copy(), site_properties.copy(), site_properties.copy(),
site_properties.copy(), site_properties.copy(), site_properties.copy()]
saved_global = UpgradeHelper_HDP2.GLOBAL
saved_hdfs = UpgradeHelper_HDP2.HDFS_SITE
saved_core = UpgradeHelper_HDP2.CORE_SITE
saved_habse = UpgradeHelper_HDP2.HBASE_SITE
saved_mapred = UpgradeHelper_HDP2.MAPRED_SITE
try:
UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.main()
finally:
UpgradeHelper_HDP2.GLOBAL = saved_global
UpgradeHelper_HDP2.HDFS_SITE = saved_hdfs
UpgradeHelper_HDP2.CORE_SITE = saved_core
UpgradeHelper_HDP2.MAPRED_SITE = saved_mapred
self.assertEqual(7, len(curl_mock.call_args_list))
self.validate_update_config_call(curl_mock.call_args_list[0], "capacity-scheduler")
self.validate_update_config_call(curl_mock.call_args_list[1], "yarn-site")
self.validate_update_config_call(curl_mock.call_args_list[3], "mapred-site")
self.validate_update_config_call(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[1], "yarn-site")
self.validate_config_replacememt(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[3], "mapred-site")
self.validate_config_replacememt(curl_mock.call_args_list[4], "hdfs-site")
self.validate_config_replacememt(curl_mock.call_args_list[5], "core-site")
self.validate_config_replacememt(curl_mock.call_args_list[6], "hbase-site")
pass
def test_no_hbase(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock,
get_config_mock, get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
get_config_resp_mock.return_value = "hbase-site", None
get_config_mock.return_value = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"mapred.jobtracker.taskScheduler": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
UpgradeHelper_HDP2.main()
self.assertEqual(6, len(curl_mock.call_args_list))
def test_save_mr_mapping(self, option_parser_mock, curl_mock, json_loads_mock, write_mapping_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["save-mr-mapping"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "']
json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
self.assertTrue(write_mapping_mock.called)
write_call = write_mapping_mock.call_args
args, kargs = write_call
self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
self.assertTrue('TASKTRACKER' in args[0].keys())
self.assertTrue('TASKTRACKER' in args[0].keys())
pass
def test_print_only(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, get_yn_mock, popen_mock, read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
options = self.get_mock_options(True)
get_yn_mock.return_value = True
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"]}
UpgradeHelper_HDP2.delete_mr(options)
self.assertFalse(popen_mock.called)
pass
def test_print_only(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, get_yn_mock, popen_mock,
read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
options = self.get_mock_options(True)
get_yn_mock.return_value = True
read_mapping_mock.return_value = {"TASKTRACKER": ["c6401", "c6402"]}
UpgradeHelper_HDP2.delete_mr(options)
self.assertFalse(popen_mock.called)
pass
def test_get_and_parse_properties(self, curl_mock, backup_file_mock,
file_handler_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
options = self.get_mock_options()
curl_mock.side_effect = [
"""{ "href" : "http://localhost:8080/api/v1/clusters/c1",
"Clusters" : {
"cluster_id" : 2,
"cluster_name" : "c1",
"version" : "HDP-2.0.5",
"desired_configs" : {
"capacity-scheduler" : {
"user" : "admin",
"tag" : "version137"
},
"core-site" : {
"user" : "admin",
"tag" : "version138"
}}}}""", """{
"href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
"items" : [
{
"href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
"tag" : "version1378850572477",
"type" : "core-site",
"Config" : {
"cluster_name" : "c1"
},
"properties" : {
"name1" : "value1",
"name2" : "value2",
"name3" : "value3"
}}]}""",
"""{ "href" : "http://localhost:8080/api/v1/clusters/c1",
"Clusters" : {
"desired_configs" : {
"capacity-scheduler" : {
"user" : "admin",
"tag" : "version137"
}}}}"""
]
properties = UpgradeHelper_HDP2.get_config(options, "core-site")
self.assertTrue(len(properties.keys()) == 3)
self.assertTrue(properties["name1"] == "value1")
self.assertTrue(properties["name2"] == "value2")
self.assertTrue(properties["name3"] == "value3")
try:
UpgradeHelper_HDP2.get_config(options, "hdfs-site")
except Exception, e:
self.assertTrue(
'Unable to get the current version for config type hdfs-site'
in e.reason)
pass
def test_save_configs(self, option_parser_mock, path_exists_mock,
shutil_copy_mock, os_remove_mock,
get_config_resp_mock, write_config_mock,
file_handler_mock, logging_mock, set_formatter_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
path_exists_mock.return_value = True
shutil_copy_mock = MagicMock()
options = self.get_mock_options()
args = ["backup-configs"]
opm.parse_args.return_value = (options, args)
def get_config_resp_side_effect(ops, type, error_if_na):
if type == "global":
return ("version1", "")
elif type == "core-site":
return ("version1", "")
elif type == "hdfs-site":
return ("version1", "")
elif type == "mapred-site":
return ("version2", "")
elif type == "hbase-site":
return ("version2", "")
else:
return (None, None)
get_config_resp_mock.side_effect = get_config_resp_side_effect
UpgradeHelper_HDP2.main()
shutil_copy_mock.assert_called_once()
os_remove_mock.assert_called_once_with("logfile")
write_expected = [
call("", "global", "version1"),
call("", "core-site", "version1"),
call("", "hdfs-site", "version1"),
call("", "mapred-site", "version2"),
call("", "hbase-site", "version2")
]
write_config_mock.assert_has_calls(write_expected, any_order=True)
get_config_expected = [
call(options, "global", True),
call(options, "core-site", True),
call(options, "hdfs-site", True),
call(options, "mapred-site", True),
call(options, "hbase-site", False),
call(options, "oozie-site", False),
call(options, "webhcat-site", False),
call(options, "hive-site", False)
]
get_config_resp_mock.assert_has_calls(get_config_expected,
any_order=True)
pass
def test_add_yarn_mr_without_ATS(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock, get_stack_mock, has_comp_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
has_comp_mock.return_value = False
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"HISTORYSERVER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
call(False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari', '-X', 'POST',
'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"),
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
# assert no ATS was added
self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")
self.assert_(not call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari', "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER") in curl_mock.call_args_list, "ATS should not be added if it's not present!")
pass
def test_get_and_parse_properties(self, curl_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
options = self.get_mock_options()
curl_mock.side_effect = [
"""{ "href" : "http://localhost:8080/api/v1/clusters/c1",
"Clusters" : {
"cluster_id" : 2,
"cluster_name" : "c1",
"version" : "HDP-2.0.5",
"desired_configs" : {
"capacity-scheduler" : {
"user" : "admin",
"tag" : "version137"
},
"core-site" : {
"user" : "admin",
"tag" : "version138"
}}}}""",
"""{
"href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
"items" : [
{
"href" : "http://localhost:8080/api/v1/clusters/c1/configurations?type=core-site&tag=version138",
"tag" : "version1378850572477",
"type" : "core-site",
"Config" : {
"cluster_name" : "c1"
},
"properties" : {
"name1" : "value1",
"name2" : "value2",
"name3" : "value3"
}}]}""",
"""{ "href" : "http://localhost:8080/api/v1/clusters/c1",
"Clusters" : {
"desired_configs" : {
"capacity-scheduler" : {
"user" : "admin",
"tag" : "version137"
}}}}"""]
properties = UpgradeHelper_HDP2.get_config(options, "core-site")
self.assertTrue(len(properties.keys()) == 3)
self.assertTrue(properties["name1"] == "value1")
self.assertTrue(properties["name2"] == "value2")
self.assertTrue(properties["name3"] == "value3")
try:
UpgradeHelper_HDP2.get_config(options, "hdfs-site")
except Exception, e:
self.assertTrue('Unable to get the current version for config type hdfs-site' in e.reason)
pass
def test_update_with_append(self, optparse_mock, get_config_mock, update_config_mock):
opm = optparse_mock.return_value
update_config_mock.return_value = None
options = MagicMock()
args = ["save-configs"]
opm.parse_args.return_value = (options, args)
get_config_mock.return_value = {"a1": "va1", "a2": "va2", "b1": "vb1", "b2": "vb2", "c1": "vc1", "d1": "d1"}
site_template = {"y1": "vy1", "a1": "REPLACE_WITH_", "a2": "REPLACE_WITH_", "nb1": "REPLACE_WITH_b1",
"nb2": "REPLACE_WITH_b2", "d1": "DELETE_OLD", "b1" : "DELETE_OLD"}
expected_site = {"y1": "vy1", "a1": "va1", "a2": "va2", "nb1": "vb1", "nb2": "vb2", "c1": "vc1"}
UpgradeHelper_HDP2.update_config_using_existing(opm, "global", site_template, True)
get_config_mock.assert_called_once_with(opm, "global")
update_config_mock.assert_called_once_with(opm, expected_site, "global")
pass
def test_install_yarn_mr2(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["install-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = '"href" : "'
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install YARN"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install MapReduce2"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=False)
pass
def test_save_mr_mapping(self, option_parser_mock, curl_mock,
json_loads_mock, write_mapping_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["save-mr-mapping"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = [
'"href" : "', '"href" : "', '"href" : "', '"href" : "'
]
json_loads_mock.return_value = {
"host_components": [{
"HostRoles": {
"host_name": "host1"
}
}]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(
False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"
),
call(
False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"
),
call(
False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER"
)
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
self.assertTrue(write_mapping_mock.called)
write_call = write_mapping_mock.call_args
args, kargs = write_call
self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
self.assertTrue('TASKTRACKER' in args[0].keys())
self.assertTrue('TASKTRACKER' in args[0].keys())
pass
def test_add_yarn_mr(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_update_with_appen_II(self, optparse_mock, get_config_mock,
update_config_mock):
opm = optparse_mock.return_value
update_config_mock.return_value = None
options = MagicMock()
args = ["save-configs"]
opm.parse_args.return_value = (options, args)
get_config_mock.return_value = {
"a1": "va1",
"a2": "va2",
"b1": "vb1",
"b2": "vb2",
"c1": "vc1",
"x1": "x1",
"X1": "X1"
}
site_template = {
"y1": "vy1",
"a1": "REPLACE_WITH_",
"a2": "REPLACE_WITH_",
"nb1": "REPLACE_WITH_b1",
"nb2": "REPLACE_WITH_b2",
"x1": "DELETE_OLD",
"X1": "DELETE"
}
expected_site = {
"y1": "vy1",
"a1": "va1",
"a2": "va2",
"nb1": "vb1",
"nb2": "vb2",
"c1": "vc1",
"X1": "DELETE"
}
UpgradeHelper_HDP2.update_config_using_existing(
opm, "global", site_template)
get_config_mock.assert_called_once_with(opm, "global")
update_config_mock.assert_called_once_with(opm, expected_site,
"global")
pass
def test_rename_all_properties(self):
site_properties = {
"mapred.task.is.map": "mapreduce.task.ismap",
"mapred.task.partition": "mapreduce.task.partition",
"mapred.task.profile": "mapreduce.task.profile",
"abc": "abc"
}
site_properties = \
UpgradeHelper_HDP2.rename_all_properties(site_properties, UpgradeHelper_HDP2.PROPERTY_MAPPING)
for key in site_properties.keys():
self.assertEqual(key, site_properties[key])
self.assertEqual(4, len(site_properties))
pass
def test_install_yarn_mr2(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["install-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = '"href" : "'
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(
False, "-u", "admin:admin", "-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install YARN"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(
False, "-u", "admin:admin", "-X", "PUT", "-d",
"""{"RequestInfo":{"context":"Install MapReduce2"}, "Body":{"ServiceInfo": {"state":"INSTALLED"}}}""",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2")
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=False)
pass
def test_delete_mr(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, read_mapping_mock, get_yn_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["delete-mr"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
get_yn_mock.return_value = True
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6401"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "PUT", "-d", """{"HostRoles": {"state": "DISABLED"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "DELETE",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_rename_all_properties(self):
site_properties = {
"mapred.task.is.map": "mapreduce.task.ismap",
"mapred.task.partition": "mapreduce.task.partition",
"mapred.task.profile": "mapreduce.task.profile",
"abc": "abc"
}
site_properties = \
UpgradeHelper_HDP2.rename_all_properties(site_properties, UpgradeHelper_HDP2.PROPERTY_MAPPING)
for key in site_properties.keys():
self.assertEqual(key, site_properties[key])
self.assertEqual(4, len(site_properties))
pass
def test_save_configs(self, option_parser_mock, path_exists_mock, shutil_copy_mock, os_remove_mock,
get_config_resp_mock, write_config_mock, file_handler_mock, logging_mock, set_formatter_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
path_exists_mock.return_value = True
shutil_copy_mock = MagicMock()
options = self.get_mock_options()
args = ["backup-configs"]
opm.parse_args.return_value = (options, args)
def get_config_resp_side_effect(ops, type, error_if_na):
if type == "global":
return ("version1", "")
elif type == "core-site":
return ("version1", "")
elif type == "hdfs-site":
return ("version1", "")
elif type == "mapred-site":
return ("version2", "")
elif type == "hbase-site":
return ("version2", "")
else:
return (None, None)
get_config_resp_mock.side_effect = get_config_resp_side_effect
UpgradeHelper_HDP2.main()
shutil_copy_mock.assert_called_once()
os_remove_mock.assert_called_once_with("logfile")
write_expected = [call("", "global", "version1"), call("", "core-site", "version1"),
call("", "hdfs-site", "version1"), call("", "mapred-site", "version2"),
call("", "hbase-site", "version2")]
write_config_mock.assert_has_calls(write_expected, any_order=True)
get_config_expected = [call(options, "global", True), call(options, "core-site", True),
call(options, "hdfs-site", True), call(options, "mapred-site", True),
call(options, "hbase-site", False), call(options, "oozie-site", False),
call(options, "webhcat-site", False), call(options, "hive-site", False)]
get_config_resp_mock.assert_has_calls(get_config_expected, any_order=True)
pass
def test_backup_configs(self, backup_config_mock): UpgradeHelper_HDP2.backup_configs(None) self.assertTrue(backup_config_mock.called)
def test_add_yarn_mr(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"
)
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_backup_configs(self, backup_config_mock):
UpgradeHelper_HDP2.backup_configs(None)
self.assertTrue(backup_config_mock.called)
def test_add_yarn_mr_without_ATS(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock,
read_mapping_mock, get_stack_mock,
has_comp_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
has_comp_mock.return_value = False
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"HISTORYSERVER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"
),
call(
False, '-u', 'admin:admin', '-H', 'X-Requested-By: ambari',
'-X', 'POST',
'http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER'
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"
),
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
# assert no ATS was added
self.assert_(
not call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/APP_TIMELINE_SERVER"
) in curl_mock.call_args_list,
"ATS should not be added if it's not present!")
self.assert_(
not call(
False, "-u", "admin:admin", '-H', 'X-Requested-By: ambari',
"-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/APP_TIMELINE_SERVER"
) in curl_mock.call_args_list,
"ATS should not be added if it's not present!")
pass
def test_update_configs(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock,
read_mapping_mock, get_config_mock,
get_config_resp_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = MagicMock()
args = ["update-configs"]
opm.parse_args.return_value = (options, args)
options.logfile = "logfile"
options.user = "******"
options.password = "******"
options.hostname = "localhost"
options.clustername = "c1"
curl_mock.side_effect = ['', '', '', '', '', '', '', '']
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
get_config_resp_mock.return_value = "hbase-site", {}
site_properties = {
"mapred.hosts": "an_old_value",
"mapred.hosts.exclude": "an_old_value",
"mapred.jobtracker.maxtasks.per.job": "an_old_value",
"hbase.rpc.engine": "an_old_value",
"dfs.df.interval": "an_old_value",
"mapred.userlog.retain.hours": "will_not_be_stored",
"global1": "global11"
}
get_config_mock.side_effect = [
site_properties.copy(),
site_properties.copy(),
site_properties.copy(),
site_properties.copy(),
site_properties.copy(),
site_properties.copy()
]
saved_global = UpgradeHelper_HDP2.GLOBAL
saved_hdfs = UpgradeHelper_HDP2.HDFS_SITE
saved_core = UpgradeHelper_HDP2.CORE_SITE
saved_habse = UpgradeHelper_HDP2.HBASE_SITE
saved_hive = UpgradeHelper_HDP2.HIVE_SITE
saved_mapred = UpgradeHelper_HDP2.MAPRED_SITE
try:
UpgradeHelper_HDP2.GLOBAL = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.HDFS_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.CORE_SITE = {"global2": "REPLACE_WITH_global1"}
UpgradeHelper_HDP2.main()
finally:
UpgradeHelper_HDP2.GLOBAL = saved_global
UpgradeHelper_HDP2.HDFS_SITE = saved_hdfs
UpgradeHelper_HDP2.CORE_SITE = saved_core
UpgradeHelper_HDP2.MAPRED_SITE = saved_mapred
self.assertEqual(8, len(curl_mock.call_args_list))
self.validate_update_config_call(curl_mock.call_args_list[0],
"capacity-scheduler")
self.validate_update_config_call(curl_mock.call_args_list[1],
"yarn-site")
self.validate_update_config_call(curl_mock.call_args_list[3],
"mapred-site")
self.validate_update_config_call(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[1],
"yarn-site")
self.validate_config_replacememt(curl_mock.call_args_list[2], "global")
self.validate_config_replacememt(curl_mock.call_args_list[3],
"mapred-site")
self.validate_config_replacememt(curl_mock.call_args_list[4],
"hdfs-site")
self.validate_config_replacememt(curl_mock.call_args_list[5],
"core-site")
self.validate_config_replacememt(curl_mock.call_args_list[6],
"hbase-site")
pass
