def test_namespace_deletion(self, execute_command, unify):
execute_command.side_effect = ['namespace', None]
namespace = Namespace('namespace')
namespace.delete()
self.assertTrue(execute_command.call_count, 2)
self.assertTrue(call('ip netns list') in execute_command.mock_calls)
self.assertTrue(call('ip netns del namespace') in execute_command.mock_calls)
def test_format_protoclaims_ref_adds_column(self):
claim_1 = pywikibot.Claim(self.repo, 'P123')
claim_1.setTarget('1')
ref_1 = Reference(claim_1)
itis_1 = Statement('foo')
itis_2 = Statement('bar').add_reference(ref_1)
self.preview_item.protoclaims = {'P123': [itis_1, itis_2]}
expected = (
"{| class='wikitable'\n"
"|-\n"
"! Property\n"
"! Value\n"
"! Qualifiers\n"
"! References\n"
'|-\n'
'| wd_template_1 \n'
'| formatted_itis_1 \n'
'| \n'
'| \n'
'|-\n'
'| wd_template_1 \n'
'| formatted_itis_2 \n'
'| \n'
'| \nformatted_reference_1 \n'
"|}"
)
self.assertEqual(self.preview_item.format_protoclaims(), expected)
self.mock_wd_template.assert_called_once_with('P123')
self.mock_format_itis.assert_has_calls([
mock.call(itis_1),
mock.call(itis_2)
])
self.mock_format_qual.assert_not_called()
self.mock_format_ref.assert_called_once_with(ref_1)
def test_two_index_pages(self, mock_cache, mock_requests_get):
index_pages = [['book link href="http://host/foo.zip"',
'next page href="local_page.html" link'],
['book link href="http://host/bar.zip"',
'last page, no link']]
mock_requests_get.return_value.iter_lines.side_effect = index_pages
expected_cache_requests = [call('http://host/foo.zip'),
call('http://host/bar.zip')]
text1_lines = ['Title',
'The text of a book.']
text2_lines = ['A short book.']
mock_cache.return_value.fetch_lines.side_effect = [text1_lines,
text2_lines]
expected_words = ['Title',
'The', 'text', 'of', 'a', 'book.',
'A', 'short', 'book.']
words = list(fetch_all_words())
self.assertEqual(expected_words, words)
self.assertEqual(expected_cache_requests,
mock_cache.return_value.fetch_lines.mock_calls)
mock_requests_get.assert_called_with(
'http://www.gutenberg.org/robot/local_page.html',
stream=True)
def test_format_protoclaims_multple_different_prop(self):
itis_1 = Statement('foo')
itis_2 = Statement('bar')
protoclaims = {'P123': itis_1, 'P321': itis_2}
self.preview_item.protoclaims = OrderedDict(
sorted(protoclaims.items(), key=lambda t: int(t[0][1:])))
expected = (
"{| class='wikitable'\n"
"|-\n"
"! Property\n"
"! Value\n"
"! Qualifiers\n"
'|-\n'
'| wd_template_1 \n'
'| formatted_itis_1 \n'
'| \n'
'|-\n'
'| wd_template_2 \n'
'| formatted_itis_2 \n'
'| \n'
"|}"
)
self.assertEqual(self.preview_item.format_protoclaims(), expected)
self.mock_wd_template.assert_has_calls([
mock.call('P123'),
mock.call('P321')],
any_order=True
)
self.mock_format_itis.assert_has_calls([
mock.call(itis_1),
mock.call(itis_2)],
any_order=True
)
self.mock_format_qual.assert_not_called()
self.mock_format_ref.assert_not_called()
def test_format_protoclaims_multple_same_prop(self):
itis_1 = Statement('foo')
itis_2 = Statement('bar')
self.preview_item.protoclaims = {'P123': [itis_1, itis_2]}
expected = (
"{| class='wikitable'\n"
"|-\n"
"! Property\n"
"! Value\n"
"! Qualifiers\n"
'|-\n'
'| wd_template_1 \n'
'| formatted_itis_1 \n'
'| \n'
'|-\n'
'| wd_template_1 \n'
'| formatted_itis_2 \n'
'| \n'
"|}"
)
self.assertEqual(self.preview_item.format_protoclaims(), expected)
self.mock_wd_template.assert_called_once_with('P123')
self.mock_format_itis.assert_has_calls([
mock.call(itis_1),
mock.call(itis_2)
])
self.mock_format_qual.assert_not_called()
self.mock_format_ref.assert_not_called()
def test_do_cleanup_all(self, do_erase_packages_method, do_delete_users_method,
do_erase_dir_silent_method,
do_erase_files_silent_method, do_kill_processes_method,
get_os_type_method, find_repo_files_for_repos_method,
do_erase_alternatives_method, get_additional_dirs_method, clear_cache_mock):
out = StringIO.StringIO()
sys.stdout = out
get_additional_dirs_method.return_value = ['/tmp/hadoop-yarn','/tmp/hsperfdata_007']
propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'],
REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'],
PROCESS_SECTION:['abcd', 'pqrst'],
ALT_SECTION:{ALT_KEYS[0]:['alt1','alt2'], ALT_KEYS[1]:[
'dir1']}, USER_HOMEDIR_SECTION:['decf']}
get_os_type_method.return_value = 'redhat'
find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst']
self.hostcleanup.do_cleanup(propertyMap)
self.assertTrue(do_delete_users_method.called)
self.assertTrue(do_erase_dir_silent_method.called)
self.assertTrue(do_erase_files_silent_method.called)
self.assertTrue(do_erase_packages_method.called)
self.assertTrue(do_kill_processes_method.called)
self.assertTrue(do_erase_alternatives_method.called)
calls = [call(['decf']), call(['abcd', 'pqrst']), call(['/tmp/hadoop-yarn','/tmp/hsperfdata_007'])]
do_erase_dir_silent_method.assert_has_calls(calls)
do_erase_packages_method.assert_called_once_with(['abcd', 'pqrst'])
do_erase_files_silent_method.assert_called_once_with(['abcd', 'pqrst'])
do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst'])
do_erase_alternatives_method.assert_called_once_with({ALT_KEYS[0]:['alt1',
'alt2'], ALT_KEYS[1]:['dir1']})
sys.stdout = sys.__stdout__
def run_test(self, mock_field_map, mock_build_sql_query,
mock_ext_output, mock_projection):
mock_field_map.return_value = [u'id_cfpb', u'name']
mock_build_sql_query.return_value = 'SELECT blah blah'
mock_cursor_attrs = {'execute.return_value': True}
mock_cursor = MagicMock(**mock_cursor_attrs)
mock_cursor_enter_return = MagicMock(
return_value=["row1", "row2", "row3"], **mock_cursor_attrs)
mock_cursor_enter_return.__iter__.return_value = [
'row1', 'row2', 'row3']
mock_cursor.__enter__ = Mock(return_value=mock_cursor_enter_return)
mock_connect_attrs = {'cursor.return_value': mock_cursor}
mock_connect = Mock(**mock_connect_attrs)
mock_extractor_attrs = {'connect.return_value': mock_connect}
mock_extractor = Mock(table="table", **mock_extractor_attrs)
mock_ext_output.return_value = mock_extractor
mock_projection.return_value = "1"
format = Format("mapping_file", "docs_file", "table",
sql_filter="WHERE id_cfpb is not NULL", marker_table=True)
format.run()
mock_field_map.assert_called_once()
assert_equal(mock_projection.call_count, 3)
mock_projection.assert_has_calls([
mock.call(mock_field_map.return_value, "row1"),
mock.call(mock_field_map.return_value, "row2"),
mock.call(mock_field_map.return_value, "row3")
])
def test_format_protoclaims_single_with_multiple_qual(self):
itis = Statement('dummy')
qual_1 = Qualifier('P321', 'qual_dummy')
qual_2 = Qualifier('P213', 'qual_dummy')
itis._quals.add(qual_1)
itis._quals.add(qual_2)
self.preview_item.protoclaims = {'P123': itis}
expected = (
"{| class='wikitable'\n"
"|-\n"
"! Property\n"
"! Value\n"
"! Qualifiers\n"
'|-\n'
'| wd_template_1 \n'
'| formatted_itis_1 \n'
'| * formatted_qual_1 \n'
'* formatted_qual_2 \n'
"|}"
)
self.assertEqual(self.preview_item.format_protoclaims(), expected)
self.mock_wd_template.assert_called_once_with('P123')
self.mock_format_itis.assert_called_once_with(itis)
self.mock_format_qual.assert_has_calls([
mock.call(qual_1),
mock.call(qual_2)],
any_order=True
)
self.mock_format_ref.assert_not_called()
def test_delete_mr(self, option_parser_mock, curl_mock,
backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["delete-mr"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
get_yn_mock.return_value = True
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6401"]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
call(False, "-u", "admin:admin", "-X", "DELETE",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_save_mr_mapping(self, option_parser_mock, curl_mock, json_loads_mock, write_mapping_mock,
backup_file_mock, file_handler_mock):
file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["save-mr-mapping"]
opm.parse_args.return_value = (options, args)
curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "']
json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
call(False, "-u", "admin:admin",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
self.assertTrue(write_mapping_mock.called)
write_call = write_mapping_mock.call_args
args, kargs = write_call
self.assertTrue('MAPREDUCE_CLIENT' in args[0].keys())
self.assertTrue(["host1"] == args[0]['MAPREDUCE_CLIENT'])
self.assertTrue('TASKTRACKER' in args[0].keys())
self.assertTrue('TASKTRACKER' in args[0].keys())
pass
def test_execute_retryable_command_with_time_lapse(self, CustomServiceOrchestrator_mock,
read_stack_version_mock, sleep_mock, time_mock
):
CustomServiceOrchestrator_mock.return_value = None
dummy_controller = MagicMock()
actionQueue = ActionQueue(AmbariConfig(), dummy_controller)
python_execution_result_dict = {
'exitcode': 1,
'stdout': 'out',
'stderr': 'stderr',
'structuredOut': '',
'status': 'FAILED'
}
time_mock.side_effect = [4, 8, 10, 14, 18, 22]
def side_effect(command, tmpoutfile, tmperrfile, override_output_files=True, retry=False):
return python_execution_result_dict
command = copy.deepcopy(self.retryable_command)
with patch.object(CustomServiceOrchestrator, "runCommand") as runCommand_mock:
runCommand_mock.side_effect = side_effect
actionQueue.execute_command(command)
#assert that python executor start
self.assertTrue(runCommand_mock.called)
self.assertEqual(2, runCommand_mock.call_count)
self.assertEqual(1, sleep_mock.call_count)
sleep_mock.assert_has_calls([call(2)], False)
runCommand_mock.assert_has_calls([
call(command, '/tmp/ambari-agent/output-19.txt', '/tmp/ambari-agent/errors-19.txt', override_output_files=True, retry=False),
call(command, '/tmp/ambari-agent/output-19.txt', '/tmp/ambari-agent/errors-19.txt', override_output_files=False, retry=True)])
def test_do_cleanup_with_skip(self, do_erase_packages_method,
do_delete_users_method,
do_erase_dir_silent_method,
do_erase_files_silent_method, do_kill_processes_method,
get_os_type_method, find_repo_files_for_repos_method, clear_cache_mock):
out = StringIO.StringIO()
sys.stdout = out
propertyMap = {PACKAGE_SECTION:['abcd', 'pqrst'], USER_SECTION:['abcd', 'pqrst'],
REPO_SECTION:['abcd', 'pqrst'], DIR_SECTION:['abcd', 'pqrst'],
PROCESS_SECTION:['abcd', 'pqrst']}
get_os_type_method.return_value = 'redhat'
find_repo_files_for_repos_method.return_value = ['abcd', 'pqrst']
HostCleanup.SKIP_LIST = [PACKAGE_SECTION, REPO_SECTION]
self.hostcleanup.do_cleanup(propertyMap)
self.assertTrue(do_delete_users_method.called)
self.assertTrue(do_erase_dir_silent_method.called)
self.assertFalse(do_erase_files_silent_method.called)
self.assertFalse(do_erase_packages_method.called)
self.assertTrue(do_kill_processes_method.called)
calls = [call(None), call(['abcd', 'pqrst'])]
do_erase_dir_silent_method.assert_has_calls(calls)
do_delete_users_method.assert_called_once_with(['abcd', 'pqrst'])
do_kill_processes_method.assert_called_once_with(['abcd', 'pqrst'])
sys.stdout = sys.__stdout__
def test_action_install_pattern_suse(self, shell_mock, call_mock):
call_mock.side_effect=[(0, None), (0, "Loading repository data...\nReading installed packages...\n\nS | Name\n--+-----\n | Pack")]
with Environment('/') as env:
Package("some_package*",
)
call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
call("zypper --non-interactive search --type package --uninstalled-only --match-exact 'some_package*'")])
shell_mock.assert_called_with(['/usr/bin/zypper', '--quiet', 'install', '--auto-agree-with-licenses', '--no-confirm', 'some_package*'], logoutput=False, sudo=True)
def test_action_install_pattern_suse(self, shell_mock, call_mock):
call_mock.side_effect=[(0, None), (0, "Loading repository data...\nReading installed packages...\nNo packages found.\n")]
with Environment('/') as env:
Package("some_package*",
)
call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
call("zypper --non-interactive search --type package --uninstalled-only --match-exact 'some_package*'")])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_action_install_pattern_rhel(self, shell_mock, call_mock):
call_mock.side_effect=[(0, None), (1, "Some text")]
with Environment('/') as env:
Package("some_package*",
)
call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
call("! yum list available 'some_package*'")])
shell_mock.assert_called_with(['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install', 'some_package*'], logoutput=False, sudo=True)
def test_action_install_pattern_installed_rhel(self, shell_mock, call_mock):
call_mock.side_effect=[(0, None), (0, "Some text")]
with Environment('/') as env:
Package("some_package*",
)
call_mock.assert_has_calls([call("installed_pkgs=`rpm -qa 'some_package*'` ; [ ! -z \"$installed_pkgs\" ]"),
call("! yum list available 'some_package*'")])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_action_install_ubuntu(self, shell_mock, call_mock):
call_mock.side_effect = [(1, None), (0, None)]
with Environment('/') as env:
Package("some_package",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package$'"),
call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_scan_dict_values_executes_value_handler_for_all_matching_prefixes(self):
dictionary = {'a': 'foo123', 'b': {'c': 'foo234'}}
handler = Mock()
handler.return_value = "foo"
result = CloudFormationTemplateTransformer.scan_dict_values(dictionary, handler)
expected_calls = [mock.call('foo123'), mock.call('foo234')]
six.assertCountEqual(self, expected_calls, handler.mock_calls)
six.assertCountEqual(self, result, {'a': 'foo', 'b': {'c': 'foo'}})
def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):
config_file = self.get_src_folder()+"/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
copy_to_hdfs_mock.return_value = True
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/historyserver.py",
classname = "HistoryServer",
command = "pre_upgrade_restart",
config_dict = json_content,
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks = [(0, None, ''), (0, None, None), (0, None, None), (0, None, None), (0, None, None)],
mocks_dict = mocks_dict)
self.assertResourceCalledIgnoreEarlier('Execute', ('hdp-select', 'set', 'hadoop-mapreduce-historyserver', version), sudo=True)
self.assertTrue(call("tez", "hadoop", "hdfs", host_sys_prepped=False) in copy_to_hdfs_mock.call_args_list)
self.assertTrue(call("slider", "hadoop", "hdfs", host_sys_prepped=False) in copy_to_hdfs_mock.call_args_list)
# Repeated twice due to call to conf_select.get_hadoop_conf_dir()
self.assertResourceCalled("Execute", ("cp", "-R", "-p", "/etc/hadoop/conf", "/etc/hadoop/conf.backup"),
not_if = "test -e /etc/hadoop/conf.backup",
sudo = True)
self.assertResourceCalled("Directory", "/etc/hadoop/conf",
action = ["delete"])
self.assertResourceCalled("Link", "/etc/hadoop/conf", to="/usr/hdp/current/hadoop-client/conf")
self.assertResourceCalled("Execute", ("cp", "-R", "-p", "/etc/hadoop/conf", "/etc/hadoop/conf.backup"),
not_if = "test -e /etc/hadoop/conf.backup",
sudo = True)
self.assertResourceCalled("Directory", "/etc/hadoop/conf",
action = ["delete"])
self.assertResourceCalled("Link", "/etc/hadoop/conf", to="/usr/hdp/current/hadoop-client/conf")
self.assertResourceCalled('HdfsResource', None,
security_enabled = False,
hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
keytab = UnknownConfigurationMock(),
kinit_path_local = '/usr/bin/kinit',
user = '******',
action = ['execute'], hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore', hdfs_site=self.getConfig()['configurations']['hdfs-site'], principal_name=UnknownConfigurationMock(), default_fs='hdfs://c6401.ambari.apache.org:8020',
hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
)
self.assertNoMoreResources()
self.assertEquals(5, mocks_dict['call'].call_count)
self.assertEquals(5, mocks_dict['checked_call'].call_count)
self.assertEquals(
('conf-select', 'set-conf-dir', '--package', 'hadoop', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
mocks_dict['checked_call'].call_args_list[0][0][0])
self.assertEquals(
('conf-select', 'create-conf-dir', '--package', 'hadoop', '--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
mocks_dict['call'].call_args_list[0][0][0])
def test_do_delete_by_owner(self, listdir_mock, join_mock, stat_mock, do_erase_dir_silent_method): listdir_mock.return_value = ["k", "j"] join_mock.return_value = "path" response = MagicMock() response.st_uid = 1 stat_mock.return_value = response self.hostcleanup.do_delete_by_owner([1, 2], ["a"]) self.assertTrue(do_erase_dir_silent_method.called) calls = [call(["path"]), call(["path"])] do_erase_dir_silent_method.assert_has_calls(calls)
def test_action_install_ubuntu_update(self, shell_mock, call_mock):
call_mock.return_value= (1, None)
with Environment('/') as env:
Package("some_package",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package$'"),
call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'}),
call(['/usr/bin/apt-get', 'update', '-qq'], logoutput=False, sudo=True)])
shell_mock.assert_has_calls([call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef',
'--allow-unauthenticated', '--assume-yes', 'install', 'some-package'], logoutput=False, sudo=True)])
def test_action_install_ubuntu(self, shell_mock, call_mock):
call_mock.side_effect = [(1, None), (0, None)]
with Environment('/') as env:
Package("some_package",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"),
call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'"
" --allow-unauthenticated --assume-yes install some-package")
])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_action_install_ubuntu_update(self, shell_mock, call_mock):
call_mock.return_value= (1, None)
with Environment('/') as env:
Package("some_package",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep ^some-package$ | grep -v deinstall"),
call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef'"
" --allow-unauthenticated --assume-yes install some-package"),
call("apt-get update -qq")
])
shell_mock.assert_has_calls([call("DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get -q -o Dpkg::Options::='--force-confdef' --allow-unauthenticated --assume-yes install some-package")
])
def test_format_labels_with_multiple_langs(self):
labels = {
'en': ['en_label'],
'sv': ['sv_label']
}
self.preview_item.labels_dict = OrderedDict(
sorted(labels.items(), key=lambda t: t[0]))
expected = (
'* bold_en: italics_en_label\n'
'* bold_sv: italics_sv_label\n'
)
self.assertEqual(self.preview_item.format_labels(), expected)
self.mock_bold.assert_has_calls([mock.call('en'), mock.call('sv')])
def test_make_wikidata_template_with_data(self):
descriptions = {
'en': 'en_desc',
'sv': 'sv_desc'
}
self.preview_item.desc_dict = OrderedDict(
sorted(descriptions.items(), key=lambda t: t[0]))
expected = (
'* bold_en: en_desc\n'
'* bold_sv: sv_desc\n'
)
self.assertEqual(self.preview_item.format_descriptions(), expected)
self.mock_bold.assert_has_calls([mock.call('en'), mock.call('sv')])
def test_sat_6_dot_1(self):
"""Check if can parse major 6.2.x versions"""
ssh_result_success = mock.Mock()
ssh_result_success.return_code = 0
ssh_result_success.stdout = [u' VERSION = "6.1.8"']
self._command.side_effect = (self.SSH_RESULT_ERROR, ssh_result_success)
sat_version = host_info.get_host_sat_version.__wrapped__()
self.assertEqual(u"6.1", sat_version)
calls = [
call(host_info._SAT_6_2_VERSION_COMMAND),
call(host_info._SAT_6_1_VERSION_COMMAND)
]
self._command.assert_has_calls(calls)
def test_action_install_regex_installed_ubuntu(self, shell_mock, call_mock):
call_mock.side_effect = [(0, None),
(0, "some-package1\nsome-package2"),
(0, "Some text.\nStatus: install ok installed\nSome text"),
(0, "Some text.\nStatus: install ok installed\nSome text"),
(0, None)]
with Environment('/') as env:
Package("some_package.*",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package.*$'"),
call("apt-cache --names-only search '^some-package.*$' | awk '{print $1}'"),
call("dpkg --status 'some-package1'"),
call("dpkg --status 'some-package2'")])
self.assertEqual(call_mock.call_count, 4, "Package should not be installed")
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_command_error(self, logger):
"""Check returns 'Not Available' on error
"""
self._command.return_value = self.SSH_RESULT_ERROR
sat_version = host_info.get_host_sat_version.__wrapped__()
self.assertEqual('Not Available', sat_version)
calls = [
call(host_info._SAT_6_2_VERSION_COMMAND),
call(host_info._SAT_6_1_VERSION_COMMAND)
]
self._command.assert_has_calls(calls)
logger.warning.assert_called_once_with(
u'Host Satellite version not available: %r' % self.SSH_RESULT_ERROR
)
def test_action_install_regex_ubuntu(self, shell_mock, call_mock):
call_mock.side_effect = [(0, None),
(0, "some-package1\nsome-package2"),
(0, "Some text.\nStatus: install ok installed\nSome text"),
(0, "Some text.\nStatus: not installed\nSome text"),
(0, None)]
with Environment('/') as env:
Package("some_package.*",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep '^some-package.*$'"),
call("apt-cache --names-only search '^some-package.*$' | awk '{print $1}'"),
call("dpkg --status 'some-package1'"),
call("dpkg --status 'some-package2'"),
call(['/usr/bin/apt-get', '-q', '-o', 'Dpkg::Options::=--force-confdef', '--allow-unauthenticated', '--assume-yes', 'install', 'some-package.*'], logoutput=False, sudo=True, env={'DEBIAN_FRONTEND': 'noninteractive'})])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_action_install_regex_ubuntu(self, shell_mock, call_mock):
call_mock.side_effect = [(0, None)]
with Environment('/') as env:
Package("some_package.*",
)
call_mock.assert_has_calls([call("dpkg --get-selections | grep -v deinstall | awk '{print $1}' | grep ^some-package.*$")])
self.assertEqual(shell_mock.call_count, 0, "shell.checked_call shouldn't be called")
def test_daemonize_and_stop(self, exists_mock, sleep_mock):
from ambari_commons.shell import shellRunnerLinux
oldpid = ProcessHelper.pidfile
pid = str(os.getpid())
_, tmpoutfile = tempfile.mkstemp()
ProcessHelper.pidfile = tmpoutfile
# Test daemonization
main.daemonize()
saved = open(ProcessHelper.pidfile, 'r').read()
self.assertEqual(pid, saved)
main.GRACEFUL_STOP_TRIES = 1
with patch("ambari_commons.shell.shellRunnerLinux.run") as kill_mock:
# Reuse pid file when testing agent stop
# Testing normal exit
exists_mock.return_value = False
kill_mock.side_effect = [{
'exitCode': 0,
'output': '',
'error': ''
}, {
'exitCode': 1,
'output': '',
'error': ''
}]
try:
main.stop_agent()
raise Exception("main.stop_agent() should raise sys.exit(0).")
except SystemExit as e:
self.assertEquals(0, e.code)
kill_mock.assert_has_calls([
call(['ambari-sudo.sh', 'kill', '-15', pid]),
call(['ambari-sudo.sh', 'kill', '-0', pid])
])
# Restore
kill_mock.reset_mock()
kill_mock.side_effect = [{
'exitCode': 0,
'output': '',
'error': ''
}, {
'exitCode': 0,
'output': '',
'error': ''
}, {
'exitCode': 0,
'output': '',
'error': ''
}]
# Testing exit when failed to remove pid file
exists_mock.return_value = True
try:
main.stop_agent()
raise Exception("main.stop_agent() should raise sys.exit(0).")
except SystemExit as e:
self.assertEquals(0, e.code)
kill_mock.assert_has_calls([
call(['ambari-sudo.sh', 'kill', '-15', pid]),
call(['ambari-sudo.sh', 'kill', '-0', pid]),
call(['ambari-sudo.sh', 'kill', '-9', pid])
])
# Restore
ProcessHelper.pidfile = oldpid
os.remove(tmpoutfile)
def test_purge_stacks_and_mpacks(self, get_ambari_version_mock, os_path_exists_mock):
options = self._create_empty_options_mock()
get_ambari_version_mock.return_value = configs
stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
os_path_exists_mock.side_effect = [True]
purge_stacks_and_mpacks(None)
os_path_exists_calls = [
call('/var/lib/ambari-server/resources'),
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
os_path_exists_mock.side_effect = [True, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(mpacks_directory)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
os_path_exists_mock.side_effect = [True, False, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(common_services_directory),
call(mpacks_directory)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, EXTENSION_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
os_path_exists_mock.side_effect = [True, False, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(extensions_directory),
call(mpacks_directory)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
options.purge_list = ",".join([STACK_DEFINITIONS_RESOURCE_NAME, SERVICE_DEFINITIONS_RESOURCE_NAME, MPACKS_RESOURCE_NAME])
options.replay_mode = True
os_path_exists_mock.side_effect = [True, False, False, False]
purge_stacks_and_mpacks(options.purge_list.split(","))
os_path_exists_calls = [
call('/var/lib/ambari-server/resources'),
call(stacks_directory),
call(common_services_directory),
call(mpacks_directory)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
def test_do_delete_users(self, run_os_command_mock):
run_os_command_mock.return_value = (1, "", "")
self.hostcleanup.do_delete_users(["a", "b"])
self.assertTrue(run_os_command_mock.called)
calls = [call('userdel -rf a'), call('userdel -rf b'), call('groupdel hadoop')]
run_os_command_mock.assert_has_calls(calls)
def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):
config_file = self.get_src_folder(
) + "/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
copy_to_hdfs_mock.return_value = True
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/historyserver.py",
classname="HistoryServer",
command="pre_upgrade_restart",
config_dict=json_content,
hdp_stack_version=self.STACK_VERSION,
target=RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks=[(0, None, ''), (0, None, None),
(0, None, None), (0, None, None),
(0, None, None)],
mocks_dict=mocks_dict)
self.assertResourceCalledIgnoreEarlier(
'Execute',
('hdp-select', 'set', 'hadoop-mapreduce-historyserver', version),
sudo=True)
self.assertTrue(
call("tez", "hadoop", "hdfs", host_sys_prepped=False) in
copy_to_hdfs_mock.call_args_list)
self.assertTrue(
call("slider", "hadoop", "hdfs", host_sys_prepped=False) in
copy_to_hdfs_mock.call_args_list)
# Repeated twice due to call to conf_select.get_hadoop_conf_dir()
self.assertResourceCalled(
"Execute",
("cp", "-R", "-p", "/etc/hadoop/conf", "/etc/hadoop/conf.backup"),
not_if="test -e /etc/hadoop/conf.backup",
sudo=True)
self.assertResourceCalled("Directory",
"/etc/hadoop/conf",
action=["delete"])
self.assertResourceCalled("Link",
"/etc/hadoop/conf",
to="/usr/hdp/current/hadoop-client/conf")
self.assertResourceCalled(
"Execute",
("cp", "-R", "-p", "/etc/hadoop/conf", "/etc/hadoop/conf.backup"),
not_if="test -e /etc/hadoop/conf.backup",
sudo=True)
self.assertResourceCalled("Directory",
"/etc/hadoop/conf",
action=["delete"])
self.assertResourceCalled("Link",
"/etc/hadoop/conf",
to="/usr/hdp/current/hadoop-client/conf")
self.assertResourceCalled(
'HdfsResource',
None,
security_enabled=False,
hadoop_bin_dir='/usr/hdp/current/hadoop-client/bin',
keytab=UnknownConfigurationMock(),
kinit_path_local='/usr/bin/kinit',
user='******',
action=['execute'],
hdfs_site=self.getConfig()['configurations']['hdfs-site'],
principal_name=UnknownConfigurationMock(),
default_fs='hdfs://c6401.ambari.apache.org:8020',
hadoop_conf_dir='/usr/hdp/current/hadoop-client/conf',
)
self.assertNoMoreResources()
self.assertEquals(5, mocks_dict['call'].call_count)
self.assertEquals(5, mocks_dict['checked_call'].call_count)
self.assertEquals(
('conf-select', 'set-conf-dir', '--package', 'hadoop',
'--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
mocks_dict['checked_call'].call_args_list[0][0][0])
self.assertEquals(
('conf-select', 'create-conf-dir', '--package', 'hadoop',
'--stack-version', '2.3.0.0-1234', '--conf-version', '0'),
mocks_dict['call'].call_args_list[0][0][0])
def test_heartbeatWithServer(self, dumpsMock, sleepMock, event_mock):
out = StringIO.StringIO()
sys.stdout = out
hearbeat = MagicMock()
self.controller.heartbeat = hearbeat
dumpsMock.return_value = "data"
sendRequest = MagicMock(name="sendRequest")
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
response = {"responseId": "2", "restartAgent": "false"}
sendRequest.return_value = response
def one_heartbeat(*args, **kwargs):
self.controller.DEBUG_STOP_HEARTBEATING = True
return response
sendRequest.side_effect = one_heartbeat
actionQueue = MagicMock()
actionQueue.isIdle.return_value = True
# one successful request, after stop
self.controller.actionQueue = actionQueue
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
calls = []
def retry(*args, **kwargs):
if len(calls) == 0:
calls.append(1)
response["responseId"] = "3"
raise Exception()
if len(calls) > 0:
self.controller.DEBUG_STOP_HEARTBEATING = True
return response
# exception, retry, successful and stop
sendRequest.side_effect = retry
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
# retry registration
self.controller.responseId = 2
response["registrationCommand"] = "true"
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.repeatRegistration)
# components are not mapped
self.controller.responseId = 2
response["registrationCommand"] = "false"
response["hasMappedComponents"] = False
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertFalse(self.controller.hasMappedComponents)
# components are mapped
self.controller.responseId = 2
response["hasMappedComponents"] = True
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.hasMappedComponents)
# components are mapped
self.controller.responseId = 2
del response["hasMappedComponents"]
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.hasMappedComponents)
# wrong responseId => restart
self.controller.responseId = 2
response = {"responseId": "2", "restartAgent": "false"}
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
restartAgent.assert_called_once_with()
# executionCommands
self.controller.responseId = 1
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
response["executionCommands"] = "executionCommands"
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([call("executionCommands")])
# statusCommands
self.controller.responseId = 1
addToStatusQueue = MagicMock(name="addToStatusQueue")
self.controller.addToStatusQueue = addToStatusQueue
response["statusCommands"] = "statusCommands"
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToStatusQueue.assert_has_calls([call("statusCommands")])
# restartAgent command
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
response["restartAgent"] = "true"
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.heartbeatWithServer()
restartAgent.assert_called_once_with()
# actionQueue not idle
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
actionQueue.isIdle.return_value = False
response["restartAgent"] = "false"
self.controller.heartbeatWithServer()
sleepMock.assert_called_with(
self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
sys.stdout = sys.__stdout__
self.controller.sendRequest = Controller.Controller.sendRequest
self.controller.sendRequest = Controller.Controller.addToQueue
self.controller.sendRequest = Controller.Controller.addToStatusQueue
def test_get_installed_package_version_centos_suse(self, is_ubuntu_family_mock, checked_call_mock):
is_ubuntu_family_mock.return_value = False
checked_call_mock.return_value = (0, '0.0.1-SNAPSHOT','')
result = packages_analyzer.getInstalledPackageVersion("package1")
self.assertEqual(result, '0.0.1-SNAPSHOT')
self.assertEqual(checked_call_mock.call_args_list, [call("rpm -q --queryformat '%{version}-%{release}' package1 | sed -e 's/\\.el[0-9]//g'", stderr=-1)])
def test_decompress_data_2_chunks(self,
mocked_decompress_data,
mocked_reverse_dictionary,
mocked_generate_dictionary,
mocked_chunk_file):
bytes_data = TestLZW._wiki_bytes
dictionary = TestLZW._wiki_dictionary()
reverse_dictionary = TestLZW._wiki_reversed_dictionary()
decompress_data_first_call_result = ('TOBEOR', '00', 'R')
decompress_data_second_call_result = ('NOTTOBEORTOBEORNOT', TestLZW._empty_str, 'OT')
mocked_chunk_file.return_value = {2, 4}
mocked_generate_dictionary.return_value = dictionary
mocked_reverse_dictionary.return_value = reverse_dictionary
mocked_decompress_data.side_effect = [decompress_data_first_call_result, decompress_data_second_call_result]
expected_decompress_data_calls_num = 2
expected_decompress_data_calls = [
mock.call(
'10101100000001100110100001001100',
dictionary,
reverse_dictionary,
initial_phrase=TestLZW._empty_str
),
mock.call(
'00' + '1111010000010101011011011101011111100100011110100000100010' + '000000',
dictionary,
reverse_dictionary,
initial_phrase='R'
)]
expected_decompressed_data = 'TOBEORNOTTOBEORTOBEORNOT'
read_file_path = configuration.test_file_path('decompress_wiki_2_chunks')
write_file_path = configuration.test_file_path('decompress_wiki_2_chunks_decompressed')
initializing_stream = None
check_stream = None
try:
initializing_stream = open(read_file_path, 'wb')
initializing_stream.write(bytes_data)
initializing_stream.close()
lzw.decompress(read_file_path, write_file_path)
check_stream = open(write_file_path, 'r')
decompressed_data = check_stream.read()
check_stream.close()
self.assertEqual(expected_decompress_data_calls_num, mocked_decompress_data.call_count)
mocked_decompress_data.assert_has_calls(expected_decompress_data_calls)
self.assertEqual(expected_decompressed_data, decompressed_data)
finally:
if not (initializing_stream is None) and not initializing_stream.closed:
initializing_stream.close()
if not (check_stream is None) and not check_stream.closed:
initializing_stream.close()
os.remove(read_file_path)
os.remove(write_file_path)
def test_upgrade_stack_mpack(self, download_mpack_mock, expand_mpack_mock,
purge_stacks_and_mpacks_mock,
uninstall_mpack_mock, add_replay_log_mock,
get_ambari_properties_mock,
get_ambari_version_mock, create_symlink_mock,
os_mkdir_mock, shutil_move_mock,
os_path_exists_mock):
options = self._create_empty_options_mock()
options.mpack_path = "/path/to/mystack-1.0.0.1.tar.gz"
download_mpack_mock.side_effect = [
"/tmp/mystack-1.0.0.1.tar.gz", "/tmp/mystack-1.0.0.1.tar.gz"
]
expand_mpack_mock.side_effect = [
"mpacks/mystack-ambari-mpack-1.0.0.1",
"mpacks/mystack-ambari-mpack-1.0.0.1"
]
get_ambari_version_mock.return_value = "2.4.0.0"
"""
os_path_exists_calls = [call('/tmp/mystack-1.0.0.1.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call(mpacks_directory),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call('/tmp/mystack-1.0.0.1.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/common-services'),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1'),
call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
call('/var/lib/ambari-server/resources/common-services/SERVICEB'),
call('/var/lib/ambari-server/resources/common-services/SERVICEC'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.1'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.1/services'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0/services')]
call('/var/lib/ambari-server/resources/stacks/MYSTACK/3.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/3.0/services'),
call(mpacks_directory),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/mpack.json')]
"""
os_path_exists_mock.side_effect = [
True, True, True, True, True, True, True, True, True, True, True,
True, False, True, True, False, True, True, True, True, True, True,
True, False, False, True, True, True, True
]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
upgrade_mpack(options)
stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
common_services_directory = configs[
serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[
serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(
mpacks_directory, "mystack-ambari-mpack-1.0.0.1")
os_mkdir_calls = [
call(os.path.join(common_services_directory, "SERVICEC")),
call(os.path.join(stacks_directory, "MYSTACK/3.0")),
call(os.path.join(stacks_directory, "MYSTACK/3.0/services"))
]
create_symlink_calls = [
call(
os.path.join(mpacks_staging_directory,
"common-services/SERVICEA"),
os.path.join(common_services_directory, "SERVICEA"), "1.0",
True),
call(
os.path.join(mpacks_staging_directory,
"common-services/SERVICEA"),
os.path.join(common_services_directory, "SERVICEA"), "2.0",
True),
call(
os.path.join(mpacks_staging_directory,
"common-services/SERVICEB"),
os.path.join(common_services_directory, "SERVICEB"), "1.0.0",
True),
call(
os.path.join(mpacks_staging_directory,
"common-services/SERVICEB"),
os.path.join(common_services_directory, "SERVICEB"), "2.0.0",
True),
call(
os.path.join(mpacks_staging_directory,
"common-services/SERVICEC"),
os.path.join(common_services_directory, "SERVICEC"), "1.0.0",
True),
call(
os.path.join(mpacks_staging_directory,
"common-services/SERVICEC"),
os.path.join(common_services_directory, "SERVICEC"), "2.0.0",
True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.0"),
os.path.join(stacks_directory, "MYSTACK/1.0"), "metainfo.xml",
True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/1.0/services"),
os.path.join(stacks_directory, "MYSTACK/1.0/services"),
"SERVICEA", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.1"),
os.path.join(stacks_directory, "MYSTACK/1.1"), "metainfo.xml",
True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/1.1/services"),
os.path.join(stacks_directory, "MYSTACK/1.1/services"),
"SERVICEA", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0"),
os.path.join(stacks_directory, "MYSTACK/2.0"), "metainfo.xml",
True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/2.0/services"),
os.path.join(stacks_directory, "MYSTACK/2.0/services"),
"SERVICEA", True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/2.0/services"),
os.path.join(stacks_directory, "MYSTACK/2.0/services"),
"SERVICEB", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0"),
os.path.join(stacks_directory, "MYSTACK/3.0"), "metainfo.xml",
True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/3.0/services"),
os.path.join(stacks_directory, "MYSTACK/3.0/services"),
"SERVICEA", True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/3.0/services"),
os.path.join(stacks_directory, "MYSTACK/3.0/services"),
"SERVICEB", True),
call(
os.path.join(mpacks_staging_directory,
"stacks/MYSTACK/3.0/services"),
os.path.join(stacks_directory, "MYSTACK/3.0/services"),
"SERVICEC", True)
]
self.assertFalse(purge_stacks_and_mpacks_mock.called)
os_mkdir_mock.assert_has_calls(os_mkdir_calls)
create_symlink_mock.assert_has_calls(create_symlink_calls)
uninstall_mpack_mock.assert_has_calls(
[call("mystack-ambari-mpack", "1.0.0.0")])
self.assertTrue(add_replay_log_mock.called)
def test_execute_will_put_op_request_with_url_and_params_when_form_data_param_set(self, mock_put):
self.http_op = HttpOperation('put', 'https://server.de/', 'pet/{petId}/uploadImage',
self.SAMPLE_OP_INFOS, False)
self.http_op.execute(ReplicatorTest.SAMPLE_DEFINITION)
self.assertIn(mock.call(data={'status': '', 'name': ''},
url='https://server.de/pet/0/uploadImage'), mock_put.call_args_list)
def test_retrieve_file_with_dirs(self, mock_ssh):
""" Test run RemoteHost.exists """
remote = common.remote_host.RemoteHost('53.1.1.1', "ssh_user", "ssh_key_file")
remote.remote_exists = Mock()
remote.remote_isdir = Mock()
remote._retrieve_file = Mock()
remote.ftp.listdir = Mock()
# remote path is a directory, with 1 directory with a single empty dir
remote.remote_exists.reset_mock()
remote.remote_isdir.reset_mock()
remote._retrieve_file.reset_mock()
remote.ftp.listdir.reset_mock()
remote.remote_exists.return_value = True
isdir_map = {'remote_dir': True, 'remote_dir/data': True}
remote.remote_isdir.side_effect = lambda name: isdir_map.get(name, False)
listdir_map = {'remote_dir': ['data']}
remote.ftp.listdir.side_effect = lambda name: listdir_map.get(name, [])
remote.retrieve_path('remote_dir', 'reports/local_dir')
self.assertTrue(
remote.ftp.listdir.mock_calls ==
[mock.call('remote_dir'), mock.call('remote_dir/data')])
# remote.ftp.listdir.assert_called_with('remote_dir')
remote._retrieve_file.assert_not_called()
# remote path is a directory, with 1 directory (with a single file)
remote.remote_exists.reset_mock()
remote.remote_isdir.reset_mock()
remote._retrieve_file.reset_mock()
remote.ftp.listdir.reset_mock()
remote.remote_exists.return_value = True
isdir_map = {'remote_dir': True, 'remote_dir/data': True}
remote.remote_isdir.side_effect = lambda name: isdir_map.get(name, False)
listdir_map = {'remote_dir': ['data'], 'remote_dir/data': ['metrics.interim']}
remote.ftp.listdir.side_effect = lambda name: listdir_map.get(name, [])
remote.retrieve_path('remote_dir', 'reports/local_dir')
self.assertTrue(
remote.ftp.listdir.mock_calls ==
[mock.call('remote_dir'), mock.call('remote_dir/data')])
# remote.ftp.listdir.assert_called_with('remote_dir')
remote._retrieve_file.assert_called_with('remote_dir/data/metrics.interim',
'reports/local_dir/data/metrics.interim')
# remote path is a directory, with multiple files
remote.remote_exists.reset_mock()
remote.remote_isdir.reset_mock()
remote._retrieve_file.reset_mock()
remote.ftp.listdir.reset_mock()
remote.remote_exists.return_value = True
isdir_map = {'remote_dir': True}
remote.remote_isdir.side_effect = lambda name: isdir_map.get(name, False)
listdir_map = {'remote_dir': ['data', 'logs']}
remote.ftp.listdir.side_effect = lambda name: listdir_map.get(name, [])
remote.retrieve_path('remote_dir', 'reports/local_dir')
remote.ftp.listdir.assert_called_with('remote_dir')
self.assertTrue(remote._retrieve_file.mock_calls == [
mock.call('remote_dir/data', 'reports/local_dir/data'),
mock.call('remote_dir/logs', 'reports/local_dir/logs')
])
def test_add_yarn_mr(self, option_parser_mock, curl_mock, backup_file_mock,
file_handler_mock, read_mapping_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
options = self.get_mock_options()
args = ["add-yarn-mr2"]
opm.parse_args.return_value = (options, args)
curl_mock.return_value = ''
read_mapping_mock.return_value = {
"TASKTRACKER": ["c6401", "c6402"],
"JOBTRACKER": ["c6401"],
"MAPREDUCE_CLIENT": ["c6403"]
}
UpgradeHelper_HDP2.main()
expected_curl_calls = [
call(False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN"),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"
),
call(
False, "-u", "admin:admin", "-X", "POST",
"http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER"
)
]
curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
pass
def test_retrieve_file_for_files(self, mock_ssh):
""" Test run RemoteHost.exists """
# remote path does not exist
remote = common.remote_host.RemoteHost('53.1.1.1', "ssh_user", "ssh_key_file")
remote.remote_exists = Mock()
remote.remote_isdir = Mock()
remote._retrieve_file = Mock()
remote.ftp.listdir = Mock()
remote.remote_exists.return_value = False
remote.retrieve_path('remote_file', 'reports/local_file')
remote.remote_isdir.assert_not_called()
# remote path is a single file
remote.remote_exists.reset_mock()
remote.remote_isdir.reset_mock()
remote._retrieve_file.reset_mock()
remote.ftp.listdir.reset_mock()
remote.remote_exists.return_value = True
remote.remote_isdir.return_value = False
remote.retrieve_path('remote_file', 'reports/local_file')
remote._retrieve_file.assert_called_with('remote_file', 'reports/local_file')
# remote path is a directory containing a single file
remote.remote_exists.reset_mock()
remote.remote_isdir.reset_mock()
remote._retrieve_file.reset_mock()
remote.ftp.listdir.reset_mock()
remote.remote_exists.return_value = True
isdir_map = {'remote_dir': True}
remote.remote_isdir.side_effect = lambda name: isdir_map.get(name, False)
remote.ftp.listdir.return_value = ['mongod.log']
remote.retrieve_path('remote_dir', 'reports/local_dir')
remote.ftp.listdir.assert_called_with('remote_dir')
remote._retrieve_file.assert_called_with('remote_dir/mongod.log',
'reports/local_dir/mongod.log')
# remote path is a directory, with multiple files
remote.remote_exists.reset_mock()
remote.remote_isdir.reset_mock()
remote._retrieve_file.reset_mock()
remote.ftp.listdir.reset_mock()
remote.remote_exists.return_value = True
isdir_map = {'remote_dir': True}
remote.remote_isdir.side_effect = lambda name: isdir_map.get(name, False)
remote.ftp.listdir.return_value = [
'mongod.log', 'metrics.2017-04-27T09-14-33Z-00000', 'metrics.interim'
]
remote.retrieve_path('remote_dir', 'reports/local_dir')
remote.ftp.listdir.assert_called_with('remote_dir')
self.assertTrue(remote._retrieve_file.mock_calls == [
mock.call('remote_dir/mongod.log', 'reports/local_dir/mongod.log'),
mock.call('remote_dir/metrics.2017-04-27T09-14-33Z-00000',
'reports/local_dir/metrics.2017-04-27T09-14-33Z-00000'),
mock.call('remote_dir/metrics.interim', 'reports/local_dir/metrics.interim')
])
def test_update_rm_config(self, mock_uc):
rm = RecoveryManager()
rm.update_recovery_config(None)
mock_uc.assert_has_calls([call(6, 60, 5, 12, False, False, False, [])])
mock_uc.reset_mock()
rm.update_recovery_config({})
mock_uc.assert_has_calls([call(6, 60, 5, 12, False, False, False, [])])
mock_uc.reset_mock()
rm.update_recovery_config(
{"recoveryConfig": {
"type" : "DEFAULT"}}
)
mock_uc.assert_has_calls([call(6, 60, 5, 12, False, False, False, [])])
mock_uc.reset_mock()
rm.update_recovery_config(
{"recoveryConfig": {
"type" : "FULL"}}
)
mock_uc.assert_has_calls([call(6, 60, 5, 12, True, False, False, [])])
mock_uc.reset_mock()
rm.update_recovery_config(
{"recoveryConfig": {
"type" : "AUTO_START",
"max_count" : "med"}}
)
mock_uc.assert_has_calls([call(6, 60, 5, 12, True, True, False, [])])
mock_uc.reset_mock()
rm.update_recovery_config(
{"recoveryConfig": {
"type" : "AUTO_INSTALL_START",
"max_count" : "med"}}
)
mock_uc.assert_has_calls([call(6, 60, 5, 12, True, False, True, [])])
mock_uc.reset_mock()
rm.update_recovery_config(
{"recoveryConfig": {
"type": "AUTO_START",
"maxCount": "5",
"windowInMinutes" : 20,
"retryGap": 2,
"maxLifetimeCount" : 5,
"components": [
{
"service_name": "A",
"component_name": "A",
"desired_state": "INSTALLED"
},
{
"service_name": "B",
"component_name": "B",
"desired_state": "INSTALLED"
}
],
"recoveryTimestamp": 1}}
)
mock_uc.assert_has_calls([call(5, 20, 2, 5, True, True, False, [
{'component_name': 'A', 'service_name': 'A', 'desired_state': 'INSTALLED'},
{'component_name': 'B', 'service_name': 'B', 'desired_state': 'INSTALLED'}
])])
def test_runCommand_with_shell_config(self, run_file_mock, unlink_mock,
isfile_mock, hostname_mock):
hostname_mock.return_value = "test.hst"
isfile_mock.return_value = True
command = {
'role': 'MEMCACHED',
'componentName': 'MEMCACHED',
'hostLevelParams': {
'jdk_location': 'some_location'
},
'commandParams': {
'script_type': 'SHELL',
'command_timeout': '600'
},
'configurations': {
"memcached-site": {
"memcached.log": "${AGENT_LOG_ROOT}",
"memcached.number": "10485760"
},
"memcached-log4j": {
"a": "b"
}
},
'taskId': '3',
'roleCommand': 'INSTALL',
'commandType': 'EXECUTION_COMMAND',
'commandId': '1-1'
}
command_get = {
'roleCommand': 'GET_CONFIG',
'commandType': 'STATUS_COMMAND'
}
command_get_specific = {
'roleCommand': 'GET_CONFIG',
'commandType': 'STATUS_COMMAND',
'commandParams': {
'config_type': 'memcached-site'
}
}
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller,
self.agentToggleLogger)
# normal run case
run_file_mock.return_value = {
'stdout': 'sss',
'stderr': 'eee',
'exitcode': 0,
}
expected = {
'memcached-site': {
'memcached.log': tempdir,
'memcached.number': '10485760'
},
'memcached-log4j': {
'a': 'b'
}
}
expected_specific = {
'memcached-site': {
'memcached.log': tempdir,
'memcached.number': '10485760'
},
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt", True,
True)
self.assertEqual.__self__.maxDiff = None
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
ret = orchestrator.requestComponentStatus(command_get)
self.assertEqual(ret['configurations'], expected)
ret = orchestrator.requestComponentStatus(command_get_specific)
self.assertEqual(ret['configurations'], expected_specific)
script_path = os.path.realpath(
posixpath.join(tempdir, "infra", "agent", "slider-agent",
"scripts", "shell_cmd", "basic_installer.py"))
run_file_mock.assert_has_calls([
call(
script_path, [
'INSTALL',
os.path.realpath(posixpath.join(tempdir,
'command-3.json')),
os.path.realpath(posixpath.join(tempdir, 'package'))
], 'out.txt', 'err.txt', 600,
os.path.realpath(
posixpath.join(tempdir, 'structured-out-3.json')), 'INFO',
True, [('PYTHONPATH', ":".join([
os.path.realpath(
posixpath.join(tempdir, 'infra', 'agent',
'slider-agent', 'jinja2')),
os.path.realpath(
posixpath.join(tempdir, 'infra', 'agent',
'slider-agent'))
]))])
])
pass
def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):
config_file = self.get_src_folder(
) + "/test/python/stacks/2.0.6/configs/default.json"
with open(config_file, "r") as f:
json_content = json.load(f)
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
json_content['hostLevelParams']['stack_version'] = '2.3'
copy_to_hdfs_mock.return_value = True
mocks_dict = {}
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR +
"/scripts/historyserver.py",
classname="HistoryServer",
command="pre_upgrade_restart",
config_dict=json_content,
stack_version=self.STACK_VERSION,
target=RMFTestCase.TARGET_COMMON_SERVICES,
call_mocks=[(0, None, ''), (0, None, None),
(0, None, None), (0, None, None),
(0, None, None)],
mocks_dict=mocks_dict)
self.assertResourceCalledIgnoreEarlier(
'Execute', ('ambari-python-wrap', '/usr/bin/hdp-select', 'set',
'hadoop-mapreduce-historyserver', version),
sudo=True)
self.assertTrue(
call("tez", "hadoop", "hdfs", skip=False) in
copy_to_hdfs_mock.call_args_list)
self.assertTrue(
call("slider", "hadoop", "hdfs", skip=False) in
copy_to_hdfs_mock.call_args_list)
# From call to conf_select.get_hadoop_conf_dir()
self.assert_call_to_get_hadoop_conf_dir()
self.assert_call_to_get_hadoop_conf_dir()
self.assertResourceCalled(
'HdfsResource',
None,
immutable_paths=self.DEFAULT_IMMUTABLE_PATHS,
security_enabled=False,
hadoop_bin_dir='/usr/hdp/current/hadoop-client/bin',
keytab=UnknownConfigurationMock(),
kinit_path_local='/usr/bin/kinit',
user='******',
dfs_type='',
action=['execute'],
hdfs_resource_ignore_file=
'/var/lib/ambari-agent/data/.hdfs_resource_ignore',
hdfs_site=self.getConfig()['configurations']['hdfs-site'],
principal_name=UnknownConfigurationMock(),
default_fs='hdfs://c6401.ambari.apache.org:8020',
hadoop_conf_dir='/usr/hdp/current/hadoop-client/conf',
)
self.assertNoMoreResources()
self.assertEquals(5, mocks_dict['call'].call_count)
self.assertEquals(5, mocks_dict['checked_call'].call_count)
self.assertEquals(
('ambari-python-wrap', '/usr/bin/conf-select', 'set-conf-dir',
'--package', 'hadoop', '--stack-version', '2.3.0.0-1234',
'--conf-version', '0'),
mocks_dict['checked_call'].call_args_list[0][0][0])
self.assertEquals(
('ambari-python-wrap', '/usr/bin/conf-select', 'create-conf-dir',
'--package', 'hadoop', '--stack-version', '2.3.0.0-1234',
'--conf-version', '0'),
mocks_dict['call'].call_args_list[0][0][0])
def test_install_stack_mpack(self, set_file_permissions_mock, validate_purge_mock, run_os_command_mock, download_mpack_mock, expand_mpack_mock, purge_stacks_and_mpacks_mock,
add_replay_log_mock, get_ambari_properties_mock, get_ambari_version_mock,
create_symlink_mock, os_mkdir_mock, shutil_move_mock, os_path_exists_mock):
options = self._create_empty_options_mock()
options.mpack_path = "/path/to/mystack.tar.gz"
options.purge = True
download_mpack_mock.return_value = "/tmp/mystack.tar.gz"
expand_mpack_mock.return_value = "mpacks/mystack-ambari-mpack-1.0.0.0"
get_ambari_version_mock.return_value = "2.4.0.0"
run_os_command_mock.return_value = (0, "", "")
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
os_path_exists_calls = [call('/tmp/mystack.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.0/mpack.json'),
call('mpacks/mystack-ambari-mpack-1.0.0.0/hooks/before_install.py'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/1.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEA/2.0/dashboards'),
call('/var/lib/ambari-server/resources/common-services/SERVICEB'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/1.0.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/common-services/SERVICEB/2.0.0/dashboards'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/1.0/services/SERVICEA/dashboards'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.1'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.1/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/1.1/services/SERVICEA/dashboards'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEA/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/stacks/MYSTACK/2.0/services/SERVICEB/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/hooks/after_install.py')]
os_path_exists_mock.side_effect = [True, True, True, True, False, True, False, False, False, False,
False, True, False, False, False, False, False, False, True, False,
False, False, False, False, False, False, False, False, False, False,
True]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
try:
install_mpack(options)
except Exception as e:
print e
stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
extensions_directory = configs[serverConfiguration.EXTENSION_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.0")
resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
dashboards_directory = os.path.join(resources_directory, "dashboards")
run_os_command_calls = [
call([
"/usr/bin/ambari-python-wrap",
"mpacks/mystack-ambari-mpack-1.0.0.0/hooks/before_install.py"
]),
call([
"/usr/bin/ambari-python-wrap",
mpacks_directory + "/mystack-ambari-mpack-1.0.0.0/hooks/after_install.py"
])
]
os_mkdir_calls = [
call(stacks_directory),
call(common_services_directory),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call(dashboards_directory),
call(os.path.join(dashboards_directory, GRAFANA_DASHBOARDS_DIRNAME)),
call(os.path.join(dashboards_directory, SERVICE_METRICS_DIRNAME)),
call(os.path.join(common_services_directory, "SERVICEA")),
call(os.path.join(common_services_directory, "SERVICEB")),
call(os.path.join(stacks_directory, "MYSTACK")),
call(os.path.join(stacks_directory, "MYSTACK/1.0")),
call(os.path.join(stacks_directory, "MYSTACK/1.0/services")),
call(os.path.join(stacks_directory, "MYSTACK/1.1")),
call(os.path.join(stacks_directory, "MYSTACK/1.1/services")),
call(os.path.join(stacks_directory, "MYSTACK/2.0")),
call(os.path.join(stacks_directory, "MYSTACK/2.0/services"))
]
create_symlink_calls = [
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEA"),
os.path.join(common_services_directory, "SERVICEA"),
"1.0", None),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEA"),
os.path.join(common_services_directory, "SERVICEA"),
"2.0", None),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEB"),
os.path.join(common_services_directory, "SERVICEB"),
"1.0.0", None),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEB"),
os.path.join(common_services_directory, "SERVICEB"),
"2.0.0", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.0"),
os.path.join(stacks_directory, "MYSTACK/1.0"),
"metainfo.xml", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.0/services"),
os.path.join(stacks_directory, "MYSTACK/1.0/services"),
"SERVICEA", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.1"),
os.path.join(stacks_directory, "MYSTACK/1.1"),
"metainfo.xml", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.1/services"),
os.path.join(stacks_directory, "MYSTACK/1.1/services"),
"SERVICEA", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0"),
os.path.join(stacks_directory, "MYSTACK/2.0"),
"metainfo.xml", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0/services"),
os.path.join(stacks_directory, "MYSTACK/2.0/services"),
"SERVICEA", None),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0/services"),
os.path.join(stacks_directory, "MYSTACK/2.0/services"),
"SERVICEB", None)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
self.assertTrue(purge_stacks_and_mpacks_mock.called)
run_os_command_mock.assert_has_calls(run_os_command_calls)
os_mkdir_mock.assert_has_calls(os_mkdir_calls)
create_symlink_mock.assert_has_calls(create_symlink_calls)
self.assertTrue(add_replay_log_mock.called)
def test_save_configs(self, option_parser_mock, path_exists_mock,
shutil_copy_mock, os_remove_mock,
get_config_resp_mock, write_config_mock,
file_handler_mock, logging_mock, set_formatter_mock):
file_handler_mock.return_value = logging.FileHandler(
'') # disable creating real file
opm = option_parser_mock.return_value
path_exists_mock.return_value = True
shutil_copy_mock = MagicMock()
options = self.get_mock_options()
args = ["backup-configs"]
opm.parse_args.return_value = (options, args)
def get_config_resp_side_effect(ops, type, error_if_na):
if type == "global":
return ("version1", "")
elif type == "core-site":
return ("version1", "")
elif type == "hdfs-site":
return ("version1", "")
elif type == "mapred-site":
return ("version2", "")
elif type == "hbase-site":
return ("version2", "")
else:
return (None, None)
get_config_resp_mock.side_effect = get_config_resp_side_effect
UpgradeHelper_HDP2.main()
shutil_copy_mock.assert_called_once()
os_remove_mock.assert_called_once_with("logfile")
write_expected = [
call("", "global", "version1"),
call("", "core-site", "version1"),
call("", "hdfs-site", "version1"),
call("", "mapred-site", "version2"),
call("", "hbase-site", "version2")
]
write_config_mock.assert_has_calls(write_expected, any_order=True)
get_config_expected = [
call(options, "global", True),
call(options, "core-site", True),
call(options, "hdfs-site", True),
call(options, "mapred-site", True),
call(options, "hbase-site", False),
call(options, "oozie-site", False),
call(options, "webhcat-site", False),
call(options, "hive-site", False)
]
get_config_resp_mock.assert_has_calls(get_config_expected,
any_order=True)
pass
def test_copy_to_output_dir_exists_and_empty(os_, shutil_,
_copy_soscleaner_files):
'''
Test that writing to an existing but empty directory is
performed
'''
config = InsightsConfig(output_dir='dest')
client = InsightsClient(config)
# raise file exists error first, then raise for "b" and "c" below
shutil_.copytree.side_effect = [OSError(17, 'File exists'), None, None]
# returns empty list for destination, file list for source
os_.listdir.side_effect = [[], ['a', 'b', 'c']]
# os.path.join called 6 times, once for each file per src and dest
os_.path.join.side_effect = [
os.path.join('src', 'a'),
os.path.join(config.output_dir, 'a'),
os.path.join('src', 'b'),
os.path.join(config.output_dir, 'b'),
os.path.join('src', 'c'),
os.path.join(config.output_dir, 'c')
]
# 'a' is file, 'b', 'c' are dirs
os_.path.isfile.side_effect = [True, False, False]
# a is file so the check for 'a' does not fall through to the elif
os_.path.isdir.side_effect = [True, True]
client.copy_to_output_dir('src')
os_.listdir.assert_has_calls([call(config.output_dir), call('src')])
os_.path.isfile.assert_has_calls(
[call('src/a'), call('src/b'),
call('src/c')])
# a is file so the check for 'a' does not fall through to the elif
os_.path.isdir.assert_has_calls([call('src/b'), call('src/c')])
# initial (failed) copy is part of the calls
shutil_.copytree.assert_has_calls([
call('src', config.output_dir),
call(os.path.join('src', 'b'), os.path.join(config.output_dir, 'b')),
call(os.path.join('src', 'c'), os.path.join(config.output_dir, 'c'))
])
shutil_.copyfile.assert_has_calls(
[call(os.path.join('src', 'a'), os.path.join(config.output_dir, 'a'))])
_copy_soscleaner_files.assert_not_called()
def test_get_installed_package_version_ubuntu(self, is_ubuntu_family_mock, checked_call_mock):
is_ubuntu_family_mock.return_value = True
checked_call_mock.return_value = (0, '1.2.3','')
result = packages_analyzer.getInstalledPackageVersion("package1")
self.assertEqual(result, '1.2.3')
self.assertEqual(checked_call_mock.call_args_list, [call("dpkg -s package1 | grep Version | awk '{print $2}'", stderr=-1)])
def test_execute_will_delete_op_request_with_url_only(self, mock_delete):
self.http_op = HttpOperation('delete', 'https://server.de/', 'pet/{petId}/uploadImage',
self.SAMPLE_OP_INFOS, False)
self.http_op.execute(ReplicatorTest.SAMPLE_DEFINITION)
self.assertIn(mock.call(url='https://server.de/pet/0/uploadImage'), mock_delete.call_args_list)
def test_kill_it_with_fire(self, victim):
victim._kill_it_with_fire("my-id")
calls = [mock.call("run1"), mock.call("run2")]
victim.runs_api.cancel_run.assert_has_calls(calls)
class TestLogger(test_case.DslTestCase):
FORMAT_CALLS = [
call(ANY, 'str', (), {}),
call(ANY, u'тест', (), {}),
call(ANY, 'str', (1,), {}),
call(ANY, 'str {0}', ('message',), {}),
call(ANY, 'str {message}', (), {'message': 'message'}),
call(ANY, 'str {message}{0}', (), {})]
LOG_CALLS = FORMAT_CALLS
def setUp(self):
super(TestLogger, self).setUp()
self._runner = self.new_runner(om.Object('TestLogger'))
self.package_loader.load_package('io.murano', None).register_class(
logger.Logger)
def test_create(self):
logger_instance = self._runner.testCreate()
self.assertTrue(
helpers.is_instance_of(logger_instance, 'io.murano.system.Logger'),
'Function should return io.murano.system.Logger instance')
def _create_logger_mock(self):
logger_instance = self._runner.testCreate()
logger_ext = logger_instance.extension
underlying_logger_mock = MagicMock()
logger_ext._underlying_logger = underlying_logger_mock
logger_ext._underlying_logger.return_value = None
format_mock = MagicMock(return_value='format_mock')
# do not verify number of conversions to string
format_mock.__str__ = MagicMock(return_value='format_mock')
format_mock.__unicode__ = MagicMock(return_value='format_mock')
logger_ext._format_without_exceptions = format_mock
return logger_instance, format_mock, underlying_logger_mock
def test_trace(self):
logger_instance, format_mock, underlying_logger_mock \
= self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.trace = log_method
self._runner.testTrace(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.LOG_CALLS))
def test_debug(self):
logger_instance, format_mock, underlying_logger_mock \
= self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.debug = log_method
self._runner.testDebug(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.LOG_CALLS))
def test_info(self):
logger_instance, format_mock, underlying_logger_mock \
= self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.info = log_method
self._runner.testInfo(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.LOG_CALLS))
def test_warning(self):
logger_instance, format_mock, underlying_logger_mock \
= self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.warning = log_method
self._runner.testWarning(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.LOG_CALLS))
def test_error(self):
logger_instance, format_mock, underlying_logger_mock \
= self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.error = log_method
self._runner.testError(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.FORMAT_CALLS))
def test_critical(self):
logger_instance, format_mock, underlying_logger_mock \
= self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.critical = log_method
self._runner.testCritical(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.LOG_CALLS))
def test_exception(self):
logger_instance, format_mock, underlying_logger_mock = \
self._create_logger_mock()
log_method = MagicMock()
underlying_logger_mock.error = log_method
self._runner.testException(logger_instance)
format_mock.assert_has_calls(self.FORMAT_CALLS, any_order=False)
self.assertEqual(log_method.call_count, len(self.LOG_CALLS))
def test_close(self):
self.tool.view
self.tool._tool_view.closeEvent()
self.content_ctrl_mock.unsubscribe.assert_has_calls(
[mock.call(1), mock.call(22)])
def test_heartbeatWithServer(self, dumpsMock, sleepMock, event_mock,
exit_mock):
out = StringIO.StringIO()
sys.stdout = out
hearbeat = MagicMock()
self.controller.heartbeat = hearbeat
event_mock.return_value = False
dumpsMock.return_value = "data"
sendRequest = MagicMock(name="sendRequest")
self.controller.sendRequest = sendRequest
self.controller.responseId = 1
response = {"responseId": "2", "restartAgent": "false"}
sendRequest.return_value = response
def one_heartbeat(*args, **kwargs):
self.controller.DEBUG_STOP_HEARTBEATING = True
return response
sendRequest.side_effect = one_heartbeat
actionQueue = MagicMock()
actionQueue.isIdle.return_value = True
# one successful request, after stop
self.controller.actionQueue = actionQueue
self.controller.alert_scheduler_handler = MagicMock()
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.called)
calls = []
def retry(*args, **kwargs):
if len(calls) == 0:
calls.append(1)
response["responseId"] = "3"
raise Exception()
if len(calls) > 0:
self.controller.DEBUG_STOP_HEARTBEATING = True
return response
# exception, retry, successful and stop
sendRequest.side_effect = retry
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertEqual(1, self.controller.DEBUG_SUCCESSFULL_HEARTBEATS)
# retry registration
self.controller.responseId = 2
response["registrationCommand"] = "true"
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.repeatRegistration)
# components are not mapped
self.controller.responseId = 2
response["registrationCommand"] = "false"
response["hasMappedComponents"] = False
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertFalse(self.controller.hasMappedComponents)
# components are mapped
self.controller.responseId = 2
response["hasMappedComponents"] = True
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.hasMappedComponents)
# components are mapped
self.controller.responseId = 2
del response["hasMappedComponents"]
sendRequest.side_effect = one_heartbeat
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
self.assertTrue(self.controller.hasMappedComponents)
# wrong responseId => restart
self.controller.responseId = 2
response = {"responseId": "2", "restartAgent": "false"}
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
restartAgent.assert_called_with()
# executionCommands
self.controller.responseId = 1
addToQueue = MagicMock(name="addToQueue")
self.controller.addToQueue = addToQueue
response["executionCommands"] = "executionCommands"
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToQueue.assert_has_calls([call("executionCommands")])
# statusCommands
self.controller.responseId = 1
addToStatusQueue = MagicMock(name="addToStatusQueue")
self.controller.addToStatusQueue = addToStatusQueue
response["statusCommands"] = "statusCommands"
self.controller.DEBUG_STOP_HEARTBEATING = False
self.controller.heartbeatWithServer()
addToStatusQueue.assert_has_calls([call("statusCommands")])
# restartAgent command
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
response["restartAgent"] = "true"
restartAgent = MagicMock(name="restartAgent")
self.controller.restartAgent = restartAgent
self.controller.heartbeatWithServer()
restartAgent.assert_called_with()
# actionQueue not idle
self.controller.responseId = 1
self.controller.DEBUG_STOP_HEARTBEATING = False
actionQueue.isIdle.return_value = False
response["restartAgent"] = "false"
self.controller.heartbeatWithServer()
# Check that server continues to heartbeat after connection errors
self.controller.responseId = 1
self.controller.TEST_IOERROR_COUNTER = 1
sendRequest.reset()
def util_throw_IOErrors(*args, **kwargs):
"""
Throws IOErrors 100 times and then stops heartbeats/registrations
"""
if self.controller.TEST_IOERROR_COUNTER == 10:
self.controller.DEBUG_STOP_HEARTBEATING = True
self.controller.TEST_IOERROR_COUNTER += 1
raise IOError("Sample error")
self.controller.DEBUG_STOP_HEARTBEATING = False
actionQueue.isIdle.return_value = False
sendRequest.side_effect = util_throw_IOErrors
self.controller.heartbeatWithServer()
self.assertTrue(sendRequest.call_count > 5)
sys.stdout = sys.__stdout__
self.controller.sendRequest = Controller.Controller.sendRequest
self.controller.sendRequest = Controller.Controller.addToQueue
self.controller.sendRequest = Controller.Controller.addToStatusQueue
def test_install_addon_service_mpack(self, set_file_permissions_mock, download_mpack_mock, expand_mpack_mock, purge_stacks_and_mpacks_mock,
add_replay_log_mock, get_ambari_properties_mock, get_ambari_version_mock,
create_symlink_mock, os_mkdir_mock, shutil_move_mock,os_symlink_mock,
os_path_isdir_mock, os_path_exists_mock ):
options = self._create_empty_options_mock()
options.mpack_path = "/path/to/myservice.tar.gz"
options.purge = False
download_mpack_mock.return_value = "/tmp/myservice.tar.gz"
expand_mpack_mock.return_value = "mpacks/myservice-ambari-mpack-1.0.0.0"
get_ambari_version_mock.return_value = "2.4.0.0"
os_path_exists_mock.side_effect = [True, True, True, True, True, True, True,
True, True, False, False, True, False, False,
True, True, True, True, False, True, True,
True, False]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
os_path_isdir_mock.return_value = True
install_mpack(options)
stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(mpacks_directory, "myservice-ambari-mpack-1.0.0.0")
resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
dashboards_directory = os.path.join(resources_directory, "dashboards")
os_path_exists_calls = [call('/tmp/myservice.tar.gz'),
call('mpacks/myservice-ambari-mpack-1.0.0.0/mpack.json'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/common-services/MYSERVICE'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/common-services/MYSERVICE/1.0.0/dashboards'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/custom-services/MYSERVICE/1.0.0/dashboards'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0/services'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/custom-services/MYSERVICE/2.0.0/dashboards')]
os_mkdir_calls = [
call(dashboards_directory),
call(os.path.join(dashboards_directory, GRAFANA_DASHBOARDS_DIRNAME)),
call(os.path.join(dashboards_directory, SERVICE_METRICS_DIRNAME)),
call(os.path.join(common_services_directory, "MYSERVICE"))
]
create_symlink_calls = [
call(os.path.join(mpacks_staging_directory, "common-services/MYSERVICE"),
os.path.join(common_services_directory, "MYSERVICE"), "1.0.0", None)
]
os_symlink_calls = [
call(os.path.join(mpacks_staging_directory, "custom-services/MYSERVICE/1.0.0"),
os.path.join(stacks_directory, "MYSTACK/1.0/services/MYSERVICE")),
call(os.path.join(mpacks_staging_directory, "custom-services/MYSERVICE/2.0.0"),
os.path.join(stacks_directory, "MYSTACK/2.0/services/MYSERVICE"))
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
self.assertFalse(purge_stacks_and_mpacks_mock.called)
os_mkdir_mock.assert_has_calls(os_mkdir_calls)
create_symlink_mock.assert_has_calls(create_symlink_calls)
os_symlink_mock.assert_has_calls(os_symlink_calls)
self.assertTrue(add_replay_log_mock.called)
def test_execute_will_post__op_request_with_params_when_form_data_param_set(self, mock_post):
self.http_op.execute(ReplicatorTest.SAMPLE_DEFINITION)
self.assertIn(mock.call(data={'status': '', 'name': ''}, json=None,
url='https://server.de/pet/0/uploadImage'), mock_post.call_args_list)
def test_install_extension_mpack(
self, download_mpack_mock, expand_mpack_mock, add_replay_log_mock,
purge_stacks_and_mpacks_mock, get_ambari_properties_mock,
get_ambari_version_mock, create_symlink_mock, os_mkdir_mock,
shutil_move_mock, os_path_exists_mock):
options = self._create_empty_options_mock()
options.mpack_path = "/path/to/myextension.tar.gz"
options.purge = False
download_mpack_mock.return_value = "/tmp/myextension.tar.gz"
expand_mpack_mock.return_value = "mpacks/myextension-ambari-mpack-1.0.0.0"
get_ambari_version_mock.return_value = "2.4.0.0"
os_path_exists_mock.side_effect = [
True, True, True, False, True, False, False, False, False, True,
False, False, False
]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
install_mpack(options)
extensions_directory = configs[
serverConfiguration.EXTENSION_PATH_PROPERTY]
mpacks_directory = configs[
serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(
mpacks_directory, "myextension-ambari-mpack-1.0.0.0")
dashboards_directory = serverConfiguration.get_dashboard_location(
configs)
os_path_exists_calls = [
call('/tmp/myextension.tar.gz'),
call('mpacks/myextension-ambari-mpack-1.0.0.0/mpack.json'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/extensions/MYEXTENSION'),
call(
mpacks_directory +
'/myextension-ambari-mpack-1.0.0.0/extensions/MYEXTENSION/1.0/services'
),
call(
mpacks_directory +
'/myextension-ambari-mpack-1.0.0.0/extensions/MYEXTENSION/1.1/services'
)
]
os_mkdir_calls = [
call(extensions_directory),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call(dashboards_directory),
call(os.path.join(dashboards_directory,
GRAFANA_DASHBOARDS_DIRNAME)),
call(os.path.join(dashboards_directory, SERVICE_METRICS_DIRNAME)),
call(os.path.join(extensions_directory, "MYEXTENSION"))
]
create_symlink_calls = [
call(
os.path.join(mpacks_staging_directory,
"extensions/MYEXTENSION"),
os.path.join(extensions_directory, "MYEXTENSION"), "1.0",
None),
call(
os.path.join(mpacks_staging_directory,
"extensions/MYEXTENSION"),
os.path.join(extensions_directory, "MYEXTENSION"), "1.1", None)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
self.assertFalse(purge_stacks_and_mpacks_mock.called)
os_mkdir_mock.assert_has_calls(os_mkdir_calls)
create_symlink_mock.assert_has_calls(create_symlink_calls)
self.assertTrue(add_replay_log_mock.called)
def test_init_calls(run_command_get_output):
sched.InsightsSchedulerSystemd()
calls = (
call("systemctl show --property LoadState insights-client.timer"),
call(
"systemctl show --property LoadState insights-client-checkin.timer"
),
)
run_command_get_output.assert_has_calls(calls)
@mark.parametrize(("outputs", "calls"), (
(
("LoadState=loaded\n", "LoadState=loaded\n", "", "", "", ""),
(
call("systemctl enable --now insights-client.timer"),
call("systemctl enable --now insights-client-checkin.timer"),
call("systemctl is-enabled insights-client.timer"),
call("systemctl is-enabled insights-client-checkin.timer"),
),
),
(
("LoadState=loaded\n", "LoadState=not-found\n", "", ""),
(
call("systemctl enable --now insights-client.timer"),
call("systemctl is-enabled insights-client.timer"),
),
),
(
("LoadState=not-found\n", "LoadState=not-found\n", "", ""),
(),
def test_upgrade_stack_mpack(self, set_file_permissions_mock, run_os_command_mock, download_mpack_mock, expand_mpack_mock, purge_stacks_and_mpacks_mock,
_uninstall_mpack_mock, add_replay_log_mock, get_ambari_properties_mock,
get_ambari_version_mock, create_symlink_mock, os_mkdir_mock, shutil_move_mock,
os_path_exists_mock, create_symlink_using_path_mock):
options = self._create_empty_options_mock()
options.mpack_path = "/path/to/mystack-1.0.0.1.tar.gz"
download_mpack_mock.side_effect = ["/tmp/mystack-1.0.0.1.tar.gz", "/tmp/mystack-1.0.0.1.tar.gz"]
expand_mpack_mock.side_effect = ["mpacks/mystack-ambari-mpack-1.0.0.1", "mpacks/mystack-ambari-mpack-1.0.0.1"]
get_ambari_version_mock.return_value = "2.4.0.0"
run_os_command_mock.return_value = (0, "", "")
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
os_path_exists_mock.side_effect = [True, True, True, True, True, True, True, True, True, True,
True, True, True, True, True, True, True, False, False, True,
True, False, False, True, False, False, False, False, False, True,
True, True, True, False, True, True, False, True, True, False,
False, False, False, False, True, True, True, True, True, True,
True, False, True, False, True, True, True, True, True, True,
True]
get_ambari_properties_mock.return_value = configs
shutil_move_mock.return_value = True
upgrade_mpack(options)
stacks_directory = configs[serverConfiguration.STACK_LOCATION_KEY]
common_services_directory = configs[serverConfiguration.COMMON_SERVICES_PATH_PROPERTY]
mpacks_directory = configs[serverConfiguration.MPACKS_STAGING_PATH_PROPERTY]
mpacks_staging_directory = os.path.join(mpacks_directory, "mystack-ambari-mpack-1.0.0.1")
resources_directory = configs[serverConfiguration.RESOURCES_DIR_PROPERTY]
dashboards_directory = os.path.join(resources_directory, "dashboards")
os_path_exists_calls = [call('/tmp/mystack-1.0.0.1.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call('/var/lib/ambari-server/resources'),
call(mpacks_directory),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call('/tmp/mystack-1.0.0.1.tar.gz'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call('mpacks/mystack-ambari-mpack-1.0.0.1/hooks/before_upgrade.py'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks'),
call('/var/lib/ambari-server/resources/extensions'),
call('/var/lib/ambari-server/resources/common-services'),
call(mpacks_directory),
call(mpacks_directory + '/cache'),
call('/var/lib/ambari-server/resources/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/common-services/SERVICEA'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/1.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEA/2.0/dashboards'),
call('/var/lib/ambari-server/resources/common-services/SERVICEB'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEB/1.0.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEB/2.0.0/dashboards'),
call('/var/lib/ambari-server/resources/common-services/SERVICEC'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/1.0.0/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/common-services/SERVICEC/2.0.0/dashboards'),
call('/var/lib/ambari-server/resources'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.0/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/1.0/services/SERVICEA/dashboards'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.1'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/1.1/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/1.1/services/SERVICEA/dashboards'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/2.0/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/2.0/services/SERVICEA/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/2.0/services/SERVICEB/dashboards'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/3.0'),
call('/var/lib/ambari-server/resources/stacks/MYSTACK/3.0/services'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEA/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEB/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEB/dashboards/grafana-dashboards'),
call('/var/lib/ambari-server/resources/dashboards/grafana-dashboards/SERVICEB'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEB/dashboards/service-metrics/SERVICEB.txt'),
call('/var/lib/ambari-server/resources/dashboards/service-metrics/SERVICEB.txt'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEC/dashboards'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEC/dashboards/grafana-dashboards'),
call('/var/lib/ambari-server/resources/dashboards/grafana-dashboards/SERVICEC'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/stacks/MYSTACK/3.0/services/SERVICEC/dashboards/service-metrics/SERVICEC.txt'),
call('/var/lib/ambari-server/resources/dashboards/service-metrics/SERVICEC.txt'),
call('/var/lib/ambari-server/resources'),
call(mpacks_directory),
call(mpacks_directory + '/myextension-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/myservice-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.0/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/mpack.json'),
call(mpacks_directory + '/mystack-ambari-mpack-1.0.0.1/hooks/after_upgrade.py')]
run_os_command_calls = [
call([
"/usr/bin/ambari-python-wrap",
"mpacks/mystack-ambari-mpack-1.0.0.1/hooks/before_upgrade.py"
]),
call([
"/usr/bin/ambari-python-wrap",
mpacks_directory + "/mystack-ambari-mpack-1.0.0.1/hooks/after_upgrade.py"
])
]
os_mkdir_calls = [
call(dashboards_directory),
call(os.path.join(dashboards_directory, GRAFANA_DASHBOARDS_DIRNAME)),
call(os.path.join(dashboards_directory, SERVICE_METRICS_DIRNAME)),
call(os.path.join(common_services_directory, "SERVICEC")),
call(os.path.join(stacks_directory, "MYSTACK/3.0")),
call(os.path.join(stacks_directory, "MYSTACK/3.0/services"))
]
create_symlink_calls = [
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEA"),
os.path.join(common_services_directory, "SERVICEA"),
"1.0", True),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEA"),
os.path.join(common_services_directory, "SERVICEA"),
"2.0", True),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEB"),
os.path.join(common_services_directory, "SERVICEB"),
"1.0.0", True),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEB"),
os.path.join(common_services_directory, "SERVICEB"),
"2.0.0", True),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEC"),
os.path.join(common_services_directory, "SERVICEC"),
"1.0.0", True),
call(os.path.join(mpacks_staging_directory, "common-services/SERVICEC"),
os.path.join(common_services_directory, "SERVICEC"),
"2.0.0", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.0"),
os.path.join(stacks_directory, "MYSTACK/1.0"),
"metainfo.xml", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.0/services"),
os.path.join(stacks_directory, "MYSTACK/1.0/services"),
"SERVICEA", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.1"),
os.path.join(stacks_directory, "MYSTACK/1.1"),
"metainfo.xml", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/1.1/services"),
os.path.join(stacks_directory, "MYSTACK/1.1/services"),
"SERVICEA", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0"),
os.path.join(stacks_directory, "MYSTACK/2.0"),
"metainfo.xml", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0/services"),
os.path.join(stacks_directory, "MYSTACK/2.0/services"),
"SERVICEA", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/2.0/services"),
os.path.join(stacks_directory, "MYSTACK/2.0/services"),
"SERVICEB", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0"),
os.path.join(stacks_directory, "MYSTACK/3.0"),
"metainfo.xml", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0/services"),
os.path.join(stacks_directory, "MYSTACK/3.0/services"),
"SERVICEA", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0/services"),
os.path.join(stacks_directory, "MYSTACK/3.0/services"),
"SERVICEB", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0/services"),
os.path.join(stacks_directory, "MYSTACK/3.0/services"),
"SERVICEC", True),
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0/services/SERVICEC/dashboards/service-metrics"),
os.path.join(dashboards_directory, "service-metrics"),
"SERVICEC.txt", True)
]
create_symlink_using_path_calls = [
call(os.path.join(mpacks_staging_directory, "stacks/MYSTACK/3.0/services/SERVICEC/dashboards/grafana-dashboards"),
os.path.join(dashboards_directory, "grafana-dashboards/SERVICEC"), True)
]
os_path_exists_mock.assert_has_calls(os_path_exists_calls)
self.assertFalse(purge_stacks_and_mpacks_mock.called)
run_os_command_mock.assert_has_calls(run_os_command_calls)
os_mkdir_mock.assert_has_calls(os_mkdir_calls)
create_symlink_mock.assert_has_calls(create_symlink_calls)
self.assertEqual(18, create_symlink_mock.call_count)
create_symlink_using_path_mock.assert_has_calls(create_symlink_using_path_calls)
self.assertEqual(1, create_symlink_using_path_mock.call_count)
_uninstall_mpack_mock.assert_has_calls([call("mystack-ambari-mpack", "1.0.0.0")])
self.assertTrue(add_replay_log_mock.called)
def test_reset_master_key_persisted(
self, get_is_persisted_method, get_is_secure_method,
sensitive_data_encryption_metod, is_root_method,
get_ambari_properties_method, search_file_message,
get_YN_input_method, save_master_key_method,
update_properties_method, read_passwd_for_alias_method,
save_passwd_for_alias_method, read_ambari_user_method, exists_mock,
read_master_key_method):
# Testing call under root
is_root_method.return_value = True
search_file_message.return_value = "filepath"
read_ambari_user_method.return_value = None
p = Properties()
FAKE_PWD_STRING = '${alias=fakealias}'
p.process_pair(JDBC_PASSWORD_PROPERTY, FAKE_PWD_STRING)
p.process_pair(SSL_TRUSTSTORE_PASSWORD_PROPERTY, FAKE_PWD_STRING)
p.process_pair(JDBC_RCA_PASSWORD_FILE_PROPERTY, FAKE_PWD_STRING)
get_ambari_properties_method.return_value = p
master_key = "aaa"
get_is_persisted_method.return_value = (True, "filepath")
get_is_secure_method.return_value = True
get_YN_input_method.side_effect = [False, True, True]
read_master_key_method.return_value = master_key
read_passwd_for_alias_method.return_value = "fakepassword"
save_passwd_for_alias_method.return_value = 0
exists_mock.return_value = False
options = self._create_empty_options_mock()
setup_sensitive_data_encryption(options)
calls = [call(options, "decryption"), call(options, "encryption")]
sensitive_data_encryption_metod.assert_has_calls(calls)
self.assertTrue(save_master_key_method.called)
self.assertTrue(get_YN_input_method.called)
self.assertTrue(read_master_key_method.called)
self.assertTrue(update_properties_method.called)
self.assertTrue(read_passwd_for_alias_method.called)
self.assertTrue(2, read_passwd_for_alias_method.call_count)
self.assertTrue(2, save_passwd_for_alias_method.call_count)
result_expected = {
JDBC_PASSWORD_PROPERTY:
get_alias_string(JDBC_RCA_PASSWORD_ALIAS),
JDBC_RCA_PASSWORD_FILE_PROPERTY:
get_alias_string(JDBC_RCA_PASSWORD_ALIAS),
SSL_TRUSTSTORE_PASSWORD_PROPERTY:
get_alias_string(SSL_TRUSTSTORE_PASSWORD_ALIAS),
SECURITY_IS_ENCRYPTION_ENABLED:
'true',
SECURITY_SENSITIVE_DATA_ENCRYPTON_ENABLED:
'true'
}
sorted_x = sorted(result_expected.iteritems(),
key=operator.itemgetter(0))
sorted_y = sorted(update_properties_method.call_args[0][1].iteritems(),
key=operator.itemgetter(0))
self.assertEquals(sorted_x, sorted_y)
pass
