def test_install_packages(self, package_provider_mock):
no_such_entry_config = {}
empty_config = {'hostLevelParams': {'package_list': ''}}
dummy_config = {
'hostLevelParams': {
'package_list':
"[{\"type\":\"rpm\",\"name\":\"hbase\"},"
"{\"type\":\"rpm\",\"name\":\"yet-another-package\"}]"
}
}
# Testing config without any keys
with Environment(".") as env:
script = Script()
Script.config = no_such_entry_config
script.install_packages(env)
self.assertEquals(len(env.resource_list), 0)
# Testing empty package list
with Environment(".") as env:
script = Script()
Script.config = empty_config
script.install_packages(env)
self.assertEquals(len(env.resource_list), 0)
# Testing installing of a list of packages
with Environment(".") as env:
Script.config = dummy_config
script.install_packages("env")
resource_dump = pprint.pformat(env.resource_list)
self.assertEqual(resource_dump,
"[Package['hbase'], Package['yet-another-package']]")
def test_install_packages(self, package_provider_mock):
no_packages_config = {
'hostLevelParams': {
'repo_info':
"[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
'agent_stack_retry_count': '5',
'agent_stack_retry_on_unavailability': 'false'
}
}
empty_config = {
'hostLevelParams': {
'package_list': '',
'repo_info':
"[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
'agent_stack_retry_count': '5',
'agent_stack_retry_on_unavailability': 'false'
}
}
dummy_config = {
'hostLevelParams': {
'package_list':
"[{\"type\":\"rpm\",\"name\":\"hbase\", \"condition\": \"\"},"
"{\"type\":\"rpm\",\"name\":\"yet-another-package\", \"condition\": \"\"}]",
'repo_info':
"[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
'service_repo_info':
"[{\"mirrorsList\":\"abc\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]",
'agent_stack_retry_count':
'5',
'agent_stack_retry_on_unavailability':
'false'
}
}
# Testing config without any keys
with Environment(".", test_mode=True) as env:
script = Script()
Script.config = no_packages_config
script.install_packages(env)
resource_dump = pprint.pformat(env.resource_list)
self.assertEquals(resource_dump, "[]")
# Testing empty package list
with Environment(".", test_mode=True) as env:
script = Script()
Script.config = empty_config
script.install_packages(env)
resource_dump = pprint.pformat(env.resource_list)
self.assertEquals(resource_dump, "[]")
# Testing installing of a list of packages
with Environment(".", test_mode=True) as env:
script = Script()
Script.config = dummy_config
script.install_packages("env")
resource_dump = pprint.pformat(env.resource_list)
self.assertEqual(
resource_dump,
'[Package[\'hbase\'], Package[\'yet-another-package\']]')
def test_format_with_reload(self):
from resource_management.libraries.functions import format
from resource_management.libraries.functions.format import ConfigurationFormatter
from resource_management.core.environment import Environment
env = Environment()
env._instances.append(env)
# declare some environment variables
env_params = {}
env_params["envfoo"] = "env-foo1"
env_params["envbar"] = "env-bar1"
env.config.params = env_params
# declare some local variables
foo = "foo1"
bar = "bar1"
# make sure local variables and env variables work
message = "{foo} {bar} {envfoo} {envbar}"
formatted_message = format(message)
self.assertEquals("foo1 bar1 env-foo1 env-bar1", formatted_message)
# try the same thing with an instance; we pass in keyword args to be
# combined with the env params
formatter = ConfigurationFormatter()
formatted_message = formatter.format(message, foo="foo2", bar="bar2")
self.assertEquals("foo2 bar2 env-foo1 env-bar1", formatted_message)
# now supply keyword args to override env params
formatted_message = formatter.format(message, envfoo="foobar", envbar="foobarbaz", foo="foo3", bar="bar3")
self.assertEquals("foo3 bar3 foobar foobarbaz", formatted_message)
def test_attribute_environment_non_root(self, popen_mock, select_mock,
os_read_mock):
expected_user = '******'
subproc_mock = MagicMock()
subproc_mock.wait.return_value = MagicMock()
subproc_mock.stdout = MagicMock()
subproc_mock.returncode = 0
popen_mock.return_value = subproc_mock
select_mock.return_value = ([subproc_mock.stdout], None, None)
os_read_mock.return_value = None
with Environment("/") as env:
execute_resource = Execute('echo "1"',
user=expected_user,
environment={
'JAVA_HOME': '/test/java/home',
'PATH': "/bin"
})
expected_command = [
'/bin/bash', '--login', '--noprofile', '-c',
'ambari-sudo.sh su test_user -l -s /bin/bash -c ' +
quote_bash_args('export PATH=' +
quote_bash_args(os.environ['PATH'] + ':/bin') +
' JAVA_HOME=/test/java/home ; echo "1"')
]
self.assertEqual(popen_mock.call_args_list[0][0][0], expected_command)
def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock, execute_mock,
tempfile_mock, checked_call_mock):
"""
Checks that GPG key is extracted from output without \r sign
"""
tempfile_mock.return_value = MagicMock(spec=file)
tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
checked_call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n"
with Environment('/') as env:
with patch.object(repository, "__file__", new='/ambari/test/repo/dummy/path/file'):
Repository('HDP',
base_url='http://download.base_url.org/rpm/',
repo_file_name='HDP',
repo_template = DEBIAN_DEFAUTL_TEMPLATE,
components = ['a','b','c']
)
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']
self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content, 'deb http://download.base_url.org/rpm/ a b c')
copy_item = str(file_mock.call_args_list[1])
self.assertEqual(copy_item, "call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))")
execute_command_item = execute_mock.call_args_list[0][0][0]
self.assertEqual(checked_call_mock.call_args_list[0][0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'])
self.assertEqual(execute_command_item, 'apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 123ABCD')
def test_unlink_configs_missing_backup(self, islink_mock, isdir_mock):
# required for the test to run since the Execute calls need this
from resource_management.core.environment import Environment
env = Environment(test_mode=True)
with env:
# Case: missing backup directory
isdir_mock.return_value = False
ru_execute = UpgradeSetAll()
self.assertEqual(len(env.resource_list), 0)
# Case: missing symlink
isdir_mock.reset_mock()
isdir_mock.return_value = True
islink_mock.return_value = False
ru_execute._unlink_config("/fake/config")
self.assertEqual(len(env.resource_list), 2)
# Case: missing symlink
isdir_mock.reset_mock()
isdir_mock.return_value = True
islink_mock.reset_mock()
islink_mock.return_value = True
ru_execute._unlink_config("/fake/config")
self.assertEqual(pprint.pformat(env.resource_list),
"[Directory['/fake/config'],\n "
"Execute[('mv', '/fake/conf.backup', '/fake/config')],\n "
"Execute[('rm', '/fake/config')],\n "
"Execute[('mv', '/fake/conf.backup', '/fake/config')]]")
def test_gpsshify(self):
with Environment('/'):
self.assertEqual(
utilities.gpsshify('command', host='host1'),
dedent("""
cat <<EOF | gpssh -h "host1"
command
EOF"""))
self.assertEqual(
utilities.gpsshify('command', hostfile='hosts'),
dedent("""
cat <<EOF | gpssh -f "hosts"
command
EOF"""))
self.assertEqual(
utilities.gpsshify('command\'"', host='host1'),
dedent("""
cat <<EOF | gpssh -h "host1"
command'"
EOF"""))
self.assertEqual(
utilities.gpsshify('command', host='host1', args='test'),
dedent("""
cat <<EOF | gpssh -h "host1" test
command
EOF"""))
with self.assertRaises(ValueError):
utilities.gpsshify('command', args='test'),
def test_attribute_command_escaping(self, popen_mock):
expected_command0 = "arg1 arg2 'quoted arg'"
expected_command1 = "arg1 arg2 'command \"arg\"'"
expected_command2 = 'arg1 arg2 \'command \'"\'"\'arg\'"\'"\'\''
expected_command3 = "arg1 arg2 'echo `ls /root`'"
expected_command4 = "arg1 arg2 '$ROOT'"
expected_command5 = "arg1 arg2 '`ls /root`'"
subproc_mock = MagicMock()
subproc_mock.returncode = 0
subproc_mock.stdout.readline = MagicMock(side_effect=['OK'])
popen_mock.return_value = subproc_mock
with Environment("/") as env:
Execute(('arg1', 'arg2', 'quoted arg'), )
Execute(('arg1', 'arg2', 'command "arg"'), )
Execute(('arg1', 'arg2', "command 'arg'"), )
Execute(('arg1', 'arg2', "echo `ls /root`"), )
Execute(('arg1', 'arg2', "$ROOT"), )
Execute(('arg1', 'arg2', "`ls /root`"), )
self.assertEqual(popen_mock.call_args_list[0][0][0][4],
expected_command0)
self.assertEqual(popen_mock.call_args_list[1][0][0][4],
expected_command1)
self.assertEqual(popen_mock.call_args_list[2][0][0][4],
expected_command2)
self.assertEqual(popen_mock.call_args_list[3][0][0][4],
expected_command3)
self.assertEqual(popen_mock.call_args_list[4][0][0][4],
expected_command4)
self.assertEqual(popen_mock.call_args_list[5][0][0][4],
expected_command5)
def test_remove_repo_ubuntu_repo_doenst_exist(self, file_mock,
execute_mock):
with Environment('/') as env:
Repository('HDP', action="remove", repo_file_name='HDP')
self.assertEqual(file_mock.call_count, 0)
self.assertEqual(execute_mock.call_count, 0)
def test_remove_repo_suse(self, file_mock, is_redhat_family,
is_ubuntu_family, is_suse_family):
is_redhat_family.return_value = False
is_ubuntu_family.return_value = False
is_suse_family.return_value = True
with Environment('/') as env:
Repository(
'hadoop',
action='remove',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
expected_arguments = {
'action': ['remove'],
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[1]['action'], 'delete')
self.assertEqual(file_mock.call_args[0][0],
'/etc/zypp/repos.d/Repository.repo')
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
logger, chout, cherr = Logger.initialize_logger(__name__)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[self.logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(
k, Script.config) and get_path_from_configuration(
v, Script.config):
Script.passwords[get_path_from_configuration(
k, Script.config)] = get_path_from_configuration(
v, Script.config)
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
method(env)
finally:
if self.should_expose_component_version(command_name):
self.save_component_version_to_structured_out()
def test_create_repo_ubuntu_doesnt_repo_exist(self, file_mock,
execute_mock, tempfile_mock):
tempfile_mock.return_value = MagicMock(spec=file)
tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
with Environment('/') as env:
with patch.object(repository,
"__file__",
new='/ambari/test/repo/dummy/path/file'):
Repository('HDP',
base_url='http://download.base_url.org/rpm/',
repo_file_name='HDP',
repo_template=DEBIAN_DEFAUTL_TEMPLATE,
components=['a', 'b', 'c'])
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']
self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content,
'deb http://download.base_url.org/rpm/ a b c')
self.assertEqual(file_mock.call_count, 2)
self.assertEqual(execute_mock.call_count, 0)
def expand_mpack(archive_path):
"""
Expand management pack
:param archive_path: Local path to management pack
:return: Path where the management pack was expanded
"""
tmpdir = _get_temp_dir()
archive_root_dir = get_archive_root_dir(archive_path)
if not archive_root_dir:
print_error_msg("Malformed management pack. Root directory missing!")
raise FatalException(-1, 'Malformed management pack. Root directory missing!')
# Expand management pack in temp directory
tmp_root_dir = os.path.join(tmpdir, archive_root_dir)
print_info_msg("Expand management pack at temp location {0}".format(tmp_root_dir))
if os.path.exists(tmp_root_dir):
sudo.rmtree(tmp_root_dir)
with Environment():
untar_archive(archive_path, tmpdir)
if not os.path.exists(tmp_root_dir):
print_error_msg("Malformed management pack. Failed to expand management pack!")
raise FatalException(-1, 'Malformed management pack. Failed to expand management pack!')
return tmp_root_dir
def _collect(self):
cmd_module = self._load_source()
if cmd_module is not None:
configurations = {}
try:
tokens = cmd_module.get_tokens()
if tokens is not None:
# for each token, if there is a value, store in; otherwise don't store
# a key with a value of None
for token in tokens:
value = self._get_configuration_value(token)
if value is not None:
configurations[token] = value
except AttributeError:
# it's OK if the module doesn't have get_tokens() ; no tokens will
# be passed in so hopefully the script doesn't need any
logger.debug("The script {0} does not have a get_tokens() function".format(str(cmd_module)))
# try to get basedir for scripts
# it's needed for server side scripts to properly use resource management
matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
with Environment(basedir, tmp_dir=self.config.get('agent', 'tmp_dir')) as env:
return cmd_module.execute(configurations, self.parameters, self.host_name)
else:
return cmd_module.execute(configurations, self.parameters, self.host_name)
else:
return (self.RESULT_UNKNOWN, ["Unable to execute script {0}".format(self.path)])
def test_attribute_wait_and_poll_and_success(self,
proc_communicate_mock):
with Environment("/") as env:
Execute('sleep 6', wait_for_finish=False, poll_after=2)
self.assertFalse(proc_communicate_mock.called,
'proc.communicate should not have been called!')
def _collect(self):
cmd_module = self._load_source()
if cmd_module is not None:
# convert the dictionary from
# {'foo-site/bar': 'baz'} into
# {'{{foo-site/bar}}': 'baz'}
parameters = {}
for key in self.config_value_dict:
parameters['{{' + key + '}}'] = self.config_value_dict[key]
# try to get basedir for scripts
# it's needed for server side scripts to properly use resource management
matchObj = re.match(r'((.*)services\/(.*)\/package\/)',
self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
with Environment(basedir,
tmp_dir=self.config.get('agent',
'tmp_dir')) as env:
return cmd_module.execute(parameters, self.host_name)
else:
return cmd_module.execute(parameters, self.host_name)
else:
return (self.RESULT_UNKNOWN,
["Unable to execute script {0}".format(self.path)])
def setUp(self):
Logger.initialize_logger()
# required for the test to run since the Execute calls need this
from resource_management.core.environment import Environment
self.env = Environment(test_mode=True)
self.env.__enter__()
def test_status_commands_clear_structured_out(self, open_mock):
"""
Tests that status commands will clear and stored structured output from prior status commands.
:param open_mock:
:return:
"""
class MagicFile(object):
def read(self):
return "{}"
def write(self, data):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
pass
def __enter__(self):
return self
sys.argv = ["", "status", "foo.py", "", "", "INFO", ""]
open_mock.side_effect = [MagicFile()]
try:
with Environment(".", test_mode=True) as env:
script = Script()
Script.structuredOut = {"version": "old_version"}
script.execute()
except:
pass
self.assertTrue(open_mock.called)
self.assertEquals({}, Script.structuredOut)
def test_create_repo_redhat(self, file_mock,
is_redhat_family, is_ubuntu_family, is_suse_family):
is_redhat_family.return_value = True
is_ubuntu_family.return_value = False
is_suse_family.return_value = False
with Environment('/') as env:
with patch.object(repository, "__file__", new='/ambari/test/repo/dummy/path/file'):
Repository('hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository',
repo_template=RHEL_SUSE_DEFAULT_TEMPLATE)
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
expected_arguments = {'repo_template': RHEL_SUSE_DEFAULT_TEMPLATE,
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'}
expected_template_arguments = {'base_url': 'http://download.base_url.org/rpm/',
'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[0][0], '/etc/yum.repos.d/Repository.repo')
template_item = file_mock.call_args[1]['content']
template = str(template_item.name)
expected_template_arguments.update({'repo_id': 'hadoop'})
self.assertEqual(expected_template_arguments, template_item.context._dict)
self.assertEqual(RHEL_SUSE_DEFAULT_TEMPLATE, template)
def test_action_stop_stopped(self, shell_mock):
return_values = [(1, None)]
shell_mock.side_effect = return_values
with Environment('/') as env:
Service('some_service', action="stop")
self.assertEqual(shell_mock.call_count, 1)
self.assertEqual(shell_mock.call_args_list[0][0][0],
['/etc/init.d/some_service', 'status'])
def test_append_to_file(self):
mocked_file = self.__mock_file()
with self.__with_mocked_file(mocked_file), Environment('/'):
utilities.append_to_file('/test', 'some_command')
mocked_file.assert_called_once_with('/test', 'a+')
handle = mocked_file()
handle.write.assert_called_once_with("some_command\n")
def _collect(self):
cmd_module = self._load_source()
full_configurations = self.configuration_builder.get_configuration(
self.cluster_id, None, None)
if cmd_module is not None:
configurations = {}
try:
tokens = cmd_module.get_tokens()
if tokens is not None:
# for each token, if there is a value, store in; otherwise don't store
# a key with a value of None
for token in tokens:
value = self._get_configuration_value(
full_configurations, token)
if value is not None:
configurations[token] = value
except AttributeError:
# it's OK if the module doesn't have get_tokens() ; no tokens will
# be passed in so hopefully the script doesn't need any
logger.debug(
"The script {0} does not have a get_tokens() function".
format(str(cmd_module)))
Script.config = full_configurations
# try to get basedir for scripts
# it's needed for server side scripts to properly use resource management
matchObj = re.match(r'((.*)services(.*)package)',
self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
with Environment(
basedir,
tmp_dir=AGENT_TMP_DIR,
logger=logging.getLogger('ambari_alerts')) as env:
result = cmd_module.execute(configurations,
self.parameters,
self.host_name)
else:
result = cmd_module.execute(configurations, self.parameters,
self.host_name)
loggerMsg = "[Alert][{0}] Failed with result {2}: {3}".format(
self.get_name(), self.path_to_script, result[0], result[1])
if result[0] == self.RESULT_CRITICAL:
logger.error(loggerMsg)
elif result[0] == self.RESULT_WARNING or result[
0] == self.RESULT_UNKNOWN:
logger.debug(loggerMsg)
return result
else:
return (self.RESULT_UNKNOWN,
["Unable to execute script {0}".format(self.path)])
def test_remove_repo_ubuntu_repo_exist(self, file_mock, execute_mock):
with Environment('/') as env:
Repository('HDP',
action = "remove",
repo_file_name='HDP'
)
self.assertEqual(str(file_mock.call_args), "call('/etc/apt/sources.list.d/HDP.list', action='delete')")
self.assertEqual(execute_mock.call_args[0][0], ['apt-get', 'update', '-qq', '-o', 'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o', 'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'])
def test_action_not_existing(self):
try:
with Environment('/') as env:
Service('some_service', action="not_existing_action")
self.fail("Service should fail with nonexistent action")
except Fail as e:
self.assertEqual(
"ServiceProvider[Service['some_service']] does not implement action not_existing_action",
str(e))
def test_attribute_wait(self, popen_mock, proc_communicate_mock):
with Environment("/") as env:
Execute('echo "1"',
wait_for_finish=False)
Execute('echo "2"',
wait_for_finish=False)
self.assertTrue(popen_mock.called, 'subprocess.Popen should have been called!')
self.assertFalse(proc_communicate_mock.called, 'proc.communicate should not have been called!')
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
# set up logging (two separate loggers for stderr and stdout with different loglevels)
logger = logging.getLogger('resource_management')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
chout = logging.StreamHandler(sys.stdout)
chout.setLevel(logging.INFO)
chout.setFormatter(formatter)
cherr = logging.StreamHandler(sys.stderr)
cherr.setLevel(logging.ERROR)
cherr.setFormatter(formatter)
logger.addHandler(cherr)
logger.addHandler(chout)
# parse arguments
if len(sys.argv) < 7:
logger.error("Script expects at least 6 arguments")
print USAGE.format(os.path.basename(
sys.argv[0])) # print to stdout
sys.exit(1)
command_name = str.lower(sys.argv[1])
command_data_file = sys.argv[2]
basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
logging_level_str = logging._levelNames[logging_level]
chout.setLevel(logging_level_str)
logger.setLevel(logging_level_str)
try:
with open(command_data_file, "r") as f:
pass
Script.config = ConfigDictionary(json.load(f))
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(basedir) as env:
method(env)
except ClientComponentHasNoStatus or ComponentIsNotRunning:
# Support of component status checks.
# Non-zero exit code is interpreted as an INSTALLED status of a component
sys.exit(1)
except Fail:
logger.exception(
"Error while executing command '{0}':".format(command_name))
sys.exit(1)
def executeScript(self, path, classname=None, command=None, config_file=None,
# common mocks for all the scripts
config_overrides = None,
shell_mock_value = (0, "OK."),
os_type=('Suse','11','Final'),
kinit_path_local="/usr/bin/kinit"
):
norm_path = os.path.normpath(path)
src_dir = RMFTestCase._getSrcFolder()
stack_version = norm_path.split(os.sep)[0]
stacks_path = os.path.join(src_dir, PATH_TO_STACKS)
configs_path = os.path.join(src_dir, PATH_TO_STACK_TESTS, stack_version, "configs")
script_path = os.path.join(stacks_path, norm_path)
config_file_path = os.path.join(configs_path, config_file)
try:
with open(config_file_path, "r") as f:
self.config_dict = json.load(f)
except IOError:
raise RuntimeError("Can not read config file: "+ config_file_path)
if config_overrides:
for key, value in config_overrides.iteritems():
self.config_dict[key] = value
self.config_dict = ConfigDictionary(self.config_dict)
# append basedir to PYTHONPATH
scriptsdir = os.path.dirname(script_path)
basedir = os.path.dirname(scriptsdir)
sys.path.append(scriptsdir)
# get method to execute
try:
with patch.object(platform, 'linux_distribution', return_value=os_type):
script_module = imp.load_source(classname, script_path)
except IOError:
raise RuntimeError("Cannot load class %s from %s",classname, norm_path)
script_class_inst = RMFTestCase._get_attr(script_module, classname)()
method = RMFTestCase._get_attr(script_class_inst, command)
# Reload params import, otherwise it won't change properties during next import
if 'params' in sys.modules:
del(sys.modules["params"])
# run
with Environment(basedir, test_mode=True) as RMFTestCase.env:
with patch('resource_management.core.shell.checked_call', return_value=shell_mock_value): # we must always mock any shell calls
with patch.object(Script, 'get_config', return_value=self.config_dict): # mocking configurations
with patch.object(Script, 'get_tmp_dir', return_value="/tmp"):
with patch.object(Script, 'install_packages'):
with patch('resource_management.libraries.functions.get_kinit_path', return_value=kinit_path_local):
with patch.object(platform, 'linux_distribution', return_value=os_type):
method(RMFTestCase.env)
sys.path.remove(scriptsdir)
def test_archive_directory_dereference(self, execute_mock):
archive = '/home/etc.tar.gz'
directory = '/etc'
with Environment():
tar_archive.archive_directory_dereference(archive, directory)
self.assertEqual(execute_mock.call_count, 1)
self.assertEqual(execute_mock.call_args[0][0].command,
('tar', '-zchf', archive, '-C', directory, '.'))
def test_set_version(self, get_stack_to_component_mock):
good_config = {
'hostLevelParams': {
'stack_name': "HDP",
'stack_version': "2.2"
},
'commandParams': {
'version': "2.2.0.0-2041"
}
}
get_stack_to_component_mock.return_value = {"HDP": "kafka-broker"}
# Testing default workflow
with Environment(".", test_mode=True) as env:
script = Script()
Script.config = good_config
script.set_version()
resource_dump = pprint.pformat(env.resource_list)
self.assertEquals(
resource_dump,
'[Execute[\'(\'/usr/bin/hdp-select\', \'set\', \'kafka-broker\', \'2.2.0.0-2041\')\']]'
)
# Component does not provide mapping
get_stack_to_component_mock.return_value = {}
with Environment(".", test_mode=True) as env:
script = Script()
Script.config = good_config
script.set_version()
resource_dump = pprint.pformat(env.resource_list)
self.assertEquals(resource_dump, '[]')
# Component provided mapping, but configuration is not complete (testing fallback)
get_stack_to_component_mock.return_value = {"HDP": "kafka-broker"}
bad_config = {}
with Environment(".", test_mode=True) as env:
script = Script()
Script.config = bad_config
script.set_version()
resource_dump = pprint.pformat(env.resource_list)
self.assertEquals(resource_dump, '[]')
def test_action_restart(self, shell_mock):
return_values = [(0, None), (0, None)]
shell_mock.side_effect = return_values
with Environment('/') as env:
Service('some_service', action="restart")
self.assertEqual(shell_mock.call_count, 2)
self.assertEqual(shell_mock.call_args_list[0][0][0],
['/etc/init.d/some_service', 'status'])
self.assertEqual(shell_mock.call_args_list[1][0][0],
['/etc/init.d/some_service', 'restart'])
