def format(self, format_string, *args, **kwargs):
variables = kwargs
if Environment.has_instance():
env = Environment.get_instance()
params = env.config.params
# don't use checked_unite for this as it would interfere with reload(module)
# for things like params and status_params; instead, start out copying
# the environment parameters and add in any locally declared variables to
# override existing env parameters
all_params = params.copy()
else:
all_params = {}
all_params.update(variables)
self.convert_field = self.convert_field_protected
result_protected = self.vformat(format_string, args, all_params)
self.convert_field = self.convert_field_unprotected
result_unprotected = self.vformat(format_string, args, all_params)
if result_protected != result_unprotected:
Logger.sensitive_strings[result_unprotected] = result_protected
return result_unprotected
def action_remove(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = get_repo_dir()
File(format("{repo_dir}/{repo_file_name}.repo"),
action="delete")
def action_create(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = get_repo_dir()
new_content = InlineTemplate(self.resource.repo_template, repo_id=self.resource.repo_id, repo_file_name=self.resource.repo_file_name,
base_url=self.resource.base_url, mirror_list=self.resource.mirror_list)
repo_file_path = format("{repo_dir}/{repo_file_name}.repo")
if os.path.isfile(repo_file_path):
existing_content_str = sudo.read_file(repo_file_path)
new_content_str = new_content.get_content()
if existing_content_str != new_content_str and OSCheck.is_suse_family():
# We need to reset package manager's cache when we replace base urls
# at existing repo. That is a case at least under SLES
Logger.info("Flushing package manager cache since repo file content is about to change")
checked_call(self.update_cmd, sudo=True)
if self.resource.append_to_file:
content = existing_content_str + '\n' + new_content_str
else:
content = new_content_str
else: # If repo file does not exist yet
content = new_content
File(repo_file_path,
content=content
)
def action_create(self):
with Environment.get_instance_copy() as env:
with tempfile.NamedTemporaryFile() as tmpf:
repo_file_name = format("{repo_file_name}.list", repo_file_name=self.resource.repo_file_name)
repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir=self.repo_dir)
new_content = Template(
self.resource.repo_template,
package_type=self.package_type,
base_url=self.resource.base_url,
components=" ".join(self.resource.components),
).get_content()
old_content = ""
if self.resource.append_to_file and os.path.isfile(repo_file_path):
with open(repo_file_path) as repo_file:
old_content = repo_file.read() + "\n"
File(tmpf.name, content=old_content + new_content)
if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, repo_file_path):
File(repo_file_path, content=StaticFile(tmpf.name))
update_cmd_formatted = [format(x) for x in self.update_cmd]
# this is time expensive
retcode, out = checked_call(update_cmd_formatted, sudo=True)
# add public keys for new repos
missing_pkeys = set(re.findall(self.missing_pkey_regex, out))
for pkey in missing_pkeys:
Execute(
format(self.add_pkey_cmd),
timeout=15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
ignore_failures=True,
)
def action_create(self):
filename = self.resource.filename
xml_config_provider_config_dir = self.resource.conf_dir
# |e - for html-like escaping of <,>,',"
config_content = InlineTemplate(''' <configuration>
{% for key, value in configurations_dict|dictsort %}
<property>
<name>{{ key|e }}</name>
<value>{{ resource_management.core.source.InlineTemplate(str(value)).get_content() |e }}</value>
{%- if not configuration_attrs is none -%}
{%- for attrib_name, attrib_occurances in configuration_attrs.items() -%}
{%- for property_name, attrib_value in attrib_occurances.items() -%}
{% if property_name == key and attrib_name %}
<{{attrib_name|e}}>{{attrib_value|e}}</{{attrib_name|e}}>
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
{%- endif %}
</property>
{% endfor %}
</configuration>''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], configurations_dict=self.resource.configurations,
configuration_attrs=self.resource.configuration_attributes)
xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename)
Logger.info("Generating config: {0}".format(xml_config_dest_file_path))
with Environment.get_instance_copy() as env:
File (xml_config_dest_file_path,
content = config_content,
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode,
encoding = self.resource.encoding
)
def falcon(type, action = None):
import params
if action == 'config':
env = Environment.get_instance()
# These 2 parameters are used in ../templates/client.properties.j2
env.config.params["falcon_host"] = params.falcon_host
env.config.params["falcon_port"] = params.falcon_port
File(os.path.join(params.falcon_conf_dir, 'falcon-env.sh'),
content = InlineTemplate(params.falcon_env_sh_template))
File(os.path.join(params.falcon_conf_dir, 'client.properties'),
content = Template('client.properties.j2'))
PropertiesFile(os.path.join(params.falcon_conf_dir, 'runtime.properties'),
properties = params.falcon_runtime_properties)
PropertiesFile(os.path.join(params.falcon_conf_dir, 'startup.properties'),
properties = params.falcon_startup_properties)
if type == 'server':
ServiceConfig(params.falcon_win_service_name,
action = "change_user",
username = params.falcon_user,
password = Script.get_password(params.falcon_user))
if action == 'start':
Service(params.falcon_win_service_name, action = "start")
if action == 'stop':
Service(params.falcon_win_service_name, action = "stop")
def get_check_command(oozie_url, host_name, configurations):
if OOZIE_USER in configurations:
oozie_user = configurations[OOZIE_USER]
else:
raise Exception("Oozie user is required")
security_enabled = False
if SECURITY_ENABLED in configurations:
security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'
kerberos_env = None
if security_enabled:
if OOZIE_KEYTAB in configurations and OOZIE_PRINCIPAL in configurations:
oozie_keytab = configurations[OOZIE_KEYTAB]
oozie_principal = configurations[OOZIE_PRINCIPAL]
# substitute _HOST in kerberos principal with actual fqdn
oozie_principal = oozie_principal.replace('_HOST', host_name)
else:
raise KerberosPropertiesNotFound('The Oozie keytab and principal are required configurations when security is enabled.')
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
# Get the configured Kerberos executable search paths, if any
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
else:
kerberos_executable_search_paths = None
klist_path_local = get_klist_path(kerberos_executable_search_paths)
klist_command = format("{klist_path_local} -s {ccache_file}")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
return_code, _ = call(klist_command, user=oozie_user)
if return_code != 0:
kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")
# kinit
Execute(kinit_command,
environment=kerberos_env,
user=oozie_user,
)
# oozie configuration directory uses a symlink when > HDP 2.2
oozie_config_directory = OOZIE_CONF_DIR_LEGACY
if os.path.exists(OOZIE_CONF_DIR):
oozie_config_directory = OOZIE_CONF_DIR
command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
oozie_config_directory, oozie_url)
return (command, kerberos_env, oozie_user)
def action_remove(self):
with Environment.get_instance_copy() as env:
repo_file_name = format("{repo_file_name}.list", repo_file_name=self.resource.repo_file_name)
repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir=self.repo_dir)
if os.path.isfile(repo_file_path):
File(repo_file_path, action="delete")
# this is time expensive
update_cmd_formatted = [format(x) for x in self.update_cmd]
Execute(update_cmd_formatted)
def test_attribute_creates(self, popen_mock, exists_mock):
exists_mock.return_value = True
subproc_mock = MagicMock()
subproc_mock.returncode = 0
subproc_mock.stdout.readline = MagicMock(side_effect = ['OK'])
subproc_mock.communicate.side_effect = [["1"]]
popen_mock.return_value = subproc_mock
with Environment("/") as env:
Execute('echo "1"',
creates="/must/be/created")
exists_mock.assert_called_with("/must/be/created")
self.assertEqual(subproc_mock.call_count, 0)
def test_attribute_group(self, getpwnam_mock, getgrnam_mock):
def error(argument):
self.assertEqual(argument, "test_group")
raise KeyError("fail")
getpwnam_mock.side_effect = 1
getgrnam_mock.side_effect = error
try:
with Environment("/") as env:
Execute(
'echo "1"',
group="test_group",
)
except Fail as e:
pass
def test_attribute_path(self, popen_mock, select_mock, os_read_mock):
subproc_mock = MagicMock()
subproc_mock.wait.return_value = MagicMock()
subproc_mock.stdout = MagicMock()
subproc_mock.returncode = 0
popen_mock.return_value = subproc_mock
select_mock.return_value = ([subproc_mock.stdout], None, None)
os_read_mock.return_value = None
with Environment("/") as env:
execute_resource = Execute('echo "1"',
path=["/test/one", "test/two"]
)
expected_command = ['/bin/bash', '--login', '--noprofile', '-c', 'echo "1"']
self.assertEqual(popen_mock.call_args_list[0][0][0], expected_command)
def test_attribute_command_one_line(self, popen_mock, select_mock, os_read_mock):
expected_command = "rm -rf /somedir"
subproc_mock = MagicMock()
subproc_mock.wait.return_value = MagicMock()
subproc_mock.stdout = MagicMock()
subproc_mock.returncode = 0
popen_mock.return_value = subproc_mock
select_mock.return_value = ([subproc_mock.stdout], None, None)
os_read_mock.return_value = None
with Environment("/") as env:
Execute(expected_command)
self.assertEqual(popen_mock.call_args_list[0][0][0][4], expected_command)
def test_create_repo_ubuntu_gpg_key_wrong_output(self, file_mock,
execute_mock,
tempfile_mock, call_mock):
"""
Checks that GPG key is extracted from output without \r sign
"""
tempfile_mock.return_value = MagicMock(spec=file)
tempfile_mock.return_value.__enter__.return_value.name = "/tmp/1.txt"
call_mock.return_value = 0, "The following signatures couldn't be verified because the public key is not available: NO_PUBKEY 123ABCD\r\n"
with Environment('/') as env:
with patch.object(repository,
"__file__",
new='/ambari/test/repo/dummy/path/file'):
Repository('HDP',
base_url='http://download.base_url.org/rpm/',
repo_file_name='HDP',
repo_template=DEBIAN_DEFAUTL_TEMPLATE,
components=['a', 'b', 'c'])
call_content = file_mock.call_args_list[0]
template_name = call_content[0][0]
template_content = call_content[1]['content']
self.assertEquals(template_name, '/tmp/1.txt')
self.assertEquals(template_content,
'deb http://download.base_url.org/rpm/ a b c')
copy_item0 = str(file_mock.call_args_list[1])
copy_item1 = str(file_mock.call_args_list[2])
self.assertEqual(
copy_item0,
"call('/tmp/1.txt', content=StaticFile('/etc/apt/sources.list.d/HDP.list'))"
)
self.assertEqual(
copy_item1,
"call('/etc/apt/sources.list.d/HDP.list', content=StaticFile('/tmp/1.txt'))"
)
execute_command_item = execute_mock.call_args_list[0][0][0]
self.assertEqual(call_mock.call_args_list[0][0][0], [
'apt-get', 'update', '-qq', '-o',
'Dir::Etc::sourcelist=sources.list.d/HDP.list', '-o',
'Dir::Etc::sourceparts=-', '-o', 'APT::Get::List-Cleanup=0'
])
self.assertEqual(execute_command_item,
('apt-key', 'adv', '--recv-keys', '--keyserver',
'keyserver.ubuntu.com', '123ABCD'))
def test_recreate_repo_suse(self, filecmp_mock, read_file_mock,
checked_call_mock, file_mock, is_redhat_family,
is_ubuntu_family, is_suse_family):
filecmp_mock.return_value = False
is_redhat_family.return_value = False
is_ubuntu_family.return_value = False
is_suse_family.return_value = True
read_file_mock.return_value = "Dummy repo file contents"
checked_call_mock.return_value = 0, "Flushing zypper cache"
with Environment('/') as env:
with patch.object(repository,
"__file__",
new='/ambari/test/repo/dummy/path/file'):
# Check that zypper cache is flushed
Repository(
'hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_template=RHEL_SUSE_DEFAULT_TEMPLATE,
repo_file_name='Repository')
Repository(None, action="create")
self.assertTrue(checked_call_mock.called)
expected_repo_file_content = "[hadoop]\nname=hadoop\nmirrorlist=https://mirrors.base_url.org/?repo=Repository&arch=$basearch\n\npath=/\nenabled=1\ngpgcheck=0"
template = file_mock.call_args_list[0][1]['content']
self.assertEqual(expected_repo_file_content, template)
# Check that if content is equal, zypper cache is not flushed
checked_call_mock.reset_mock()
filecmp_mock.return_value = True
Repository(
'hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_template=RHEL_SUSE_DEFAULT_TEMPLATE,
repo_file_name='Repository')
Repository(None, action="create")
self.assertFalse(checked_call_mock.called)
expected_repo_file_content = "[hadoop]\nname=hadoop\nmirrorlist=https://mirrors.base_url.org/?repo=Repository&arch=$basearch\n\npath=/\nenabled=1\ngpgcheck=0"
template = file_mock.call_args_list[0][1]['content']
self.assertEqual(expected_repo_file_content, template)
def action_create(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = get_repo_dir()
repo_template = self.resource.repo_template
new_content = Template(repo_template,
repo_id=self.resource.repo_id,
repo_file_name=self.resource.repo_file_name,
base_url=self.resource.base_url,
mirror_list=self.resource.mirror_list)
repo_file_path = format("{repo_dir}/{repo_file_name}.repo")
if self.resource.append_to_file and os.path.isfile(repo_file_path):
with open(repo_file_path, 'a') as repo_file:
repo_file.write('\n' + new_content.get_content())
else:
File(repo_file_path, content=new_content)
def format(self, format_string, *args, **kwargs):
env = Environment.get_instance()
variables = kwargs
params = env.config.params
all_params = checked_unite(variables, params)
self.convert_field = self.convert_field_protected
result_protected = self.vformat(format_string, args, all_params)
self.convert_field = self.convert_field_unprotected
result_unprotected = self.vformat(format_string, args, all_params)
if result_protected != result_unprotected:
Logger.sensitive_strings[result_unprotected] = result_protected
return result_unprotected
def __get_delegation_token(self, user, keytab, principal, kinit_path):
"""
Gets the kerberos delegation token from name node
"""
import params
url = params.namenode_path + "/webhdfs/v1/?op=GETDELEGATIONTOKEN"
Logger.info("Getting delegation token from {0}".format(url))
response, _, _ = curl_krb_request(Environment.get_instance().tmp_dir, keytab, principal,
url, "get_delegation_token", kinit_path, False, "Delegation Token", user)
json_response = json.loads(response)
if json_response['Token'] and json_response['Token']['urlString']:
return json_response['Token']['urlString']
error_msg = "Get Token: Unable to get kerberos delegation token from webhdfs: \nurl = {0}, user = {1}, keytab = {2}, principal = {3}, kinit-path = {4} \nresponse = {5}".format(url, user, keytab, principal, kinit_path, json_response)
Logger.error(error_msg)
self.checks_failed += 1
def format(self, format_string, *args, **kwargs):
env = Environment.get_instance()
variables = kwargs
params = env.config.params
all_params = checked_unite(variables, params)
self.convert_field = self.convert_field_protected
result_protected = self.vformat(format_string, args, all_params)
self.convert_field = self.convert_field_unprotected
result_unprotected = self.vformat(format_string, args, all_params)
if result_protected != result_unprotected:
Logger.sensitive_strings[result_unprotected] = result_protected
return result_unprotected
def test_attribute_logoutput(self, popen_mock, info_mock, select_mock,
os_read_mock):
subproc_mock = MagicMock()
subproc_mock.wait.return_value = MagicMock()
subproc_mock.stdout = MagicMock()
subproc_mock.returncode = 0
popen_mock.return_value = subproc_mock
select_mock.return_value = ([subproc_mock.stdout], None, None)
os_read_mock.return_value = None
with Environment("/") as env:
Execute('echo "1"', logoutput=True)
Execute('echo "2"', logoutput=False)
info_mock.assert_called('1')
self.assertTrue("call('2')" not in str(info_mock.mock_calls))
def test_attribute_path(self, popen_mock):
subproc_mock = MagicMock()
subproc_mock.returncode = 0
subproc_mock.communicate.side_effect = [["1"]]
popen_mock.return_value = subproc_mock
with Environment("/") as env:
execute_resource = Execute('echo "1"',
path=["/test/one", "test/two"])
if IS_WINDOWS:
self.assertEqual(execute_resource.environment["PATH"],
'/test/one;test/two')
else:
self.assertEqual(execute_resource.environment["PATH"],
'/test/one:test/two')
def test_add_block_to_filestream(self, file_contents, data, start_sentinel,
end_sentinel, expected_result):
new_file_contents = []
mocked_file = self.__mock_file(file_contents)
mocked_file.return_value.write = Mock(
side_effect=lambda s: new_file_contents.append(s) #pylint:disable=unnecessary-lambda
)
with self.__with_mocked_file(mocked_file), Environment('/'):
utilities.add_block_to_file('/test', data, None, start_sentinel,
end_sentinel)
new_file_contents = "".join(new_file_contents)
self.assertEqual(new_file_contents, expected_result)
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
# set up logging (two separate loggers for stderr and stdout with different loglevels)
logger = logging.getLogger('resource_management')
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(message)s')
chout = logging.StreamHandler(sys.stdout)
chout.setLevel(logging.INFO)
chout.setFormatter(formatter)
cherr = logging.StreamHandler(sys.stderr)
cherr.setLevel(logging.ERROR)
cherr.setFormatter(formatter)
logger.addHandler(cherr)
logger.addHandler(chout)
# parse arguments
if len(sys.argv) < 5:
logger.error("Script expects at least 4 arguments")
sys.exit(1)
command_name = str.lower(sys.argv[1])
# parse command parameters
command_data_file = sys.argv[2]
basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
try:
with open(command_data_file, "r") as f:
pass
Script.config = ConfigDictionary(json.load(f))
except IOError:
logger.exception(
"Can not read json file with command parameters: ")
sys.exit(1)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(command_name)
with Environment(basedir) as env:
method(env)
except ClientComponentHasNoStatus or ComponentIsNotRunning:
# Support of component status checks.
# Non-zero exit code is interpreted as an INSTALLED status of a component
sys.exit(1)
except Fail:
logger.exception(
"Error while executing command '{0}':".format(command_name))
sys.exit(1)
def test_attribute_cwd(self, popen_mock, select_mock, os_read_mock):
expected_cwd = "/test/work/directory"
subproc_mock = MagicMock()
subproc_mock.wait.return_value = MagicMock()
subproc_mock.stdout = MagicMock()
subproc_mock.returncode = 0
popen_mock.return_value = subproc_mock
select_mock.return_value = ([subproc_mock.stdout], None, None)
os_read_mock.return_value = None
with Environment("/") as env:
Execute('echo "1"',
cwd=expected_cwd
)
self.assertEqual(popen_mock.call_args_list[0][1]["cwd"], expected_cwd)
def test_attribute_environment(self, popen_mock, select_mock,
os_read_mock):
expected_dict = {"JAVA_HOME": "/test/java/home"}
subproc_mock = MagicMock()
subproc_mock.wait.return_value = MagicMock()
subproc_mock.stdout = MagicMock()
subproc_mock.returncode = 0
popen_mock.return_value = subproc_mock
select_mock.return_value = ([subproc_mock.stdout], None, None)
os_read_mock.return_value = None
with Environment("/") as env:
Execute('echo "1"', environment=expected_dict)
self.assertEqual(popen_mock.call_args_list[0][1]["env"], expected_dict)
pass
def action_delayed(self, action_name, main_resource):
resource = {}
env = Environment.get_instance()
if not 'hdfs_files' in env.config:
env.config['hdfs_files'] = []
# Put values in dictionary-resource
for field_name, json_field_name in RESOURCE_TO_JSON_FIELDS.iteritems():
if field_name == 'action':
resource[json_field_name] = action_name
elif field_name == 'mode' and main_resource.resource.mode:
resource[json_field_name] = oct(main_resource.resource.mode)[1:]
elif getattr(main_resource.resource, field_name):
resource[json_field_name] = getattr(main_resource.resource, field_name)
# Add resource to create
env.config['hdfs_files'].append(resource)
def action_create(self):
template_tag = self.resource.template_tag
qualified_file_name = self.resource.name
file_name = os.path.basename(qualified_file_name)
if not template_tag:
template_name = format("{file_name}.j2")
else:
template_name = format("{file_name}-{template_tag}.j2")
with Environment.get_instance_copy() as env:
File(qualified_file_name,
owner=self.resource.owner,
group=self.resource.group,
mode=self.resource.mode,
content=Template(template_name,
extra_imports=self.resource.extra_imports))
def action_create(self):
template_tag = self.resource.template_tag
qualified_file_name = self.resource.name
file_name = os.path.basename(qualified_file_name)
if not template_tag:
template_name = format("{file_name}.j2")
else:
template_name = format("{file_name}-{template_tag}.j2")
with Environment.get_instance_copy() as env:
File( qualified_file_name,
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode,
content = Template(template_name, extra_imports=self.resource.extra_imports)
)
def test_create_repo_redhat(self, file_mock, is_redhat_family,
is_ubuntu_family, is_suse_family):
is_redhat_family.return_value = True
is_ubuntu_family.return_value = False
is_suse_family.return_value = False
with Environment('/') as env:
with patch.object(repository,
"__file__",
new='/ambari/test/repo/dummy/path/file'):
Repository(
'hadoop',
base_url='http://download.base_url.org/rpm/',
mirror_list=
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository',
repo_template=RHEL_SUSE_DEFAULT_TEMPLATE)
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository'][
'hadoop'].arguments
expected_arguments = {
'repo_template': RHEL_SUSE_DEFAULT_TEMPLATE,
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
expected_template_arguments = {
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list':
'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'
}
self.assertEqual(defined_arguments, expected_arguments)
self.assertEqual(file_mock.call_args[0][0],
'/etc/yum.repos.d/Repository.repo')
template_item = file_mock.call_args[1]['content']
template = str(template_item.name)
expected_template_arguments.update({'repo_id': 'hadoop'})
self.assertEqual(expected_template_arguments,
template_item.context._dict)
self.assertEqual(RHEL_SUSE_DEFAULT_TEMPLATE, template)
def action_execute(self, main_resource, sudo=False):
env = Environment.get_instance()
env_dict_key = 'hdfs_files_sudo' if sudo else 'hdfs_files'
if not env_dict_key in env.config or not env.config[env_dict_key]:
return
# Check required parameters
if not sudo:
main_resource.assert_parameter_is_set('user')
user = main_resource.resource.user
else:
user = None
hadoop_bin_dir = main_resource.resource.hadoop_bin_dir
hadoop_conf_dir = main_resource.resource.hadoop_conf_dir
security_enabled = main_resource.resource.security_enabled
keytab_file = main_resource.resource.keytab
kinit_path = main_resource.resource.kinit_path_local
logoutput = main_resource.resource.logoutput
principal_name = main_resource.resource.principal_name
jar_path = JAR_PATH
timestamp = time.time()
json_path = format(JSON_PATH)
if security_enabled:
main_resource.kinit()
# Write json file to disk
File(json_path,
owner=user,
content=json.dumps(env.config[env_dict_key]))
# Execute jar to create/delete resources in hadoop
Execute(
('hadoop', '--config', hadoop_conf_dir, 'jar', jar_path,
json_path),
user=user,
path=[hadoop_bin_dir],
logoutput=logoutput,
sudo=sudo,
)
# Clean
env.config[env_dict_key] = []
def action_execute(self, main_resource):
env = Environment.get_instance()
# Check required parameters
if main_resource.has_core_configs:
main_resource.assert_parameter_is_set('user')
if not 'hdfs_files' in env.config or not env.config['hdfs_files']:
Logger.info(
"No resources to create. 'create_on_execute' or 'delete_on_execute' or 'download_on_execute' wasn't triggered before this 'execute' action."
)
return
hadoop_bin_dir = main_resource.resource.hadoop_bin_dir
hadoop_conf_dir = main_resource.resource.hadoop_conf_dir
user = main_resource.resource.user if main_resource.has_core_configs else None
security_enabled = main_resource.resource.security_enabled
keytab_file = main_resource.resource.keytab
kinit_path = main_resource.resource.kinit_path_local
logoutput = main_resource.resource.logoutput
principal_name = main_resource.resource.principal_name
jar_path = JAR_PATH
timestamp = time.time()
json_path = format(JSON_PATH)
if security_enabled:
main_resource.kinit()
# Write json file to disk
File(json_path,
owner=user,
content=json.dumps(env.config['hdfs_files']))
# Execute jar to create/delete resources in hadoop
Execute(
format(
"hadoop --config {hadoop_conf_dir} jar {jar_path} {json_path}"
),
user=user,
path=[hadoop_bin_dir],
logoutput=logoutput,
)
# Clean
env.config['hdfs_files'] = []
def __new__(cls, name, env=None, provider=None, **kwargs):
if isinstance(name, list):
while len(name) != 1:
cls(name.pop(0), env, provider, **kwargs)
name = name[0]
env = env or Environment.get_instance()
provider = provider or getattr(cls, 'provider', None)
r_type = cls.__name__
if r_type not in env.resources:
env.resources[r_type] = {}
obj = super(Resource, cls).__new__(cls)
env.resources[r_type][name] = obj
env.resource_list.append(obj)
return obj
def action_create(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = get_repo_dir()
new_content = InlineTemplate(
self.resource.repo_template,
repo_id=self.resource.repo_id,
repo_file_name=self.resource.repo_file_name,
base_url=self.resource.base_url,
mirror_list=self.resource.mirror_list)
repo_file_path = format("{repo_dir}/{repo_file_name}.repo")
if self.resource.append_to_file and os.path.isfile(repo_file_path):
content = sudo.read_file(
repo_file_path) + '\n' + new_content.get_content()
else:
content = new_content
File(repo_file_path, content=content)
def _get_delegation_token(namenode_address, user, keytab, principal,
kinit_path):
"""
Gets the kerberos delegation token from name node
"""
url = namenode_address + "/webhdfs/v1/?op=GETDELEGATIONTOKEN"
logger.info("Getting delegation token from {0} for PXF".format(url))
response, _, _ = curl_krb_request(Environment.get_instance().tmp_dir,
keytab, principal, url,
"get_delegation_token", kinit_path,
False, "Delegation Token", user)
json_response = json.loads(response)
if json_response['Token'] and json_response['Token']['urlString']:
return json_response['Token']['urlString']
msg = "Unable to get delegation token for PXF"
logger.error(msg)
raise Exception(msg)
def __new__(cls, name, env=None, provider=None, **kwargs):
if isinstance(name, list):
while len(name) != 1:
cls(name.pop(0), env, provider, **kwargs)
name = name[0]
env = env or Environment.get_instance()
provider = provider or getattr(cls, 'provider', None)
r_type = cls.__name__
if r_type not in env.resources:
env.resources[r_type] = {}
obj = super(Resource, cls).__new__(cls)
env.resources[r_type][name] = obj
env.resource_list.append(obj)
return obj
def test_set_kernel_parameter(self, execute_mock):
sysctl_contents = ""
mocked_file = self.__mock_file(sysctl_contents)
with self.__with_mocked_file(mocked_file), Environment('/'):
utilities.set_kernel_parameter('pname', 'pvalue')
self.assertTrue(execute_mock.called)
execute_mock_called_command = execute_mock.call_args[0][0].command
self.assertTrue(
'sysctl -w pname="pvalue"' in execute_mock_called_command,
"Invalid sysctl command called. %s" %
pformat(execute_mock_called_command))
handle = mocked_file()
handle.write.assert_called_once_with("pname = pvalue\n")
def action_create(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = repos_dirs[env.system.os_family]
repo_template = self.resource.repo_template
new_content = Template(
repo_template,
repo_id=self.resource.repo_id,
repo_file_name=self.resource.repo_file_name,
base_url=self.resource.base_url,
mirror_list=self.resource.mirror_list,
)
repo_file_path = format("{repo_dir}/{repo_file_name}.repo")
if self.resource.append_to_file and os.path.isfile(repo_file_path):
with open(repo_file_path, "a") as repo_file:
repo_file.write("\n" + new_content.get_content())
else:
File(repo_file_path, content=new_content)
def action_create(self):
with Environment.get_instance_copy() as env:
with tempfile.NamedTemporaryFile() as tmpf:
repo_file_name = format(
"{repo_file_name}.list",
repo_file_name=self.resource.repo_file_name)
repo_file_path = format("{repo_dir}/{repo_file_name}",
repo_dir=self.repo_dir)
repo_template = os.path.join(
os.path.dirname(os.path.realpath(__file__)), '..',
REPO_TEMPLATE_FOLDER, self.resource.repo_template)
new_content = Template(
repo_template,
package_type=self.package_type,
base_url=self.resource.base_url,
components=' '.join(
self.resource.components)).get_content()
old_content = ''
if self.resource.append_to_file and os.path.isfile(
repo_file_path):
with open(repo_file_path) as repo_file:
old_content = repo_file.read() + '\n'
File(tmpf.name, content=old_content + new_content)
if not os.path.isfile(repo_file_path) or not filecmp.cmp(
tmpf.name, repo_file_path):
File(repo_file_path, content=StaticFile(tmpf.name))
update_cmd_formatted = [format(x) for x in self.update_cmd]
# this is time expensive
retcode, out = checked_call(update_cmd_formatted,
sudo=True)
# add public keys for new repos
missing_pkeys = set(
re.findall(self.missing_pkey_regex, out))
for pkey in missing_pkeys:
Execute(
format(self.add_pkey_cmd),
timeout=
15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
ignore_failures=True)
def call_curl_request(self,user,keytab,principal, url, flag_http_response, request_method='GET',request_body='',header=''):
"""
:param user: service user for which call is to be made
:param keytab: keytab of service user
:param principal: principal of service user
:param url: url with which call is to be made
:param flag_http_response: flag to get only response-code or response string
:param request_method: http method (GET / POST / PUT / DELETE)
:param request_body: data to be send along with the request
:param header: http header required for the call
:return: Returns the response error_msg , time_millis
"""
response = None
error_msg = None
time_millis = 0
response, error_msg, time_millis = curl_krb_request(Environment.get_instance().tmp_dir, keytab, principal, url, 'ranger_admin_calls',
None, flag_http_response, "Ranger-Admin API calls", user,kinit_timer_ms=0,method = request_method,body=request_body,header=header)
return response, error_msg, time_millis
def test_remove_repo_redhat(self, file_mock,
is_redhat_family, is_ubuntu_family, is_suse_family):
is_redhat_family.return_value = True
is_ubuntu_family.return_value = False
is_suse_family.return_value = False
with Environment('/') as env:
Repository('hadoop',
action='remove',
base_url='http://download.base_url.org/rpm/',
mirror_list='https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
repo_file_name='Repository')
self.assertTrue('hadoop' in env.resources['Repository'])
defined_arguments = env.resources['Repository']['hadoop'].arguments
expected_arguments = {'action': ['remove'],
'base_url': 'http://download.base_url.org/rpm/',
'mirror_list': 'https://mirrors.base_url.org/?repo=Repository&arch=$basearch',
'repo_file_name': 'Repository'}
self.assertEqual(defined_arguments, expected_arguments)
def call_curl_request(self,user,keytab,principal, url, flag_http_response, request_method='GET',request_body='',header=''):
"""
:param user: service user for which call is to be made
:param keytab: keytab of service user
:param principal: principal of service user
:param url: url with which call is to be made
:param flag_http_response: flag to get only response-code or response string
:param request_method: http method (GET / POST / PUT / DELETE)
:param request_body: data to be send along with the request
:param header: http header required for the call
:return: Returns the response error_msg , time_millis
"""
response = None
error_msg = None
time_millis = 0
response, error_msg, time_millis = curl_krb_request(Environment.get_instance().tmp_dir, keytab, principal, url, 'ranger_admin_calls',
None, flag_http_response, "Ranger-Admin API calls", user,kinit_timer_ms=0,method = request_method,body=request_body,header=header)
return response, error_msg, time_millis
def get_check_command(oozie_url, host_name, configurations):
security_enabled = False
if SECURITY_ENABLED in configurations:
security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'
kerberos_env = None
if security_enabled:
if OOZIE_KEYTAB in configurations and OOZIE_PRINCIPAL in configurations:
oozie_keytab = configurations[OOZIE_KEYTAB]
oozie_principal = configurations[OOZIE_PRINCIPAL]
# substitute _HOST in kerberos principal with actual fqdn
oozie_principal = oozie_principal.replace('_HOST', host_name)
else:
raise KerberosPropertiesNotFound('The Oozie keytab and principal are required configurations when security is enabled.')
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
# Get the configured Kerberos executable search paths, if any
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
else:
kerberos_executable_search_paths = None
klist_path_local = get_klist_path(kerberos_executable_search_paths)
klist_command = format("{klist_path_local} -s {ccache_file}")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
return_code, _ = call(klist_command)
if return_code != 0:
kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
kinit_command = format("{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; ")
# kinit
Execute(kinit_command, environment=kerberos_env)
command = format("source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status")
return (command, kerberos_env)
def action_create(self):
filename = self.resource.filename
xml_config_provider_config_dir = self.resource.conf_dir
# |e - for html-like escaping of <,>,',"
config_content = InlineTemplate(
'''<!--{{time.asctime(time.localtime())}}-->
<configuration>
{% for key, value in configurations_dict|dictsort %}
<property>
<name>{{ key|e }}</name>
<value>{{ resource_management.core.source.InlineTemplate(str(value)).get_content().strip() |e }}</value>
{%- if not configuration_attrs is none -%}
{%- for attrib_name, attrib_occurances in configuration_attrs.items() -%}
{%- for property_name, attrib_value in attrib_occurances.items() -%}
{% if property_name == key and attrib_name %}
<{{attrib_name|e}}>{{attrib_value|e}}</{{attrib_name|e}}>
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
{%- endif %}
</property>
{% endfor %}
</configuration>''',
extra_imports=[
time, resource_management, resource_management.core,
resource_management.core.source
],
configurations_dict=self.resource.configurations,
configuration_attrs=self.resource.configuration_attributes)
xml_config_dest_file_path = os.path.join(
xml_config_provider_config_dir, filename)
Logger.info("Generating config: {0}".format(xml_config_dest_file_path))
with Environment.get_instance_copy() as env:
File(xml_config_dest_file_path,
content=config_content,
owner=self.resource.owner,
group=self.resource.group,
mode=self.resource.mode,
encoding=self.resource.encoding)
def action_create(self):
filename = self.resource.filename
comment_symbols = self.resource.comment_symbols
delimiter = self.resource.key_value_delimiter
properties = self.resource.properties
unsaved_values = properties.keys()
new_content_lines = []
if sudo.path_isfile(filename):
file_content = sudo.read_file(filename, encoding=self.resource.encoding)
new_content_lines += file_content.split('\n')
Logger.info(format("Modifying existing properties file: {filename}"))
for line_num in range(len(new_content_lines)):
line = new_content_lines[line_num]
if line.lstrip() and not line.lstrip()[0] in comment_symbols and delimiter in line:
in_var_name = line.split(delimiter)[0].strip()
in_var_value = line.split(delimiter)[1].strip()
if in_var_name in properties:
value = InlineTemplate(unicode(properties[in_var_name])).get_content()
new_content_lines[line_num] = u"{0}{1}{2}".format(unicode(in_var_name), delimiter, value)
unsaved_values.remove(in_var_name)
else:
Logger.info(format("Creating new properties file as {filename} doesn't exist"))
for property_name in unsaved_values:
value = InlineTemplate(unicode(properties[property_name])).get_content()
line = u"{0}{1}{2}".format(unicode(property_name), delimiter, value)
new_content_lines.append(line)
with Environment.get_instance_copy() as env:
File (filename,
content = u"\n".join(new_content_lines) + "\n",
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode,
encoding = self.resource.encoding,
)
def action_create(self):
with Environment.get_instance_copy() as env:
with tempfile.NamedTemporaryFile() as tmpf:
with tempfile.NamedTemporaryFile() as old_repo_tmpf:
repo_file_name = format("{repo_file_name}.list",repo_file_name=self.resource.repo_file_name)
repo_file_path = format("{repo_dir}/{repo_file_name}", repo_dir=self.repo_dir)
new_content = InlineTemplate(self.resource.repo_template, package_type=self.package_type,
base_url=self.resource.base_url,
components=' '.join(self.resource.components)).get_content()
old_content = ''
if self.resource.append_to_file and os.path.isfile(repo_file_path):
old_content = sudo.read_file(repo_file_path) + '\n'
File(tmpf.name,
content=old_content+new_content
)
if os.path.isfile(repo_file_path):
# a copy of old repo file, which will be readable by current user
File(old_repo_tmpf.name,
content=StaticFile(repo_file_path),
)
if not os.path.isfile(repo_file_path) or not filecmp.cmp(tmpf.name, old_repo_tmpf.name):
File(repo_file_path,
content = StaticFile(tmpf.name)
)
update_cmd_formatted = [format(x) for x in self.update_cmd]
# this is time expensive
retcode, out = checked_call(update_cmd_formatted, sudo=True, quiet=False)
# add public keys for new repos
missing_pkeys = set(re.findall(self.missing_pkey_regex, out))
for pkey in missing_pkeys:
Execute(self.app_pkey_cmd_prefix + (pkey,),
timeout = 15, # in case we are on the host w/o internet (using localrepo), we should ignore hanging
ignore_failures = True,
sudo = True,
)
def configure(self, env):
import params
env.set_params(params)
if params.monitor_security_enabled and self.component == 'monitor':
import os
import random
import string
basedir = Environment.get_instance().config.basedir
keystore_file = os.path.join(basedir, "files", "keystore.jks")
truststore_file = os.path.join(basedir, "files", "cacerts.jks")
cert_file = os.path.join(basedir, "files", "server.cer")
if os.path.exists(keystore_file) or os.path.exists(truststore_file) or os.path.exists(cert_file):
self.fail_with_error("trying to create monitor certs but they already existed")
goodchars = string.lowercase + string.uppercase + string.digits + '#%+,-./:=?@^_'
keypass = ''.join(random.choice(goodchars) for x in range(20))
storepass = ''.join(random.choice(goodchars) for x in range(20))
https_params = {}
https_params[params.keystore_property] = params.keystore_path
https_params[params.truststore_property] = params.truststore_path
https_params[params.keystore_password_property] = keypass
https_params[params.truststore_password_property] = storepass
setup_conf_dir(name=self.component, extra_params=https_params)
Execute( format("{java64_home}/bin/keytool -genkey -alias \"default\" -keyalg RSA -keypass {keypass} -storepass {storepass} -keystore {keystore_file} -dname \"CN=Unknown, OU=Unknown, O=Unknown, L=Unknown, ST=Unknown, C=Unknown\""),
user=params.accumulo_user)
Execute( format("{java64_home}/bin/keytool -export -alias \"default\" -storepass {storepass} -file {cert_file} -keystore {keystore_file}"),
user=params.accumulo_user)
Execute( format("echo \"yes\" | {java64_home}/bin/keytool -import -v -trustcacerts -alias \"default\" -file {cert_file} -keystore {truststore_file} -keypass {keypass} -storepass {storepass}"),
user=params.accumulo_user)
accumulo_StaticFile("keystore.jks")
accumulo_StaticFile("cacerts.jks")
else:
setup_conf_dir(name=self.component)
def _collect(self):
cmd_module = self._load_source()
if cmd_module is not None:
configurations = {}
try:
tokens = cmd_module.get_tokens()
if tokens is not None:
# for each token, if there is a value, store in; otherwise don't store
# a key with a value of None
for token in tokens:
value = self._get_configuration_value(token)
if value is not None:
configurations[token] = value
except AttributeError:
# it's OK if the module doesn't have get_tokens() ; no tokens will
# be passed in so hopefully the script doesn't need any
logger.debug("The script {0} does not have a get_tokens() function".format(str(cmd_module)))
# try to get basedir for scripts
# it's needed for server side scripts to properly use resource management
matchObj = re.match( r'((.*)services(.*)package)', self.path_to_script)
if matchObj:
basedir = matchObj.group(1)
with Environment(basedir, tmp_dir=AGENT_TMP_DIR, logger=logging.getLogger('ambari_alerts')) as env:
result = cmd_module.execute(configurations, self.parameters, self.host_name)
else:
result = cmd_module.execute(configurations, self.parameters, self.host_name)
loggerMsg = "[Alert][{0}] Failed with result {2}: {3}".format(
self.get_name(), self.path_to_script, result[0], result[1])
if result[0] == self.RESULT_CRITICAL:
logger.error(loggerMsg)
elif result[0] == self.RESULT_WARNING or result[0] == self.RESULT_UNKNOWN:
logger.debug(loggerMsg)
return result
else:
return (self.RESULT_UNKNOWN, ["Unable to execute script {0}".format(self.path)])
def action_execute(self, main_resource):
env = Environment.get_instance()
# Check required parameters
main_resource.assert_parameter_is_set('user')
if not 'hdfs_files' in env.config or not env.config['hdfs_files']:
Logger.info("No resources to create. 'create_on_execute' or 'delete_on_execute' wasn't triggered before this 'execute' action.")
return
hadoop_bin_dir = main_resource.resource.hadoop_bin_dir
hadoop_conf_dir = main_resource.resource.hadoop_conf_dir
user = main_resource.resource.user
security_enabled = main_resource.resource.security_enabled
keytab_file = main_resource.resource.keytab
kinit_path = main_resource.resource.kinit_path_local
logoutput = main_resource.resource.logoutput
principal_name = main_resource.resource.principal_name
jar_path=JAR_PATH
timestamp = time.time()
json_path=format(JSON_PATH)
if security_enabled:
main_resource.kinit()
# Write json file to disk
File(json_path,
owner = user,
content = json.dumps(env.config['hdfs_files'])
)
# Execute jar to create/delete resources in hadoop
Execute(format("hadoop --config {hadoop_conf_dir} jar {jar_path} {json_path}"),
user=user,
path=[hadoop_bin_dir],
logoutput=logoutput,
)
# Clean
env.config['hdfs_files'] = []
def action_create(self):
filename = self.resource.filename
dir = self.resource.dir
if dir == None:
filepath = filename
else:
filepath = os.path.join(dir, filename)
config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
{% for key, value in properties_dict|dictsort %}
{{key}}{{key_value_delimiter}}{{ resource_management.core.source.InlineTemplate(str(value)).get_content() }}{% endfor %}
''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], properties_dict=self.resource.properties, key_value_delimiter=self.resource.key_value_delimiter)
Logger.info(format("Generating properties file: {filepath}"))
with Environment.get_instance_copy() as env:
File (format("{filepath}"),
content = config_content,
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode
)
def action_run(self):
kinit__path_local = self.resource.kinit_path_local
keytab = self.resource.keytab
conf_dir = self.resource.conf_dir
command = self.resource.command
principal = self.resource.principal
if isinstance(command, (list, tuple)):
command = " ".join(quote_bash_args(x) for x in command)
with Environment.get_instance_copy() as env:
if self.resource.security_enabled and not self.resource.kinit_override:
Execute(format("{kinit__path_local} -kt {keytab} {principal}"), path=["/bin"], user=self.resource.user)
Execute(
format("hadoop --config {conf_dir} {command}"),
user=self.resource.user,
tries=self.resource.tries,
try_sleep=self.resource.try_sleep,
logoutput=self.resource.logoutput,
path=self.resource.bin_dir,
environment=self.resource.environment,
)
def __init__(self, name, env=None, provider=None, **kwargs):
if isinstance(name, list):
name = name.pop(0)
if hasattr(self, 'name'):
return
self.env = env or Environment.get_instance()
self.name = name
self.provider = provider or getattr(self, 'provider', None)
self.arguments = {}
for key, value in kwargs.items():
try:
arg = self._arguments[key]
except KeyError:
raise Fail("%s received unsupported argument %s" % (self, key))
else:
try:
self.arguments[key] = arg.validate(value)
except InvalidArgument, exc:
raise InvalidArgument("%s %s" % (self, exc))
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code, ['There were no configurations supplied to the script.'])
webhcat_port = WEBHCAT_PORT_DEFAULT
if TEMPLETON_PORT_KEY in configurations:
webhcat_port = int(configurations[TEMPLETON_PORT_KEY])
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = configurations[SECURITY_ENABLED_KEY].lower() == 'true'
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
curl_connection_timeout = CURL_CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
curl_connection_timeout = str(int(connection_timeout))
# the alert will always run on the webhcat host
if host_name is None:
host_name = socket.getfqdn()
smokeuser = SMOKEUSER_DEFAULT
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
# webhcat always uses http, never SSL
query_url = "http://{0}:{1}/templeton/v1/status?user.name={2}".format(host_name, webhcat_port, smokeuser)
# initialize
total_time = 0
json_response = {}
if security_enabled:
try:
# defaults
smokeuser_keytab = SMOKEUSER_KEYTAB_DEFAULT
smokeuser_principal = SMOKEUSER_PRINCIPAL_DEFAULT
# check script params
if SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
smokeuser_principal = parameters[SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY]
if SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
smokeuser_keytab = parameters[SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY]
# check configurations last as they should always take precedence
if SMOKEUSER_PRINCIPAL_KEY in configurations:
smokeuser_principal = configurations[SMOKEUSER_PRINCIPAL_KEY]
if SMOKEUSER_KEYTAB_KEY in configurations:
smokeuser_keytab = configurations[SMOKEUSER_KEYTAB_KEY]
# Get the configured Kerberos executable search paths, if any
kerberos_executable_search_paths = None
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
env = Environment.get_instance()
stdout, stderr, time_millis = curl_krb_request(env.tmp_dir, smokeuser_keytab, smokeuser_principal,
query_url, "webhcat_alert_cc_", kerberos_executable_search_paths, True,
"WebHCat Server Status", smokeuser,
connection_timeout=curl_connection_timeout)
# check the response code
response_code = int(stdout)
# 0 indicates no connection
if response_code == 0:
label = CRITICAL_CONNECTION_MESSAGE.format(query_url)
return (RESULT_CODE_CRITICAL, [label])
# any other response aside from 200 is a problem
if response_code != 200:
label = CRITICAL_HTTP_MESSAGE.format(response_code, query_url)
return (RESULT_CODE_CRITICAL, [label])
# now that we have the http status and it was 200, get the content
stdout, stderr, total_time = curl_krb_request(env.tmp_dir, smokeuser_keytab, smokeuser_principal,
query_url, "webhcat_alert_cc_", kerberos_executable_search_paths,
False, "WebHCat Server Status", smokeuser,
connection_timeout=curl_connection_timeout)
json_response = json.loads(stdout)
except Exception, exception:
return (RESULT_CODE_CRITICAL, [str(exception)])
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
result_code = RESULT_CODE_UNKNOWN
if configurations is None:
return (result_code, ['There were no configurations supplied to the script.'])
webhcat_port = WEBHCAT_PORT_DEFAULT
if TEMPLETON_PORT_KEY in configurations:
webhcat_port = int(configurations[TEMPLETON_PORT_KEY])
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = configurations[SECURITY_ENABLED_KEY].lower() == 'true'
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
curl_connection_timeout = CURL_CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
curl_connection_timeout = str(int(connection_timeout))
# the alert will always run on the webhcat host
if host_name is None:
host_name = socket.getfqdn()
smokeuser = SMOKEUSER_DEFAULT
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
# webhcat always uses http, never SSL
query_url = "http://{0}:{1}/templeton/v1/status?user.name={2}".format(host_name, webhcat_port, smokeuser)
# initialize
total_time = 0
json_response = {}
if security_enabled:
try:
# defaults
smokeuser_keytab = SMOKEUSER_KEYTAB_DEFAULT
smokeuser_principal = SMOKEUSER_PRINCIPAL_DEFAULT
# check script params
if SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
smokeuser_principal = parameters[SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY]
if SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
smokeuser_keytab = parameters[SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY]
# check configurations last as they should always take precedence
if SMOKEUSER_PRINCIPAL_KEY in configurations:
smokeuser_principal = configurations[SMOKEUSER_PRINCIPAL_KEY]
if SMOKEUSER_KEYTAB_KEY in configurations:
smokeuser_keytab = configurations[SMOKEUSER_KEYTAB_KEY]
# Get the configured Kerberos executable search paths, if any
kerberos_executable_search_paths = None
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
env = Environment.get_instance()
stdout, stderr, time_millis = curl_krb_request(env.tmp_dir, smokeuser_keytab, smokeuser_principal,
query_url, "webhcat_alert_cc_", kerberos_executable_search_paths, True,
"WebHCat Server Status", smokeuser,
connection_timeout=curl_connection_timeout)
# check the response code
response_code = int(stdout)
# 0 indicates no connection
if response_code == 0:
label = CRITICAL_CONNECTION_MESSAGE.format(query_url, traceback.format_exc())
return (RESULT_CODE_CRITICAL, [label])
# any other response aside from 200 is a problem
if response_code != 200:
label = CRITICAL_HTTP_MESSAGE.format(response_code, query_url, traceback.format_exc())
return (RESULT_CODE_CRITICAL, [label])
# now that we have the http status and it was 200, get the content
stdout, stderr, total_time = curl_krb_request(env.tmp_dir, smokeuser_keytab, smokeuser_principal,
query_url, "webhcat_alert_cc_", kerberos_executable_search_paths,
False, "WebHCat Server Status", smokeuser,
connection_timeout=curl_connection_timeout)
json_response = json.loads(stdout)
except:
return (RESULT_CODE_CRITICAL, [traceback.format_exc()])
else:
url_response = None
try:
# execute the query for the JSON that includes WebHCat status
start_time = time.time()
url_response = urllib2.urlopen(query_url, timeout=connection_timeout)
total_time = time.time() - start_time
json_response = json.loads(url_response.read())
except urllib2.HTTPError as httpError:
label = CRITICAL_HTTP_MESSAGE.format(httpError.code, query_url, traceback.format_exc())
return (RESULT_CODE_CRITICAL, [label])
except:
label = CRITICAL_CONNECTION_MESSAGE.format(query_url, traceback.format_exc())
return (RESULT_CODE_CRITICAL, [label])
finally:
if url_response is not None:
try:
url_response.close()
except:
pass
# if status is not in the response, we can't do any check; return CRIT
if 'status' not in json_response:
return (RESULT_CODE_CRITICAL, [CRITICAL_WEBHCAT_UNKNOWN_JSON_MESSAGE + str(json_response)])
# URL response received, parse it
try:
webhcat_status = json_response['status']
except:
return (RESULT_CODE_CRITICAL, [CRITICAL_WEBHCAT_UNKNOWN_JSON_MESSAGE + "\n" + traceback.format_exc()])
# proper JSON received, compare against known value
if webhcat_status.lower() == WEBHCAT_OK_RESPONSE:
result_code = RESULT_CODE_OK
label = OK_MESSAGE.format(total_time, query_url)
else:
result_code = RESULT_CODE_CRITICAL
label = CRITICAL_WEBHCAT_STATUS_MESSAGE.format(webhcat_status)
return (result_code, [label])
def get_check_command(oozie_url, host_name, configurations, parameters, only_kinit):
kerberos_env = None
user = USER_DEFAULT
if USER_KEY in configurations:
user = configurations[USER_KEY]
if is_security_enabled(configurations):
# defaults
user_keytab = USER_KEYTAB_DEFAULT
user_principal = USER_PRINCIPAL_DEFAULT
# check script params
if USER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
user_principal = parameters[USER_PRINCIPAL_SCRIPT_PARAM_KEY]
user_principal = user_principal.replace('_HOST', host_name.lower())
if USER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
user_keytab = parameters[USER_KEYTAB_SCRIPT_PARAM_KEY]
# check configurations last as they should always take precedence
if USER_PRINCIPAL_KEY in configurations:
user_principal = configurations[USER_PRINCIPAL_KEY]
user_principal = user_principal.replace('_HOST', host_name.lower())
if USER_KEYTAB_KEY in configurations:
user_keytab = configurations[USER_KEYTAB_KEY]
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
# Get the configured Kerberos executable search paths, if any
kerberos_executable_search_paths = None
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
klist_path_local = get_klist_path(kerberos_executable_search_paths)
kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
kinit_part_command = format("{kinit_path_local} -l 5m20s -c {ccache_file} -kt {user_keytab} {user_principal}; ")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
if only_kinit:
kinit_command = kinit_part_command
else:
kinit_command = "{0} -s {1} || ".format(klist_path_local, ccache_file) + kinit_part_command
Execute(kinit_command, environment=kerberos_env, user=user)
# oozie configuration directory uses a symlink when > HDP 2.2
oozie_config_directory = OOZIE_CONF_DIR_LEGACY
if os.path.exists(OOZIE_CONF_DIR):
oozie_config_directory = OOZIE_CONF_DIR
command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
oozie_config_directory, oozie_url)
return (command, kerberos_env, user)
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
if configurations is None:
return (('UNKNOWN', ['There were no configurations supplied to the script.']))
scheme = 'http'
http_uri = None
https_uri = None
http_policy = 'HTTP_ONLY'
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
executable_paths = None
if EXECUTABLE_SEARCH_PATHS in configurations:
executable_paths = configurations[EXECUTABLE_SEARCH_PATHS]
kerberos_keytab = None
if KERBEROS_KEYTAB in configurations:
kerberos_keytab = configurations[KERBEROS_KEYTAB]
kerberos_principal = None
if KERBEROS_PRINCIPAL in configurations:
kerberos_principal = configurations[KERBEROS_PRINCIPAL]
kerberos_principal = kerberos_principal.replace('_HOST', host_name)
if NODEMANAGER_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NODEMANAGER_HTTP_ADDRESS_KEY]
if NODEMANAGER_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NODEMANAGER_HTTPS_ADDRESS_KEY]
if YARN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[YARN_HTTP_POLICY_KEY]
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == 'HTTPS_ONLY':
scheme = 'https'
if https_uri is not None:
uri = https_uri
uri = str(host_name) + ":" + uri.split(":")[1]
live_nodemanagers_qry = "{0}://{1}/jmx?qry=Hadoop:service=ResourceManager,name=RMNMInfo".format(scheme, uri)
convert_to_json_failed = False
response_code = None
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
env = Environment.get_instance()
# curl requires an integer timeout
curl_connection_timeout = int(connection_timeout)
url_response, error_msg, time_millis = curl_krb_request(env.tmp_dir, kerberos_keytab, kerberos_principal,
live_nodemanagers_qry, "nm_health_summary_alert", executable_paths, False,
"NodeManager Health Summary", smokeuser, connection_timeout=curl_connection_timeout)
try:
url_response_json = json.loads(url_response)
live_nodemanagers = json.loads(url_response_json["beans"][0]["LiveNodeManagers"])
except ValueError, error:
convert_to_json_failed = True
logger.exception("[Alert][{0}] Convert response to json failed or json doesn't contain needed data: {1}".
format("NodeManager Health Summary", str(error)))
if convert_to_json_failed:
response_code, error_msg, time_millis = curl_krb_request(env.tmp_dir, kerberos_keytab, kerberos_principal,
live_nodemanagers_qry, "nm_health_summary_alert", executable_paths, True,
"NodeManager Health Summary", smokeuser, connection_timeout=curl_connection_timeout)
else:
def execute(configurations={}, parameters={}, host_name=None):
"""
Returns a tuple containing the result code and a pre-formatted result label
Keyword arguments:
configurations (dictionary): a mapping of configuration key to value
parameters (dictionary): a mapping of script parameter key to value
host_name (string): the name of this host where the alert is running
"""
if configurations is None:
return ("UNKNOWN", ["There were no configurations supplied to the script."])
uri = None
scheme = "http"
http_uri = None
https_uri = None
http_policy = "HTTP_ONLY"
checkpoint_tx = CHECKPOINT_TX_DEFAULT
checkpoint_period = CHECKPOINT_PERIOD_DEFAULT
if NN_HTTP_ADDRESS_KEY in configurations:
http_uri = configurations[NN_HTTP_ADDRESS_KEY]
if NN_HTTPS_ADDRESS_KEY in configurations:
https_uri = configurations[NN_HTTPS_ADDRESS_KEY]
if NN_HTTP_POLICY_KEY in configurations:
http_policy = configurations[NN_HTTP_POLICY_KEY]
if NN_CHECKPOINT_TX_KEY in configurations:
checkpoint_tx = configurations[NN_CHECKPOINT_TX_KEY]
if NN_CHECKPOINT_PERIOD_KEY in configurations:
checkpoint_period = configurations[NN_CHECKPOINT_PERIOD_KEY]
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
executable_paths = None
if EXECUTABLE_SEARCH_PATHS in configurations:
executable_paths = configurations[EXECUTABLE_SEARCH_PATHS]
security_enabled = False
if SECURITY_ENABLED_KEY in configurations:
security_enabled = str(configurations[SECURITY_ENABLED_KEY]).upper() == "TRUE"
kerberos_keytab = None
if KERBEROS_KEYTAB in configurations:
kerberos_keytab = configurations[KERBEROS_KEYTAB]
kerberos_principal = None
if KERBEROS_PRINCIPAL in configurations:
kerberos_principal = configurations[KERBEROS_PRINCIPAL]
kerberos_principal = kerberos_principal.replace("_HOST", host_name)
# parse script arguments
connection_timeout = CONNECTION_TIMEOUT_DEFAULT
if CONNECTION_TIMEOUT_KEY in parameters:
connection_timeout = float(parameters[CONNECTION_TIMEOUT_KEY])
percent_warning = PERCENT_WARNING_DEFAULT
if PERCENT_WARNING_KEY in parameters:
percent_warning = float(parameters[PERCENT_WARNING_KEY]) * 100
percent_critical = PERCENT_CRITICAL_DEFAULT
if PERCENT_CRITICAL_KEY in parameters:
percent_critical = float(parameters[PERCENT_CRITICAL_KEY]) * 100
# determine the right URI and whether to use SSL
uri = http_uri
if http_policy == "HTTPS_ONLY":
scheme = "https"
if https_uri is not None:
uri = https_uri
current_time = int(round(time.time() * 1000))
last_checkpoint_time_qry = "{0}://{1}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem".format(scheme, uri)
journal_transaction_info_qry = "{0}://{1}/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo".format(scheme, uri)
# start out assuming an OK status
label = None
result_code = "OK"
try:
if kerberos_principal is not None and kerberos_keytab is not None and security_enabled:
env = Environment.get_instance()
# curl requires an integer timeout
curl_connection_timeout = int(connection_timeout)
last_checkpoint_time_response, error_msg, time_millis = curl_krb_request(
env.tmp_dir,
kerberos_keytab,
kerberos_principal,
last_checkpoint_time_qry,
"checkpoint_time_alert",
executable_paths,
False,
"NameNode Last Checkpoint",
smokeuser,
connection_timeout=curl_connection_timeout,
)
last_checkpoint_time_response_json = json.loads(last_checkpoint_time_response)
last_checkpoint_time = int(last_checkpoint_time_response_json["beans"][0]["LastCheckpointTime"])
journal_transaction_info_response, error_msg, time_millis = curl_krb_request(
env.tmp_dir,
kerberos_keytab,
kerberos_principal,
journal_transaction_info_qry,
"checkpoint_time_alert",
executable_paths,
False,
"NameNode Last Checkpoint",
smokeuser,
connection_timeout=curl_connection_timeout,
)
journal_transaction_info_response_json = json.loads(journal_transaction_info_response)
journal_transaction_info = journal_transaction_info_response_json["beans"][0]["JournalTransactionInfo"]
else:
last_checkpoint_time = int(
get_value_from_jmx(last_checkpoint_time_qry, "LastCheckpointTime", connection_timeout)
)
journal_transaction_info = get_value_from_jmx(
journal_transaction_info_qry, "JournalTransactionInfo", connection_timeout
)
journal_transaction_info_dict = json.loads(journal_transaction_info)
last_tx = int(journal_transaction_info_dict["LastAppliedOrWrittenTxId"])
most_recent_tx = int(journal_transaction_info_dict["MostRecentCheckpointTxId"])
transaction_difference = last_tx - most_recent_tx
delta = (current_time - last_checkpoint_time) / 1000
label = LABEL.format(h=get_time(delta)["h"], m=get_time(delta)["m"], tx=transaction_difference)
if (transaction_difference > int(checkpoint_tx)) and (
float(delta) / int(checkpoint_period) * 100 >= int(percent_critical)
):
result_code = "CRITICAL"
elif (transaction_difference > int(checkpoint_tx)) and (
float(delta) / int(checkpoint_period) * 100 >= int(percent_warning)
):
result_code = "WARNING"
except Exception, e:
label = str(e)
result_code = "UNKNOWN"
def __init__(self, name):
self.env = Environment.get_instance()
self.name = name
def action_remove(self):
with Environment.get_instance_copy() as env:
repo_file_name = self.resource.repo_file_name
repo_dir = repos_dirs[env.system.os_family]
File(format("{repo_dir}/{repo_file_name}.repo"), action="delete")
def __init__(self, env=None):
self.env = env or Environment.get_instance()
def get_check_command(oozie_url, host_name, configurations, parameters):
kerberos_env = None
smokeuser = SMOKEUSER_DEFAULT
if SMOKEUSER_KEY in configurations:
smokeuser = configurations[SMOKEUSER_KEY]
security_enabled = False
if SECURITY_ENABLED in configurations:
security_enabled = str(configurations[SECURITY_ENABLED]).upper() == 'TRUE'
if security_enabled:
# defaults
smokeuser_keytab = SMOKEUSER_KEYTAB_DEFAULT
smokeuser_principal = SMOKEUSER_PRINCIPAL_DEFAULT
# check script params
if SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY in parameters:
smokeuser_principal = parameters[SMOKEUSER_PRINCIPAL_SCRIPT_PARAM_KEY]
if SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY in parameters:
smokeuser_keytab = parameters[SMOKEUSER_KEYTAB_SCRIPT_PARAM_KEY]
# check configurations last as they should always take precedence
if SMOKEUSER_PRINCIPAL_KEY in configurations:
smokeuser_principal = configurations[SMOKEUSER_PRINCIPAL_KEY]
if SMOKEUSER_KEYTAB_KEY in configurations:
smokeuser_keytab = configurations[SMOKEUSER_KEYTAB_KEY]
# Create the kerberos credentials cache (ccache) file and set it in the environment to use
# when executing curl
env = Environment.get_instance()
ccache_file = "{0}{1}oozie_alert_cc_{2}".format(env.tmp_dir, os.sep, os.getpid())
kerberos_env = {'KRB5CCNAME': ccache_file}
# Get the configured Kerberos executable search paths, if any
kerberos_executable_search_paths = None
if KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY in configurations:
kerberos_executable_search_paths = configurations[KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY]
klist_path_local = get_klist_path(kerberos_executable_search_paths)
klist_command = format("{klist_path_local} -s {ccache_file}")
# Determine if we need to kinit by testing to see if the relevant cache exists and has
# non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number
# it kinits we do but recover quickly when keytabs are regenerated
return_code, _ = call(klist_command, user=smokeuser)
if return_code != 0:
kinit_path_local = get_kinit_path(kerberos_executable_search_paths)
kinit_command = format("{kinit_path_local} -l 5m -kt {smokeuser_keytab} {smokeuser_principal}; ")
# kinit
Execute(kinit_command, environment=kerberos_env, user=smokeuser)
# oozie configuration directory uses a symlink when > HDP 2.2
oozie_config_directory = OOZIE_CONF_DIR_LEGACY
if os.path.exists(OOZIE_CONF_DIR):
oozie_config_directory = OOZIE_CONF_DIR
command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
oozie_config_directory, oozie_url)
return (command, kerberos_env, smokeuser)
