def test_action_create_dir_exist(self, isdir_mock, dirname_mock):
"""
Tests if 'create' action fails when path is existent directory
"""
isdir_mock.side_effect = [True, False]
try:
with Environment('/') as env:
File('/existent_directory',
action='create',
mode=0777,
content='file-content')
self.fail("Must fail when directory with name 'path' exist")
except Fail as e:
self.assertEqual(
'Applying File[\'/existent_directory\'] failed, directory with name /existent_directory exists',
str(e))
self.assertFalse(dirname_mock.called)
def test_action_create_replace(self, isdir_mock, exists_mock,
create_file_mock, read_file_mock,
ensure_mock):
"""
Tests if 'create' action rewrite existent file with new data
"""
isdir_mock.side_effect = [False, True]
exists_mock.return_value = True
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
backup=False,
content='new-content')
read_file_mock.assert_called_with('/directory/file')
create_file_mock.assert_called_with('/directory/file', 'new-content')
def install(self, env):
Logger.info('Installing R Server Client...')
tmp_dir = Script.tmp_dir
Logger.debug('Using temp dir: {0}'.format(tmp_dir))
Logger.info(
"Will download and install the following rpm's from {0}: {1}".
format(rpm_location, rpms))
for rpm in rpms:
Logger.info('Downloading {0}'.format(rpm))
rpmFile = '{0}/{1}'.format(tmp_dir, rpm)
File(rpmFile, \
content = DownloadSource(rpm_location + rpm), \
mode = 0644)
Logger.info('Installing {0}'.format(rpm))
Package(rpmFile)
Logger.info('Installed R Server')
def test_attribute_path(self, isdir_mock):
"""
Tests 'path' attribute
"""
isdir_mock.side_effect = [True, False]
try:
with Environment('/') as env:
File('/existent_directory',
action='create',
mode=0777,
content='file-content')
env.run()
self.fail("Must fail when directory with name 'path' exist")
except Fail as e:
pass
isdir_mock.assert_called_with('/existent_directory')
def generate_configs(self, env):
"""
Generates config files and stores them as an archive in tmp_dir
based on xml_configs_list and env_configs_list from commandParams
"""
import params
env.set_params(params)
config = self.get_config()
xml_configs_list = config['commandParams']['xml_configs_list']
env_configs_list = config['commandParams']['env_configs_list']
properties_configs_list = config['commandParams'][
'properties_configs_list']
Directory(self.get_tmp_dir(), recursive=True)
conf_tmp_dir = tempfile.mkdtemp(dir=self.get_tmp_dir())
output_filename = os.path.join(self.get_tmp_dir(),
config['commandParams']['output_file'])
try:
for file_dict in xml_configs_list:
for filename, dict in file_dict.iteritems():
XmlConfig(filename,
conf_dir=conf_tmp_dir,
**self.generate_configs_get_xml_file_content(
filename, dict))
for file_dict in env_configs_list:
for filename, dicts in file_dict.iteritems():
File(os.path.join(conf_tmp_dir, filename),
content=InlineTemplate(
self.generate_configs_get_template_file_content(
filename, dicts)))
for file_dict in properties_configs_list:
for filename, dict in file_dict.iteritems():
PropertiesFile(
os.path.join(conf_tmp_dir, filename),
properties=self.generate_configs_get_xml_file_dict(
filename, dict))
archive_dir(output_filename, conf_tmp_dir)
finally:
Directory(conf_tmp_dir, action="delete")
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source)
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
not_if=format("test -f {driver_curl_target}"),
sudo=True
)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def action_create(self):
filename = self.resource.filename
xml_config_provider_config_dir = self.resource.conf_dir
configuration_attrs = {} if is_empty(
self.resource.configuration_attributes
) else self.resource.configuration_attributes
# |e - for html-like escaping of <,>,',"
config_content = InlineTemplate(
''' <configuration>
{% for key, value in configurations_dict|dictsort %}
<property>
<name>{{ key|e }}</name>
<value>{{ resource_management.core.source.InlineTemplate(unicode(value)).get_content() |e }}</value>
{%- if not configuration_attrs is none -%}
{%- for attrib_name, attrib_occurances in configuration_attrs.items() -%}
{%- for property_name, attrib_value in attrib_occurances.items() -%}
{% if property_name == key and attrib_name %}
<{{attrib_name|e}}>{{attrib_value|e}}</{{attrib_name|e}}>
{%- endif -%}
{%- endfor -%}
{%- endfor -%}
{%- endif %}
</property>
{% endfor %}
</configuration>''',
extra_imports=[
time, resource_management, resource_management.core,
resource_management.core.source
],
configurations_dict=self.resource.configurations,
configuration_attrs=configuration_attrs)
xml_config_dest_file_path = os.path.join(
xml_config_provider_config_dir, filename)
Logger.info("Generating config: {0}".format(xml_config_dest_file_path))
File(xml_config_dest_file_path,
content=config_content,
owner=self.resource.owner,
group=self.resource.group,
mode=self.resource.mode,
encoding=self.resource.encoding)
def convert_properties_to_dict(jdk64_home, nifi_registry_props_file,
nifi_registry_bootstrap):
dict = {}
if sudo.path_isfile(nifi_registry_props_file):
encrypt_tool_script = nifi_toolkit_util.get_toolkit_script(
'encrypt-config.sh')
File(encrypt_tool_script, mode=0755)
command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' ' + encrypt_tool_script + ' --nifiRegistry --decrypt -r ' + nifi_registry_props_file + ' -b ' + nifi_registry_bootstrap
code, out = shell.call(command, quiet=True, logoutput=False)
lines = out.split('\n')
for line in lines:
props = line.rstrip().split('=')
if len(props) == 2:
dict[props[0]] = props[1]
elif len(props) == 1:
dict[props[0]] = ''
return dict
def setup_configuration_file_for_required_plugins(component_user, component_group, create_core_site_path,
configurations={}, configuration_attributes={}, file_name='core-site.xml',
xml_include_file=None, xml_include_file_content=None):
XmlConfig(file_name,
conf_dir = create_core_site_path,
configurations = configurations,
configuration_attributes = configuration_attributes,
owner = component_user,
group = component_group,
mode = 0644,
xml_include_file = xml_include_file
)
if xml_include_file_content:
File(xml_include_file,
owner=component_user,
group=component_group,
content=xml_include_file_content
)
def check_nifi_portal_with_toolkit(urls, jdk64_home, nifi_dir,
nifi_bootstrap, toolkit_tmp_dir,
stack_version_buildnum):
node_manager_script = nifi_toolkit_util_common.get_toolkit_script(
'node-manager.sh', toolkit_tmp_dir, stack_version_buildnum)
File(node_manager_script, mode=0755)
command = 'ambari-sudo.sh JAVA_HOME=' + jdk64_home + ' ' + node_manager_script + ' -d ' + nifi_dir + ' -b ' + nifi_bootstrap + ' -o status -u "' + ','.join(
urls) + '"'
code, out = shell.call(command, quiet=True, logoutput=False)
if code > 0:
raise Fail(
"Call to admin-toolkit encountered error: {0}".format(out))
else:
if out.find('did not complete due to exception') > -1:
raise Fail(
"Error connecting to one or more nifi nodes: {0}".format(
out))
def create_topology_script():
import params
path = params.net_topology_script_file_path
parent_dir = os.path.dirname(path)
# only create the parent directory and set its permission if it does not exist
if not os.path.exists(parent_dir):
Directory(parent_dir,
create_parents=True,
owner=params.hdfs_user,
group=params.user_group)
# installing the topology script to the specified location
File(
path,
content=StaticFile('topology_script.py'),
mode=0755,
only_if=format("test -d {net_topology_script_dir}"),
)
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user,
component_group, java_home):
stack_root = Script.get_stack_root()
cred_lib_path = format(
'{stack_root}/{stack_version}/ranger-{service_name}-plugin/install/lib/*'
)
cred_setup_prefix = (format(
'{stack_root}/{stack_version}/ranger-{service_name}-plugin/ranger_credential_helper.py'
), '-l', cred_lib_path)
if audit_db_is_enabled:
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'auditDBCred', '-v',
PasswordString(xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': java_home},
logoutput=True,
sudo=True)
File(credential_file,
owner=component_user,
group=component_group,
mode=0640)
def test_action_create_non_existent_file(self, isdir_mock, exists_mock,
open_mock, ensure_mock):
"""
Tests if 'create' action create new non existent file and write proper data
"""
isdir_mock.side_effect = [False, True]
exists_mock.return_value = False
new_file = MagicMock()
open_mock.return_value = new_file
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
content='file-content')
open_mock.assert_called_with('/directory/file', 'wb')
new_file.__enter__().write.assert_called_with('file-content')
self.assertEqual(open_mock.call_count, 1)
ensure_mock.assert_called()
def test_action_create_non_existent_file(self, isdir_mock, exists_mock,
create_file_mock, read_file_mock,
ensure_mock):
"""
Tests if 'create' action create new non existent file and write proper data
"""
isdir_mock.side_effect = [False, True]
exists_mock.return_value = False
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
content='file-content')
create_file_mock.assert_called_once('/directory/file',
'file-content',
encoding=None,
on_file_created=ANY)
ensure_mock.assert_called()
def oozie(is_server=False):
import params
XmlConfig(
"oozie-site.xml",
conf_dir=params.oozie_conf_dir,
configurations=params.config['configurations']['oozie-site'],
owner=params.oozie_user,
mode='f',
configuration_attributes=params.config['configuration_attributes']
['oozie-site'])
File(os.path.join(params.oozie_conf_dir, "oozie-env.cmd"),
owner=params.oozie_user,
content=InlineTemplate(params.oozie_env_cmd_template))
Directory(
params.oozie_tmp_dir,
owner=params.oozie_user,
recursive=True,
)
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_root, "extra_libs", "sqljdbc4.jar"))
webapps_sqljdbc_path = os.path.join(params.oozie_home, "oozie-server",
"webapps", "oozie", "WEB-INF", "lib",
"sqljdbc4.jar")
if os.path.isfile(webapps_sqljdbc_path):
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"), webapps_sqljdbc_path)
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "share", "lib", "oozie",
"sqljdbc4.jar"))
download_file(
os.path.join(params.config['hostLevelParams']['jdk_location'],
"sqljdbc4.jar"),
os.path.join(params.oozie_home, "temp", "WEB-INF", "lib",
"sqljdbc4.jar"))
def service_check(self, env):
import params
env.set_params(params)
unique = get_unique_id_and_date()
File("/tmp/wordCount.jar", content=StaticFile("wordCount.jar"))
cmd = format(
"storm jar /tmp/wordCount.jar storm.starter.WordCountTopology WordCount{unique} -c nimbus.host={nimbus_host}"
)
Execute(cmd,
logoutput=True,
path=params.storm_bin_dir,
user=params.storm_user)
Execute(format("storm kill WordCount{unique}"),
path=params.storm_bin_dir,
user=params.storm_user)
def test_action_create_encoding(self, isdir_mock, exists_mock,
create_file_mock, read_file_mock,
get_content_mock, ensure_mock):
isdir_mock.side_effect = [False, True]
content_mock = MagicMock()
old_content_mock = MagicMock()
get_content_mock.return_value = content_mock
read_file_mock.return_value = old_content_mock
exists_mock.return_value = True
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
content='file-content',
encoding="UTF-8")
read_file_mock.assert_called_with('/directory/file')
content_mock.encode.assert_called_with('UTF-8')
old_content_mock.decode.assert_called_with('UTF-8')
def setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password,
ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home):
cred_lib_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install/lib/*')
cred_setup_prefix = format('python /usr/hdp/{hdp_version}/ranger-{service_name}-plugin/ranger_credential_helper.py -l "{cred_lib_path}"')
if audit_db_is_enabled:
cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "auditDBCred" -v {xa_audit_db_password!p} -c 1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True)
cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "sslKeyStore" -v {ssl_keystore_password!p} -c 1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True)
cred_setup = format('{cred_setup_prefix} -f {credential_file} -k "sslTrustStore" -v {ssl_truststore_password!p} -c 1')
Execute(cred_setup, environment={'JAVA_HOME': java_home}, logoutput=True)
File(credential_file,
owner = component_user,
group = component_group
)
def action_create(self):
filename = self.resource.filename
comment_symbols = self.resource.comment_symbols
delimiter = self.resource.key_value_delimiter
properties = self.resource.properties
unsaved_values = properties.keys()
new_content_lines = []
if sudo.path_isfile(filename):
file_content = sudo.read_file(filename, encoding=self.resource.encoding)
new_content_lines += file_content.split('\n')
Logger.info(format("Modifying existing properties file: {filename}"))
for line_num in range(len(new_content_lines)):
line = new_content_lines[line_num]
if line.lstrip() and not line.lstrip()[0] in comment_symbols and delimiter in line:
in_var_name = line.split(delimiter)[0].strip()
in_var_value = line.split(delimiter)[1].strip()
if in_var_name in properties:
value = InlineTemplate(unicode(properties[in_var_name])).get_content().strip()
new_content_lines[line_num] = u"{0}{1}{2}".format(unicode(in_var_name), delimiter, value)
unsaved_values.remove(in_var_name)
else:
Logger.info(format("Creating new properties file as {filename} doesn't exist"))
for property_name in unsaved_values:
value = InlineTemplate(unicode(properties[property_name])).get_content().strip()
line = u"{0}{1}{2}".format(unicode(property_name), delimiter, value)
new_content_lines.append(line)
with Environment.get_instance_copy() as env:
File (filename,
content = u"\n".join(new_content_lines) + "\n",
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode,
encoding = self.resource.encoding,
)
def test_action_create_parent_dir_non_exist(self, isdir_mock,
dirname_mock):
"""
Tests if 'create' action fails when parent directory of path
doesn't exist
"""
isdir_mock.side_effect = [False, False]
dirname_mock.return_value = "/non_existent_directory"
try:
with Environment('/') as env:
File('/non_existent_directory/file',
action='create',
mode=0777,
content='file-content')
self.fail('Must fail on non existent parent directory')
except Fail as e:
self.assertEqual(
"Applying File['/non_existent_directory/file'] failed, parent directory /non_existent_directory doesn't exist",
str(e))
self.assertTrue(dirname_mock.called)
def create_topology_mapping():
import params
path = params.net_topology_mapping_data_file_path
parent_dir = os.path.dirname(path)
# only create the parent directory and set its permission if it does not exist
if not os.path.exists(parent_dir):
Directory(parent_dir,
create_parents=True,
owner=params.hdfs_user,
group=params.user_group)
# placing the mappings file in the same folder where the topology script is located
File(
path,
content=Template("topology_mappings.data.j2"),
owner=params.hdfs_user,
group=params.user_group,
# if there is no hadoop components, don't create the script
only_if=format("test -d {net_topology_script_dir}"),
)
def action_create(self):
filename = self.resource.filename
dir = self.resource.dir
if dir == None:
filepath = filename
else:
filepath = os.path.join(dir, filename)
config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
{% for key, value in properties_dict|dictsort %}
{{key}}={{value}}{% endfor %}
''', extra_imports=[time], properties_dict=self.resource.properties)
Logger.info(format("Generating properties file: {filepath}"))
with Environment.get_instance_copy() as env:
File (format("{filepath}"),
content = config_content,
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode
)
def action_create(self):
filename = self.resource.filename
dir = self.resource.dir
if dir == None:
filepath = filename
else:
filepath = os.path.join(dir, filename)
config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
{% for key, value in properties_dict|dictsort %}
{{key}}{{key_value_delimiter}}{{ resource_management.core.source.InlineTemplate(str(value)).get_content() }}{% endfor %}
''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], properties_dict=self.resource.properties, key_value_delimiter=self.resource.key_value_delimiter)
Logger.info(format("Generating properties file: {filepath}"))
File (format("{filepath}"),
content = config_content,
owner = self.resource.owner,
group = self.resource.group,
mode = self.resource.mode,
encoding = self.resource.encoding,
)
def oozie_smoke_shell_file(file_name):
import params
File(format("{tmp_dir}/{file_name}"),
content=StaticFile(file_name),
mode=0755)
if params.security_enabled:
sh_cmd = format(
"{tmp_dir}/{file_name} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled} {smokeuser_keytab} {kinit_path_local} {smokeuser_principal}"
)
else:
sh_cmd = format(
"{tmp_dir}/{file_name} {oozie_lib_dir} {conf_dir} {oozie_bin_dir} {hadoop_conf_dir} {hadoop_bin_dir} {smokeuser} {security_enabled}"
)
Execute(format("{tmp_dir}/{file_name}"),
command=sh_cmd,
path=params.execute_path,
tries=3,
try_sleep=5,
logoutput=True)
def test_action_delete_is_directory(self, isdir_mock, exist_mock,
unlink_mock):
"""
Tests if 'delete' action fails when path is directory
"""
isdir_mock.return_value = True
try:
with Environment('/') as env:
File('/directory/file',
action='delete',
mode=0777,
backup=False,
content='new-content')
self.fail("Should fail when deleting directory")
except Fail:
pass
self.assertEqual(isdir_mock.call_count, 1)
self.assertEqual(exist_mock.call_count, 0)
self.assertEqual(unlink_mock.call_count, 0)
def test_action_create_encoding(self, isdir_mock, exists_mock, open_mock,
get_content_mock, ensure_mock):
isdir_mock.side_effect = [False, True]
exists_mock.return_value = True
content_mock = MagicMock()
old_content_mock = MagicMock()
get_content_mock.return_value = content_mock
new_file = MagicMock()
open_mock.return_value = new_file
enter_file_mock = MagicMock()
enter_file_mock.read = MagicMock(return_value=old_content_mock)
new_file.__enter__ = MagicMock(return_value=enter_file_mock)
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
content='file-content',
encoding="UTF-8")
open_mock.assert_called_with('/directory/file', 'wb')
content_mock.encode.assert_called_with('UTF-8')
old_content_mock.decode.assert_called_with('UTF-8')
def test_attribute_replace(self, isdir_mock, exists_mock, open_mock,
ensure_mock):
"""
Tests 'replace' attribute
"""
isdir_mock.side_effect = [False, True]
old_file, new_file = MagicMock(), MagicMock()
open_mock.side_effect = [old_file, new_file]
old_file.read.return_value = 'old-content'
exists_mock.return_value = True
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
backup=False,
content='new-content',
replace=False)
old_file.read.assert_called()
self.assertEqual(new_file.__enter__().write.call_count, 0)
ensure_mock.assert_called()
self.assertEqual(open_mock.call_count, 0)
def test_action_create_replace(self, isdir_mock, exists_mock, open_mock,
ensure_mock):
"""
Tests if 'create' action rewrite existent file with new data
"""
isdir_mock.side_effect = [False, True]
old_file, new_file = MagicMock(), MagicMock()
open_mock.side_effect = [old_file, new_file]
old_file.read.return_value = 'old-content'
exists_mock.return_value = True
with Environment('/') as env:
File('/directory/file',
action='create',
mode=0777,
backup=False,
content='new-content')
old_file.read.assert_called()
new_file.__enter__().write.assert_called_with('new-content')
ensure_mock.assert_called()
self.assertEqual(open_mock.call_count, 2)
open_mock.assert_any_call('/directory/file', 'rb')
open_mock.assert_any_call('/directory/file', 'wb')
def setup_ranger_knox(upgrade_type=None):
import params
if params.enable_ranger_knox:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/knox",
type="directory",
action="create_on_execute",
owner=params.knox_user,
group=params.knox_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.namenode_hosts is not None and len(
params.namenode_hosts) > 1:
Logger.info(
'Ranger Knox plugin is enabled in NameNode HA environment along with audit to Hdfs enabled, creating hdfs-site.xml'
)
XmlConfig("hdfs-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.knox_user,
group=params.knox_group,
mode=0644)
else:
File(format('{knox_conf_dir}/hdfs-site.xml'),
action="delete")
if params.xml_configurations_supported:
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'knox-server',
'knox',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.knox_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_knox,
conf_dict=params.knox_conf_dir,
component_user=params.knox_user,
component_group=params.knox_group,
cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']
['ranger-knox-audit'],
plugin_audit_attributes=params.
config['configurationAttributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']
['ranger-knox-security'],
plugin_security_attributes=params.
config['configurationAttributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-knox-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.knox_principal_name
if params.security_enabled else None,
component_user_keytab=params.knox_keytab_path
if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin(
'knox-server',
'knox',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.knox_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_knox,
conf_dict=params.knox_conf_dir,
component_user=params.knox_user,
component_group=params.knox_group,
cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']
['ranger-knox-audit'],
plugin_audit_attributes=params.
config['configurationAttributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']
['ranger-knox-security'],
plugin_security_attributes=params.
config['configurationAttributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-knox-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.security_enabled:
if params.has_namenode:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.knox_user,
component_group=params.knox_group,
create_core_site_path=params.knox_conf_dir,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configurationAttributes']['core-site'],
file_name='core-site.xml')
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.knox_user,
component_group=params.knox_group,
create_core_site_path=params.knox_conf_dir,
configurations={
'hadoop.security.authentication':
'kerberos' if params.security_enabled else 'simple'
},
configuration_attributes={},
file_name='core-site.xml')
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Knox plugin is not enabled')
def hbase(name=None):
import params
# ensure that matching LZO libraries are installed for HBase
lzo_utils.install_lzo_if_needed()
Directory(params.etc_prefix_dir, mode=0755)
Directory(params.hbase_conf_dir,
owner=params.hbase_user,
group=params.user_group,
create_parents=True)
Directory(params.java_io_tmpdir, create_parents=True, mode=0777)
# If a file location is specified in ioengine parameter,
# ensure that directory exists. Otherwise create the
# directory with permissions assigned to hbase:hadoop.
ioengine_input = params.ioengine_param
if ioengine_input != None:
if ioengine_input.startswith("file:/"):
ioengine_fullpath = ioengine_input[5:]
ioengine_dir = os.path.dirname(ioengine_fullpath)
Directory(ioengine_dir,
owner=params.hbase_user,
group=params.user_group,
create_parents=True,
mode=0755)
parent_dir = os.path.dirname(params.tmp_dir)
# In case if we have several placeholders in path
while ("${" in parent_dir):
parent_dir = os.path.dirname(parent_dir)
if parent_dir != os.path.abspath(os.sep):
Directory(
parent_dir,
create_parents=True,
cd_access="a",
)
Execute(("chmod", "1777", parent_dir), sudo=True)
XmlConfig("hbase-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['hbase-site'],
configuration_attributes=params.config['configurationAttributes']
['hbase-site'],
owner=params.hbase_user,
group=params.user_group)
if check_stack_feature(StackFeature.PHOENIX_CORE_HDFS_SITE_REQUIRED,
params.version_for_stack_feature_checks):
XmlConfig(
"core-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configurationAttributes']
['core-site'],
owner=params.hbase_user,
group=params.user_group)
if 'hdfs-site' in params.config['configurations']:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.hbase_user,
group=params.user_group)
else:
File(format("{params.hbase_conf_dir}/hdfs-site.xml"), action="delete")
File(format("{params.hbase_conf_dir}/core-site.xml"), action="delete")
if 'hbase-policy' in params.config['configurations']:
XmlConfig(
"hbase-policy.xml",
conf_dir=params.hbase_conf_dir,
configurations=params.config['configurations']['hbase-policy'],
configuration_attributes=params.config['configurationAttributes']
['hbase-policy'],
owner=params.hbase_user,
group=params.user_group)
# Manually overriding ownership of file installed by hadoop package
else:
File(format("{params.hbase_conf_dir}/hbase-policy.xml"),
owner=params.hbase_user,
group=params.user_group)
File(
format("{hbase_conf_dir}/hbase-env.sh"),
owner=params.hbase_user,
content=InlineTemplate(params.hbase_env_sh_template),
group=params.user_group,
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hbase.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hbase.conf.j2"))
hbase_TemplateConfig(
params.metric_prop_file_name,
tag='GANGLIA-MASTER' if name == 'master' else 'GANGLIA-RS')
hbase_TemplateConfig('regionservers')
if params.security_enabled:
hbase_TemplateConfig(format("hbase_{name}_jaas.conf"))
if name != "client":
Directory(
params.pid_dir,
owner=params.hbase_user,
create_parents=True,
cd_access="a",
mode=0755,
)
Directory(
params.log_dir,
owner=params.hbase_user,
create_parents=True,
cd_access="a",
mode=0755,
)
if (params.log4j_props != None):
File(format("{params.hbase_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.hbase_user,
content=InlineTemplate(params.log4j_props))
elif (os.path.exists(format("{params.hbase_conf_dir}/log4j.properties"))):
File(format("{params.hbase_conf_dir}/log4j.properties"),
mode=0644,
group=params.user_group,
owner=params.hbase_user)
if name == "master" and params.default_fs:
if not params.hbase_hdfs_root_dir_protocol or params.hbase_hdfs_root_dir_protocol == urlparse(
params.default_fs).scheme:
params.HdfsResource(params.hbase_hdfs_root_dir,
type="directory",
action="create_on_execute",
owner=params.hbase_user)
params.HdfsResource(params.hbase_staging_dir,
type="directory",
action="create_on_execute",
owner=params.hbase_user,
mode=0711)
if params.create_hbase_home_directory:
params.HdfsResource(params.hbase_home_directory,
type="directory",
action="create_on_execute",
owner=params.hbase_user,
mode=0755)
params.HdfsResource(None, action="execute")
if name in ('master', 'regionserver') and not params.default_fs:
Directory(
params.hbase_staging_dir,
owner=params.hbase_user,
create_parents=True,
cd_access="a",
mode=0711,
)
if params.phoenix_enabled:
Package(params.phoenix_package,
retry_on_repo_unavailability=params.
agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)