def get_running_topologies(params):
Logger.info('Getting Running Storm Topologies from Storm REST Server')
Logger.info('Security enabled? ' + str(params.security_enabled))
# Want to sudo to the metron user and kinit as them so we aren't polluting root with Metron's Kerberos tickets.
# This is becuase we need to run a command with a return as the metron user. Sigh
negotiate = '--negotiate -u : ' if params.security_enabled else ''
cmd = ambari_format(
'curl --max-time 3 ' + negotiate + '{storm_rest_addr}/api/v1/topology/summary')
if params.security_enabled:
kinit(params.kinit_path_local,
params.metron_keytab_path,
params.metron_principal_name,
execute_user=params.metron_user)
Logger.info('Running cmd: ' + cmd)
return_code, stdout, stderr = get_user_call_output(cmd,
user=params.metron_user,
is_checked_call=False)
if (return_code != 0):
return {}
try:
stormjson = json.loads(stdout)
except ValueError, e:
Logger.info('Stdout: ' + str(stdout))
Logger.info('Stderr: ' + str(stderr))
Logger.exception(str(e))
return {}
def action_delayed(self, action_name, main_resource):
main_resource.assert_parameter_is_set('user')
if main_resource.resource.security_enabled:
main_resource.kinit()
nameservices = namenode_ha_utils.get_nameservices(
main_resource.resource.hdfs_site)
if not nameservices:
self.action_delayed_for_nameservice(None, action_name,
main_resource)
else:
for nameservice in nameservices:
try:
self.action_delayed_for_nameservice(
nameservice, action_name, main_resource)
except namenode_ha_utils.NoActiveNamenodeException as ex:
# one of ns can be down (during initial start forexample) no need to worry for federated cluster
if len(nameservices) > 1:
Logger.exception(
"Cannot run HdfsResource for nameservice {0}. Due to no active namenode present"
.format(nameservice))
else:
raise
def _cleanup_past_llap_package_dirs(self):
try:
import params
Logger.info("Determining previous run 'LLAP package' folder(s) to be deleted ....")
llap_package_folder_name_prefix = "llap-slider" # Package name is like : llap-sliderYYYY-MM-DD-HH:MM:SS
num_folders_to_retain = 3 # Hardcoding it as of now, as no considerable use was found to provide an env param.
file_names = [dir_name for dir_name in os.listdir(Script.get_tmp_dir())
if dir_name.startswith(llap_package_folder_name_prefix)]
file_names.sort()
del file_names[-num_folders_to_retain:] # Ignore 'num_folders_to_retain' latest package folders.
Logger.info("Previous run 'LLAP package' folder(s) to be deleted = {0}".format(file_names))
if file_names:
for path in file_names:
abs_path = Script.get_tmp_dir()+"/"+path
Directory(abs_path,
action = "delete",
ignore_failures = True
)
else:
Logger.info("No '{0}*' folder deleted.".format(llap_package_folder_name_prefix))
except:
Logger.exception("Exception while doing cleanup for past 'LLAP package(s)':")
def unlock(self):
"""
Unlocks the lock file descriptor.
"""
if not self.enabled:
return
import fcntl
Logger.info("Releasing the lock on {0}".format(self.lock_file_name))
if self.acquired:
try:
fcntl.lockf(self.lock_file, fcntl.LOCK_UN)
except:
if self.skip_fcntl_failures:
Logger.exception("Fcntl call raised an exception. The lock was not released. "
"Continuing as skip_fcntl_failures is set to True")
else:
raise
else:
self.acquired = False
try:
self.lock_file.close()
self.lock_file = None
except IOError:
Logger.warning("Failed to close {0}".format(self.lock_file_name))
def actionexecute(self, env):
Logger.info("Host checks started.")
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
report_file_handler_dict = {}
#print "CONFIG: " + str(config)
check_execute_list = config['commandParams']['check_execute_list']
if check_execute_list == '*BEFORE_CLEANUP_HOST_CHECKS*':
check_execute_list = BEFORE_CLEANUP_HOST_CHECKS
structured_output = {}
Logger.info("Check execute list: " + str(check_execute_list))
# check each of the commands; if an unknown exception wasn't handled
# by the functions, then produce a generic exit_code : 1
if CHECK_JAVA_HOME in check_execute_list:
try :
java_home_check_structured_output = self.execute_java_home_available_check(config)
structured_output[CHECK_JAVA_HOME] = java_home_check_structured_output
except Exception, exception:
Logger.exception("There was an unexpected error while checking for the Java home location: " + str(exception))
structured_output[CHECK_JAVA_HOME] = {"exit_code" : 1, "message": str(exception)}
class CheckHost(Script):
# Package prefixes that are used to find repos (then repos are used to find other packages)
PACKAGES = [
"^hadoop.*$",
"^zookeeper.*$",
"^webhcat.*$",
"^oozie.*$",
"^ambari.*$",
"^.+-manager-server-db.*$",
"^.+-manager-daemons.*$",
"^mahout[_\-]\d.*$",
"^spark.*$",
"^falcon.*$",
"^hbase.*$",
"^kafka.*$",
"^knox.*$",
"^slider.*$",
"^sqoop.*$",
"^storm.*$",
"^flume.*$",
"^hcatalog.*$",
"^phoenix.*$",
"^ranger.*$",
"^accumulo.*$",
"^hive_.*$",
"^pig[_\-.].*$" # there's a default 'pigz' package which we should avoid
]
# ignore packages from repos whose names start with these strings
IGNORE_PACKAGES_FROM_REPOS = ["installed"]
# ignore required packages
IGNORE_PACKAGES = [
"epel-release",
"ambari-server",
"ambari-agent",
"nagios",
# ganglia related:
"ganglia",
"libganglia",
"libconfuse",
"perl",
"rrdtool",
"python-rrdtool",
"gmetad",
"librrd",
"rrdcached"
]
# Additional packages to look for (search packages that start with these)
ADDITIONAL_PACKAGES = ["ambari-log4j"]
# ignore repos from the list of repos to be cleaned
IGNORE_REPOS = ["HDP-UTILS", "AMBARI", "BASE", "EXTRAS"]
def __init__(self):
self.reportFileHandler = HostCheckReportFileHandler()
self.pkg_provider = get_provider("Package")
def actionexecute(self, env):
Logger.info("Host checks started.")
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
report_file_handler_dict = {}
#print "CONFIG: " + str(config)
check_execute_list = config['commandParams']['check_execute_list']
if check_execute_list == '*BEFORE_CLEANUP_HOST_CHECKS*':
check_execute_list = BEFORE_CLEANUP_HOST_CHECKS
structured_output = {}
Logger.info("Check execute list: " + str(check_execute_list))
# check each of the commands; if an unknown exception wasn't handled
# by the functions, then produce a generic exit_code : 1
if CHECK_JAVA_HOME in check_execute_list:
try:
java_home_check_structured_output = self.execute_java_home_available_check(
config)
structured_output[
CHECK_JAVA_HOME] = java_home_check_structured_output
except Exception, exception:
Logger.exception(
"There was an unexpected error while checking for the Java home location: "
+ str(exception))
structured_output[CHECK_JAVA_HOME] = {
"exit_code": 1,
"message": str(exception)
}
if CHECK_DB_CONNECTION in check_execute_list:
try:
db_connection_check_structured_output = self.execute_db_connection_check(
config, tmp_dir)
structured_output[
CHECK_DB_CONNECTION] = db_connection_check_structured_output
except Exception, exception:
Logger.exception(
"There was an unknown error while checking database connectivity: "
+ str(exception))
structured_output[CHECK_DB_CONNECTION] = {
"exit_code": 1,
"message": str(exception)
}
def execute_db_connection_check(self, config, tmp_dir):
Logger.info("DB connection check started.")
# initialize needed data
ambari_server_hostname = config['commandParams']['ambari_server_host']
check_db_connection_jar_name = "DBConnectionVerification.jar"
jdk_location = config['commandParams']['jdk_location']
java_home = config['commandParams']['java_home']
db_name = config['commandParams']['db_name']
no_jdbc_error_message = None
if db_name == DB_MYSQL:
jdbc_driver_mysql_name = default(
"/hostLevelParams/custom_mysql_jdbc_name", None)
if not jdbc_driver_mysql_name:
no_jdbc_error_message = "The MySQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mysql --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_mysql_name
jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL
jdbc_name = jdbc_driver_mysql_name
elif db_name == DB_ORACLE:
jdbc_driver_oracle_name = default(
"/hostLevelParams/custom_oracle_jdbc_name", None)
if not jdbc_driver_oracle_name:
no_jdbc_error_message = "The Oracle JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=oracle --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_oracle_name
jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE
jdbc_name = jdbc_driver_oracle_name
elif db_name == DB_POSTGRESQL:
jdbc_driver_postgres_name = default(
"/hostLevelParams/custom_postgres_jdbc_name", None)
if not jdbc_driver_postgres_name:
no_jdbc_error_message = "The Postgres JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=postgres --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_postgres_name
jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL
jdbc_name = jdbc_driver_postgres_name
elif db_name == DB_MSSQL:
jdbc_driver_mssql_name = default(
"/hostLevelParams/custom_mssql_jdbc_name", None)
if not jdbc_driver_mssql_name:
no_jdbc_error_message = "The MSSQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mssql --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_mssql_name
jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL
jdbc_name = jdbc_driver_mssql_name
elif db_name == DB_SQLA:
jdbc_driver_sqla_name = default(
"/hostLevelParams/custom_sqlanywhere_jdbc_name", None)
if not jdbc_driver_sqla_name:
no_jdbc_error_message = "The SQLAnywhere JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=sqlanywhere --jdbc-driver=/path/to/jdbc_driver'."
else:
jdbc_url = jdk_location + jdbc_driver_sqla_name
jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA
jdbc_name = jdbc_driver_sqla_name
else:
no_jdbc_error_message = format(
"'{db_name}' database type not supported.")
if no_jdbc_error_message:
Logger.warning(no_jdbc_error_message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": no_jdbc_error_message
}
return db_connection_check_structured_output
db_connection_url = config['commandParams']['db_connection_url']
user_name = config['commandParams']['user_name']
user_passwd = config['commandParams']['user_passwd']
agent_cache_dir = os.path.abspath(
config["hostLevelParams"]["agentCacheDir"])
check_db_connection_url = jdk_location + check_db_connection_jar_name
jdbc_path = os.path.join(agent_cache_dir, jdbc_name)
class_path_delimiter = ":"
if db_name == DB_SQLA:
jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE
java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \
LIBS_PATH_IN_ARCHIVE_SQLA
else:
jdbc_jar_path = jdbc_path
java_library_path = agent_cache_dir
check_db_connection_path = os.path.join(agent_cache_dir,
check_db_connection_jar_name)
java_bin = "java"
if OSCheck.is_windows_family():
java_bin = "java.exe"
class_path_delimiter = ";"
java_exec = os.path.join(java_home, "bin", java_bin)
if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \
or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec):
message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \
"\n"
Logger.warning(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
environment = {"no_proxy": format("{ambari_server_hostname}")}
# download and install java if it doesn't exists
if not os.path.isfile(java_exec):
jdk_name = config['commandParams']['jdk_name']
jdk_url = "{0}/{1}".format(jdk_location, jdk_name)
jdk_download_target = os.path.join(agent_cache_dir, jdk_name)
java_dir = os.path.dirname(java_home)
try:
download_file(jdk_url, jdk_download_target)
except Exception, e:
message = "Error downloading JDK from Ambari Server resources. Check network access to " \
"Ambari Server.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
if jdk_name.endswith(".exe"):
install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format(
os_utils.quote_path(jdk_download_target),
os_utils.quote_path(java_home),
)
install_path = [java_dir]
try:
Execute(install_cmd, path=install_path)
except Exception, e:
message = "Error installing java.\n" + str(e)
Logger.exception(message)
db_connection_check_structured_output = {
"exit_code": 1,
"message": message
}
return db_connection_check_structured_output
"There was an unknown error while checking database connectivity: "
+ str(exception))
structured_output[CHECK_DB_CONNECTION] = {
"exit_code": 1,
"message": str(exception)
}
if CHECK_HOST_RESOLUTION in check_execute_list:
try:
host_resolution_structured_output = self.execute_host_resolution_check(
config)
structured_output[
CHECK_HOST_RESOLUTION] = host_resolution_structured_output
except Exception, exception:
Logger.exception(
"There was an unknown error while checking IP address lookups: "
+ str(exception))
structured_output[CHECK_HOST_RESOLUTION] = {
"exit_code": 1,
"message": str(exception)
}
if CHECK_LAST_AGENT_ENV in check_execute_list:
try:
last_agent_env_structured_output = self.execute_last_agent_env_check(
)
structured_output[
CHECK_LAST_AGENT_ENV] = last_agent_env_structured_output
except Exception, exception:
Logger.exception(
"There was an unknown error while checking last host environment details: "
+ str(exception))
def setup_ranger_yarn():
import params
if params.enable_ranger_yarn:
if params.retryAble:
Logger.info(
"YARN: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/yarn",
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.yarn_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for YARN Ranger plugin failed with error:\n{0}"
.format(err))
setup_ranger_plugin(
'hadoop-yarn-resourcemanager',
'yarn',
None,
None,
None,
None,
params.java64_home,
params.repo_name,
params.yarn_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_yarn,
conf_dict=params.hadoop_conf_dir,
component_user=params.yarn_user,
component_group=params.user_group,
cache_service_list=['yarn'],
plugin_audit_properties=params.config['configurations']
['ranger-yarn-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-yarn-audit'],
plugin_security_properties=params.config['configurations']
['ranger-yarn-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-yarn-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-yarn-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-yarn-policymgr-ssl'],
component_list=['hadoop-yarn-resourcemanager'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.rm_principal_name
if params.security_enabled else None,
component_user_keytab=params.rm_keytab
if params.security_enabled else None)
def setup_ranger_kafka():
import params
if params.enable_ranger_kafka:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.has_namenode and params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kafka",
type="directory",
action="create_on_execute",
owner=params.kafka_user,
group=params.kafka_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.is_ranger_kms_ssl_enabled:
Logger.info(
'Ranger KMS is ssl enabled, configuring ssl-client for hdfs audits.'
)
setup_configuration_file_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['ssl-client'],
configuration_attributes=params.
config['configurationAttributes']['ssl-client'],
file_name='ssl-client.xml')
else:
Logger.info(
'Ranger KMS is not ssl enabled, skipping ssl-client for hdfs audits.'
)
except Exception, err:
Logger.exception(
"Audit directory creation in DDPS for KAFKA Ranger plugin failed with error:\n{0}"
.format(err))
setup_ranger_plugin(
'kafka-broker',
'kafka',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.kerberos_security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.kafka_jaas_principal
if params.kerberos_security_enabled else None,
component_user_keytab=params.kafka_keytab_path
if params.kerberos_security_enabled else None)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination',
params.setup_ranger_env_sh_source,
params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True)
File(params.setup_ranger_env_sh_target,
owner=params.kafka_user,
group=params.user_group,
mode=0755)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.kerberos_security_enabled:
# sometimes this is a link for missing /etc/hdp directory, just remove link/file and create regular file.
Execute(
('rm', '-f', os.path.join(params.conf_dir, "core-site.xml")),
sudo=True)
if params.has_namenode:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configurationAttributes']['core-site'],
file_name='core-site.xml',
xml_include_file=params.
mount_table_xml_inclusion_file_full_path,
xml_include_file_content=params.mount_table_content)
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations={
'hadoop.security.authentication':
'kerberos'
if params.kerberos_security_enabled else 'simple'
},
configuration_attributes={},
file_name='core-site.xml')
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
def setup_ranger_hbase(upgrade_type=None, service_name="hbase-master"):
import params
if params.enable_ranger_hbase:
if params.retryAble:
Logger.info(
"HBase: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"HBase: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xa_audit_hdfs_is_enabled and service_name == 'hbase-master':
try:
params.HdfsResource(
"/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource(
"/ranger/audit/hbaseMaster",
type="directory",
action="create_on_execute",
owner=params.hbase_user,
group=params.hbase_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(
"/ranger/audit/hbaseRegional",
type="directory",
action="create_on_execute",
owner=params.hbase_user,
group=params.hbase_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for HBASE Ranger plugin failed with error:\n{0}"
.format(err))
api_version = 'v2'
setup_ranger_plugin(
'hbase-client',
'hbase',
None,
None,
None,
None,
params.java64_home,
params.repo_name,
params.hbase_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hbase,
conf_dict=params.hbase_conf_dir,
component_user=params.hbase_user,
component_group=params.user_group,
cache_service_list=['hbaseMaster', 'hbaseRegional'],
plugin_audit_properties=params.config['configurations']
['ranger-hbase-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-hbase-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hbase-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-hbase-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hbase-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-hbase-policymgr-ssl'],
component_list=[
'hbase-client', 'hbase-master', 'hbase-regionserver'
],
audit_db_is_enabled=False,
credential_file=params.credential_file,
xa_audit_db_password=None,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos
if params.security_enabled else None,
component_user_principal=params.ranger_hbase_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_hbase_keytab
if params.security_enabled else None)
def setup_ranger_hive_interactive(upgrade_type=None):
import params
if params.enable_ranger_hive:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Hive2: Setup ranger: command retry enabled thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Hive2: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/hive2",
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hive_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for HIVE2 Ranger plugin failed with error:\n{0}"
.format(err))
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'hive-server2',
'hive',
params.ranger_previous_jdbc_jar,
params.ranger_downloaded_custom_connector,
params.ranger_driver_curl_source,
params.ranger_driver_curl_target,
params.java64_home,
params.repo_name,
params.hive_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hive,
conf_dict=params.hive_server_interactive_conf_dir,
component_user=params.hive_user,
component_group=params.user_group,
cache_service_list=['hive-server2'],
plugin_audit_properties=params.config['configurations']
['ranger-hive-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-hive-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hive-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-hive-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hive-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-hive-policymgr-ssl'],
component_list=['hive-client', 'hive-metastore', 'hive-server2'],
audit_db_is_enabled=False,
credential_file=params.credential_file,
xa_audit_db_password=None,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version='v2',
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.hive_principal
if params.security_enabled else None,
component_user_keytab=params.hive_server2_keytab
if params.security_enabled else None)
def setup_ranger_knox(upgrade_type=None):
import params
if params.enable_ranger_knox:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info("Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/knox",
type="directory",
action="create_on_execute",
owner=params.knox_user,
group=params.knox_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception("Audit directory creation in DDPS for KNOX Ranger plugin failed with error:\n{0}".format(err))
if params.namenode_hosts is not None and len(params.namenode_hosts) > 1:
Logger.info('Ranger Knox plugin is enabled in NameNode HA environment along with audit to Hdfs enabled, creating hdfs-site.xml')
XmlConfig("hdfs-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configurationAttributes']['hdfs-site'],
owner=params.knox_user,
group=params.knox_group,
mode=0644
)
else:
File(format('{knox_conf_dir}/hdfs-site.xml'), action="delete")
api_version = 'v2'
setup_ranger_plugin('knox-server', 'knox', params.previous_jdbc_jar,
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.knox_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_knox, conf_dict=params.knox_conf_dir,
component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configurationAttributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configurationAttributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configurationAttributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
is_security_enabled = params.security_enabled,
is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
component_user_principal=params.knox_principal_name if params.security_enabled else None,
component_user_keytab=params.knox_keytab_path if params.security_enabled else None)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.security_enabled:
if params.has_namenode:
Logger.info("Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations")
setup_configuration_file_for_required_plugins(component_user = params.knox_user, component_group = params.knox_group,
create_core_site_path = params.knox_conf_dir, configurations = params.config['configurations']['core-site'],
configuration_attributes = params.config['configurationAttributes']['core-site'], file_name='core-site.xml',
xml_include_file=params.mount_table_xml_inclusion_file_full_path, xml_include_file_content=params.mount_table_content)
else:
Logger.info("Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations")
setup_configuration_file_for_required_plugins(component_user = params.knox_user, component_group = params.knox_group,
create_core_site_path = params.knox_conf_dir, configurations = { 'hadoop.security.authentication' : 'kerberos' if params.security_enabled else 'simple' },
configuration_attributes = {}, file_name='core-site.xml')
else:
Logger.info("Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations")
def setup_ranger_atlas(upgrade_type=None):
import params
if params.enable_ranger_atlas:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retry_enabled:
Logger.info(
"ATLAS: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"ATLAS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.has_namenode and params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.metadata_user,
group=params.user_group,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/atlas",
type="directory",
action="create_on_execute",
owner=params.metadata_user,
group=params.user_group,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.is_ranger_kms_ssl_enabled:
Logger.info(
'Ranger KMS is ssl enabled, configuring ssl-client for hdfs audits.'
)
setup_configuration_file_for_required_plugins(
component_user=params.metadata_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['ssl-client'],
configuration_attributes=params.
config['configurationAttributes']['ssl-client'],
file_name='ssl-client.xml')
else:
Logger.info(
'Ranger KMS is not ssl enabled, skipping ssl-client for hdfs audits.'
)
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for ATLAS Ranger plugin failed with error:\n{0}"
.format(err))
setup_ranger_plugin(
'atlas-server',
'atlas',
None,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.atlas_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_atlas,
conf_dict=params.conf_dir,
component_user=params.metadata_user,
component_group=params.user_group,
cache_service_list=['atlas'],
plugin_audit_properties=params.config['configurations']
['ranger-atlas-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-atlas-audit'],
plugin_security_properties=params.config['configurations']
['ranger-atlas-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-atlas-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-atlas-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-atlas-policymgr-ssl'],
component_list=['atlas-server'],
audit_db_is_enabled=False,
credential_file=params.credential_file,
xa_audit_db_password=None,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retry_enabled,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.atlas_jaas_principal
if params.security_enabled else None,
component_user_keytab=params.atlas_keytab_path
if params.security_enabled else None)
def setup_ranger_storm(upgrade_type=None):
"""
:param upgrade_type: Upgrade Type such as "rolling" or "nonrolling"
"""
import params
if params.enable_ranger_storm and params.security_enabled:
site_files_create_path = format(
'{storm_component_home_dir}/extlib-daemon/ranger-storm-plugin-impl/conf'
)
Directory(site_files_create_path,
owner=params.storm_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access='a')
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.has_namenode and params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/storm",
type="directory",
action="create_on_execute",
owner=params.storm_user,
group=params.storm_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.is_ranger_kms_ssl_enabled:
Logger.info(
'Ranger KMS is ssl enabled, configuring ssl-client for hdfs audits.'
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations=params.config['configurations']
['ssl-client'],
configuration_attributes=params.
config['configurationAttributes']['ssl-client'],
file_name='ssl-client.xml')
else:
Logger.info(
'Ranger KMS is not ssl enabled, skipping ssl-client for hdfs audits.'
)
except Exception, err:
Logger.exception(
"Audit directory creation in DDPS for STORM Ranger plugin failed with error:\n{0}"
.format(err))
api_version = 'v2'
setup_ranger_plugin(
'storm-nimbus',
'storm',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.storm_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_storm,
conf_dict=params.conf_dir,
component_user=params.storm_user,
component_group=params.user_group,
cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']
['ranger-storm-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']
['ranger-storm-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-storm-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.ranger_storm_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_storm_keytab
if params.security_enabled else None)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_storm and params.security_enabled:
if params.has_namenode:
mount_table_xml_inclusion_file_full_path = None
mount_table_content = None
if 'viewfs-mount-table' in params.config['configurations']:
xml_inclusion_file_name = 'viewfs-mount-table.xml'
mount_table = params.config['configurations'][
'viewfs-mount-table']
if 'content' in mount_table and mount_table[
'content'].strip():
mount_table_xml_inclusion_file_full_path = os.path.join(
site_files_create_path, xml_inclusion_file_name)
mount_table_content = mount_table['content']
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configuration_attributes']['core-site'],
file_name='core-site.xml',
xml_include_file=mount_table_xml_inclusion_file_full_path,
xml_include_file_content=mount_table_content)
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations={
'hadoop.security.authentication':
'kerberos' if params.security_enabled else 'simple'
},
configuration_attributes={},
file_name='core-site.xml')
if len(params.namenode_hosts) > 1:
Logger.info(
'Ranger Storm plugin is enabled along with security and NameNode is HA , creating hdfs-site.xml'
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
file_name='hdfs-site.xml')
else:
Logger.info(
'Ranger Storm plugin is not enabled or security is disabled, removing hdfs-site.xml'
)
File(format('{site_files_create_path}/hdfs-site.xml'),
action="delete")
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
def execute(self):
"""
Sets up logging;
Parses command parameters and executes method relevant to command type
"""
parser = OptionParser()
parser.add_option("-o", "--out-files-logging", dest="log_out_files", action="store_true",
help="use this option to enable outputting *.out files of the service pre-start")
(self.options, args) = parser.parse_args()
self.log_out_files = self.options.log_out_files
# parse arguments
if len(args) < 6:
print "Script expects at least 6 arguments"
print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
sys.exit(1)
self.command_name = str.lower(sys.argv[1])
self.command_data_file = sys.argv[2]
self.basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
self.load_structured_out()
self.logging_level = sys.argv[5]
Script.tmp_dir = sys.argv[6]
# optional script arguments for forcing https protocol and ca_certs file
if len(sys.argv) >= 8:
Script.force_https_protocol = sys.argv[7]
if len(sys.argv) >= 9:
Script.ca_cert_file_path = sys.argv[8]
logging_level_str = logging._levelNames[self.logging_level]
Logger.initialize_logger(__name__, logging_level=logging_level_str)
# on windows we need to reload some of env variables manually because there is no default paths for configs(like
# /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
# in agent, so other Script executions will not be able to access to new env variables
if OSCheck.is_windows_family():
reload_windows_env()
# !!! status commands re-use structured output files; if the status command doesn't update the
# the file (because it doesn't have to) then we must ensure that the file is reset to prevent
# old, stale structured output from a prior status command from being used
if self.command_name == "status":
Script.structuredOut = {}
self.put_structured_out({})
# make sure that script has forced https protocol and ca_certs file passed from agent
ensure_ssl_using_protocol(Script.get_force_https_protocol_name(), Script.get_ca_cert_file_path())
try:
with open(self.command_data_file) as f:
pass
Script.config = ConfigDictionary(json.load(f))
# load passwords here(used on windows to impersonate different users)
Script.passwords = {}
for k, v in _PASSWORD_MAP.iteritems():
if get_path_from_configuration(k, Script.config) and get_path_from_configuration(v, Script.config):
Script.passwords[get_path_from_configuration(k, Script.config)] = get_path_from_configuration(v, Script.config)
except IOError:
Logger.logger.exception("Can not read json file with command parameters: ")
sys.exit(1)
from resource_management.libraries.functions import lzo_utils
repo_tags_to_skip = set()
if not lzo_utils.is_gpl_license_accepted():
repo_tags_to_skip.add("GPL")
Script.repository_util = RepositoryUtil(Script.config, repo_tags_to_skip)
# Run class method depending on a command type
try:
method = self.choose_method_to_execute(self.command_name)
with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env:
env.config.download_path = Script.tmp_dir
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'pre', env)
method(env)
if not self.is_hook():
self.execute_prefix_function(self.command_name, 'post', env)
except Fail as ex:
ex.pre_raise()
raise
finally:
try:
if self.should_expose_component_version(self.command_name):
self.save_component_version_to_structured_out(self.command_name)
except:
Logger.exception("Reporting component version failed")
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal
) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.rangerkms_keytab,
params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.spengo_keytab,
params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(
params.config['configurations']['ranger-kms-audit'])
if params.plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
params.plugin_audit_password_property] = "crypted"
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
# remove plain-text password from xml configs
ranger_kms_policymgr_ssl_copy = {}
ranger_kms_policymgr_ssl_copy.update(
params.config['configurations']['ranger-kms-policymgr-ssl'])
for prop in params.kms_plugin_password_properties:
if prop in ranger_kms_policymgr_ssl_copy:
ranger_kms_policymgr_ssl_copy[prop] = "crypted"
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_policymgr_ssl_copy,
configuration_attributes=params.config['configurationAttributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
only_if=format("test -e {credential_file}"),
mode=0640)
dot_jceks_crc_file_path = os.path.join(
os.path.dirname(params.credential_file),
"." + os.path.basename(params.credential_file) + ".crc")
File(dot_jceks_crc_file_path,
owner=params.kms_user,
group=params.kms_group,
only_if=format("test -e {dot_jceks_crc_file_path}"),
mode=0640)
# create ranger kms audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kms",
type="directory",
action="create_on_execute",
owner=params.kms_user,
group=params.kms_group,
mode=0750,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for RANGER KMS Ranger plugin failed with error:\n{0}"
.format(err))
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info(
'Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml'
)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
else:
File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
def install_packages(self, env):
"""
List of packages that are required< by service is received from the server
as a command parameter. The method installs all packages
from this list
exclude_packages - list of regexes (possibly raw strings as well), the
packages which match the regex won't be installed.
NOTE: regexes don't have Python syntax, but simple package regexes which support only * and .* and ?
"""
config = self.get_config()
if 'host_sys_prepped' in config['hostLevelParams']:
# do not install anything on sys-prepped host
if config['hostLevelParams']['host_sys_prepped'] is True:
Logger.info("Node has all packages pre-installed. Skipping.")
return
pass
try:
package_list_str = config['hostLevelParams']['package_list']
agent_stack_retry_on_unavailability = bool(
config['hostLevelParams']
['agent_stack_retry_on_unavailability'])
agent_stack_retry_count = int(
config['hostLevelParams']['agent_stack_retry_count'])
pkg_provider = get_provider("Package")
try:
available_packages_in_repos = pkg_provider.get_available_packages_in_repos(
config['repositoryFile']['repositories'])
except Exception as err:
Logger.exception("Unable to load available packages")
available_packages_in_repos = []
if isinstance(package_list_str,
basestring) and len(package_list_str) > 0:
package_list = json.loads(package_list_str)
for package in package_list:
if self.check_package_condition(package):
name = self.get_package_from_available(
package['name'], available_packages_in_repos)
# HACK: On Windows, only install ambari-metrics packages using Choco Package Installer
# TODO: Update this once choco packages for hadoop are created. This is because, service metainfo.xml support
# <osFamily>any<osFamily> which would cause installation failure on Windows.
if OSCheck.is_windows_family():
if "ambari-metrics" in name:
Package(name)
else:
Package(name,
retry_on_repo_unavailability=
agent_stack_retry_on_unavailability,
retry_count=agent_stack_retry_count)
except KeyError:
pass # No reason to worry
if OSCheck.is_windows_family():
#TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented
hadoop_user = config["configurations"]["cluster-env"][
"hadoop.user.name"]
install_windows_msi(
config['hostLevelParams']['jdk_location'],
config["hostLevelParams"]["agentCacheDir"], [
"hdp-2.3.0.0.winpkg.msi", "hdp-2.3.0.0.cab",
"hdp-2.3.0.0-01.cab"
], hadoop_user, self.get_password(hadoop_user),
str(config['hostLevelParams']['stack_version']))
reload_windows_env()
