def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal('2.3.0.0'):
conf_select.select(params.stack_name, "hadoop", params.version)
hdp_select.select("hadoop-hdfs-nfs3", params.version)
def pre_upgrade_restart(self, env, upgrade_type=None):
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal('2.3.0.0'):
conf_select.select(params.stack_name, "hadoop", params.version)
hdp_select.select("hadoop-hdfs-nfs3", params.version)
def pre_rolling_restart(self, env):
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.3"):
# phoenix uses hbase configs
conf_select.select(params.stack_name, "hbase", params.version)
hdp_select.select("phoenix-server", params.version)
def pre_rolling_restart(self, env):
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.3"):
# phoenix uses hbase configs
conf_select.select(params.stack_name, "hbase", params.version)
hdp_select.select("phoenix-server", params.version)
def pre_rolling_restart(self, env):
Logger.info("Executing Metastore Rolling Upgrade pre-restart")
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.3"):
self.upgrade_schema(env)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-metastore", params.version)
def pre_rolling_restart(self, env):
Logger.info("Executing Metastore Rolling Upgrade pre-restart")
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.3"):
self.upgrade_schema(env)
if params.version and compare_versions(format_hdp_stack_version(params.version), "2.2.0.0") >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-metastore", params.version)
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Metastore Stack Upgrade pre-restart")
import params
env.set_params(params)
is_stack_hdp_23 = Script.is_hdp_stack_greater_or_equal("2.3")
is_upgrade = params.upgrade_direction == Direction.UPGRADE
if is_stack_hdp_23 and is_upgrade:
self.upgrade_schema(env)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-metastore", params.version)
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Metastore Stack Upgrade pre-restart")
import params
env.set_params(params)
if Script.is_hdp_stack_greater_or_equal("2.3"):
# ensure that configurations are written out before trying to upgrade the schema
# since the schematool needs configs and doesn't know how to use the hive conf override
self.configure(env)
self.upgrade_schema(env)
if params.version and compare_versions(
format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-metastore", params.version)
def pre_upgrade_restart(self, env, upgrade_type=None):
Logger.info("Executing Metastore Stack Upgrade pre-restart")
import params
env.set_params(params)
is_stack_hdp_23 = Script.is_hdp_stack_greater_or_equal("2.3")
is_upgrade = params.upgrade_direction == Direction.UPGRADE
if is_stack_hdp_23 and is_upgrade:
self.upgrade_schema(env)
if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
conf_select.select(params.stack_name, "hive", params.version)
hdp_select.select("hive-metastore", params.version)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
if not Script.is_hdp_stack_greater_or_equal("2.3"):
Logger.info("Can only link configs for HDP-2.3 and higher.")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
for k, v in conf_select.PACKAGE_DIRS.iteritems():
_link_configs(k, json_version, v['conf_dir'], v['current_dir'])
def link_configs(struct_out_file):
"""
"""
if not Script.is_hdp_stack_greater_or_equal("2.3"):
Logger.info("")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info(
"Could not load 'version' from {0}".format(struct_out_file))
return
for k, v in conf_select.PACKAGE_DIRS.iteritems():
_link_configs(k, json_version, v)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
if not Script.is_hdp_stack_greater_or_equal("2.3"):
Logger.info("Can only link configs for HDP-2.3 and higher.")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info("Could not load 'version' from {0}".format(struct_out_file))
return
for k, v in conf_select.PACKAGE_DIRS.iteritems():
conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file):
"""
Links configs, only on a fresh install of HDP-2.3 and higher
"""
import params
if not Script.is_hdp_stack_greater_or_equal("2.3"):
Logger.info("Can only link configs for HDP-2.3 and higher.")
return
json_version = load_version(struct_out_file)
if not json_version:
Logger.info(
"Could not load 'version' from {0}".format(struct_out_file))
return
# On parallel command execution this should be executed by a single process at a time.
with FcntlBasedProcessLock(params.link_configs_lock_file,
enabled=params.is_parallel_execution_enabled,
skip_fcntl_failures=True):
for k, v in conf_select.PACKAGE_DIRS.iteritems():
conf_select.convert_conf_directories_to_symlinks(
k, json_version, v)
# hadoop default params
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
# upgrades would cause these directories to have a version instead of "current"
# which would cause a lot of problems when writing out hadoop-env.sh; instead
# force the use of "current" in the hook
hadoop_home = hdp_select.get_hadoop_dir("home", force_latest_on_upgrade=True)
hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True)
hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
hadoop_secure_dn_user = hdfs_user
hadoop_dir = "/etc/hadoop"
versioned_hdp_root = '/usr/hdp/current'
# HDP 2.2+ params
if Script.is_hdp_stack_greater_or_equal("2.2"):
mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
# not supported in HDP 2.2+
hadoop_conf_empty_dir = None
if not security_enabled:
hadoop_secure_dn_user = '******'
else:
dfs_dn_port = get_port(dfs_dn_addr)
dfs_dn_http_port = get_port(dfs_dn_http_addr)
dfs_dn_https_port = get_port(dfs_dn_https_addr)
# We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
if dfs_http_policy == "HTTPS_ONLY":
secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port)
elif dfs_http_policy == "HTTP_AND_HTTPS":
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
sudo = AMBARI_SUDO_BINARY
stack_name = default("/hostLevelParams/stack_name", None)
upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE)
version = default("/commandParams/version", None)
storm_component_home_dir = status_params.storm_component_home_dir
conf_dir = status_params.conf_dir
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
stack_is_hdp22_or_further = Script.is_hdp_stack_greater_or_equal("2.2")
# default hadoop params
rest_lib_dir = "/usr/lib/storm/contrib/storm-rest"
storm_bin_dir = "/usr/bin"
storm_lib_dir = "/usr/lib/storm/lib/"
# hadoop parameters for 2.2+
if stack_is_hdp22_or_further:
rest_lib_dir = format("{storm_component_home_dir}/contrib/storm-rest")
storm_bin_dir = format("{storm_component_home_dir}/bin")
storm_lib_dir = format("{storm_component_home_dir}/lib")
log4j_dir = format("{storm_component_home_dir}/log4j2")
storm_user = config['configurations']['storm-env']['storm_user']
log_dir = config['configurations']['storm-env']['storm_log_dir']
sudo = AMBARI_SUDO_BINARY
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
# current host stack version
current_version = default("/hostLevelParams/current_version", None)
# default hadoop params
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec")
hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
# HDP 2.2+ params
if Script.is_hdp_stack_greater_or_equal("2.2"):
mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
# not supported in HDP 2.2+
hadoop_conf_empty_dir = None
versioned_hdp_root = '/usr/hdp/current'
#security params
security_enabled = config['configurations']['cluster-env']['security_enabled']
#java params
java_home = config['hostLevelParams']['java_home']
#hadoop params
hdfs_log_dir_prefix = config['configurations']['hadoop-env'][
from resource_management.core.system import System
from ambari_commons.os_check import OSCheck
config = Script.get_config()
sudo = AMBARI_SUDO_BINARY
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
# default hadoop params
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec")
hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
# HDP 2.2+ params
if Script.is_hdp_stack_greater_or_equal("2.2"):
mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
# not supported in HDP 2.2+
hadoop_conf_empty_dir = None
versioned_hdp_root = '/usr/hdp/current'
#security params
security_enabled = config['configurations']['cluster-env']['security_enabled']
#java params
java_home = config['hostLevelParams']['java_home']
#hadoop params
hdfs_log_dir_prefix = config['configurations']['hadoop-env'][
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
version = default("/commandParams/version", None)
host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
xml_configurations_supported = config['configurations']['ranger-env'][
'xml_configurations_supported']
create_db_dbuser = config['configurations']['ranger-env']['create_db_dbuser']
stack_is_hdp22_or_further = Script.is_hdp_stack_greater_or_equal("2.2")
stack_is_hdp23_or_further = Script.is_hdp_stack_greater_or_equal("2.3")
downgrade_from_version = default("/commandParams/downgrade_from_version", None)
upgrade_direction = default("/commandParams/upgrade_direction", None)
ranger_conf = '/etc/ranger/admin/conf'
ranger_ugsync_conf = '/etc/ranger/usersync/conf'
if upgrade_direction == Direction.DOWNGRADE and compare_versions(
format_hdp_stack_version(version), '2.3') < 0:
stack_is_hdp22_or_further = True
stack_is_hdp23_or_further = False
ranger_home = '/usr/lib/ranger-admin'
ranger_conf = '/etc/ranger/admin/conf'
from resource_management.core.system import System
from ambari_commons.os_check import OSCheck
config = Script.get_config()
sudo = AMBARI_SUDO_BINARY
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
# default hadoop params
mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
hadoop_libexec_dir = hdp_select.get_hadoop_dir("libexec")
hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
# HDP 2.2+ params
if Script.is_hdp_stack_greater_or_equal("2.2"):
mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
# not supported in HDP 2.2+
hadoop_conf_empty_dir = None
versioned_hdp_root = '/usr/hdp/current'
#security params
security_enabled = config['configurations']['cluster-env']['security_enabled']
#java params
java_home = config['hostLevelParams']['java_home']
#hadoop params
hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
